From 454b59f97d12041b8035f3cdef6c73b7277d36a4 Mon Sep 17 00:00:00 2001 From: denys-gif Date: Mon, 6 Jul 2026 13:52:13 +0100 Subject: [PATCH 01/19] fix: nats consumer reconnect --- clients/openframe-client/src/config/update_config.rs | 1 + .../src/listener/openframe_client_update_listener.rs | 11 ++++++----- .../src/listener/tool_agent_update_listener.rs | 11 ++++++----- .../listener/tool_installation_message_listener.rs | 11 ++++++----- .../src/listener/tool_uninstall_message_listener.rs | 11 ++++++----- 5 files changed, 25 insertions(+), 20 deletions(-) diff --git a/clients/openframe-client/src/config/update_config.rs b/clients/openframe-client/src/config/update_config.rs index f1dcc1657..63f76596b 100644 --- a/clients/openframe-client/src/config/update_config.rs +++ b/clients/openframe-client/src/config/update_config.rs @@ -18,5 +18,6 @@ pub const RECONNECTION_DELAY_MS: u64 = 5000; // 5 seconds // NATS message settings pub const CONSUMER_ACK_WAIT_SECS: u64 = 120; +pub const CONSUMER_IDLE_HEARTBEAT_SECS: u64 = 30; pub const CONSUMER_MAX_DELIVER: i64 = 10; // Maximum delivery attempts pub const UNINSTALL_CONSUMER_MAX_DELIVER: i64 = 20; // Larger budget: uninstall may defer behind a long install holding the tool lock diff --git a/clients/openframe-client/src/listener/openframe_client_update_listener.rs b/clients/openframe-client/src/listener/openframe_client_update_listener.rs index ba29c3148..b2e2a7e0f 100644 --- a/clients/openframe-client/src/listener/openframe_client_update_listener.rs +++ b/clients/openframe-client/src/listener/openframe_client_update_listener.rs @@ -1,7 +1,7 @@ use crate::config::update_config::{ - CONSUMER_ACK_WAIT_SECS, CONSUMER_CYCLE_PAUSE_MS, CONSUMER_MAX_DELIVER, - CONSUMER_RETRY_ATTEMPTS_PER_CYCLE, INITIAL_RETRY_DELAY_MS, MAX_RETRY_DELAY_MS, - RECONNECTION_DELAY_MS, + CONSUMER_ACK_WAIT_SECS, CONSUMER_CYCLE_PAUSE_MS, CONSUMER_IDLE_HEARTBEAT_SECS, + CONSUMER_MAX_DELIVER, CONSUMER_RETRY_ATTEMPTS_PER_CYCLE, INITIAL_RETRY_DELAY_MS, + MAX_RETRY_DELAY_MS, RECONNECTION_DELAY_MS, }; use crate::models::openframe_client_update_message::OpenFrameClientUpdateMessage; use crate::services::nats_connection_manager::NatsConnectionManager; @@ -80,8 +80,8 @@ impl OpenFrameClientUpdateListener { let message = match msg_result { Ok(msg) => msg, Err(e) => { - error!("Failed to receive message: {:#}", e); - continue; + error!("Message stream error, recreating consumer: {:#}", e); + return Err(anyhow::anyhow!("Message stream error: {}", e)); } }; @@ -231,6 +231,7 @@ impl OpenFrameClientUpdateListener { deliver_subject, durable_name: Some(durable_name), ack_wait: Duration::from_secs(CONSUMER_ACK_WAIT_SECS), + idle_heartbeat: Duration::from_secs(CONSUMER_IDLE_HEARTBEAT_SECS), deliver_policy: DeliverPolicy::New, max_deliver: CONSUMER_MAX_DELIVER, ..Default::default() diff --git a/clients/openframe-client/src/listener/tool_agent_update_listener.rs b/clients/openframe-client/src/listener/tool_agent_update_listener.rs index 2047b97e6..743d19f74 100644 --- a/clients/openframe-client/src/listener/tool_agent_update_listener.rs +++ b/clients/openframe-client/src/listener/tool_agent_update_listener.rs @@ -1,7 +1,7 @@ use crate::config::update_config::{ - CONSUMER_ACK_WAIT_SECS, CONSUMER_CYCLE_PAUSE_MS, CONSUMER_MAX_DELIVER, - CONSUMER_RETRY_ATTEMPTS_PER_CYCLE, INITIAL_RETRY_DELAY_MS, MAX_RETRY_DELAY_MS, - RECONNECTION_DELAY_MS, + CONSUMER_ACK_WAIT_SECS, CONSUMER_CYCLE_PAUSE_MS, CONSUMER_IDLE_HEARTBEAT_SECS, + CONSUMER_MAX_DELIVER, CONSUMER_RETRY_ATTEMPTS_PER_CYCLE, INITIAL_RETRY_DELAY_MS, + MAX_RETRY_DELAY_MS, RECONNECTION_DELAY_MS, }; use crate::models::tool_agent_update_message::ToolAgentUpdateMessage; use crate::services::nats_connection_manager::NatsConnectionManager; @@ -80,8 +80,8 @@ impl ToolAgentUpdateListener { let message = match msg_result { Ok(msg) => msg, Err(e) => { - error!("Failed to receive message: {:#}", e); - continue; + error!("Message stream error, recreating consumer: {:#}", e); + return Err(anyhow::anyhow!("Message stream error: {}", e)); } }; @@ -239,6 +239,7 @@ impl ToolAgentUpdateListener { deliver_subject, durable_name: Some(durable_name), ack_wait: Duration::from_secs(CONSUMER_ACK_WAIT_SECS), + idle_heartbeat: Duration::from_secs(CONSUMER_IDLE_HEARTBEAT_SECS), deliver_policy: DeliverPolicy::New, max_deliver: CONSUMER_MAX_DELIVER, ..Default::default() diff --git a/clients/openframe-client/src/listener/tool_installation_message_listener.rs b/clients/openframe-client/src/listener/tool_installation_message_listener.rs index fe12981c0..3c533ffcc 100644 --- a/clients/openframe-client/src/listener/tool_installation_message_listener.rs +++ b/clients/openframe-client/src/listener/tool_installation_message_listener.rs @@ -1,7 +1,7 @@ use crate::config::update_config::{ - CONSUMER_ACK_WAIT_SECS, CONSUMER_CYCLE_PAUSE_MS, CONSUMER_MAX_DELIVER, - CONSUMER_RETRY_ATTEMPTS_PER_CYCLE, INITIAL_RETRY_DELAY_MS, MAX_RETRY_DELAY_MS, - RECONNECTION_DELAY_MS, + CONSUMER_ACK_WAIT_SECS, CONSUMER_CYCLE_PAUSE_MS, CONSUMER_IDLE_HEARTBEAT_SECS, + CONSUMER_MAX_DELIVER, CONSUMER_RETRY_ATTEMPTS_PER_CYCLE, INITIAL_RETRY_DELAY_MS, + MAX_RETRY_DELAY_MS, RECONNECTION_DELAY_MS, }; use crate::models::tool_installation_message::ToolInstallationMessage; use crate::services::nats_connection_manager::NatsConnectionManager; @@ -79,8 +79,8 @@ impl ToolInstallationMessageListener { let message = match msg_result { Ok(msg) => msg, Err(e) => { - error!("Failed to receive message: {:#}", e); - continue; + error!("Message stream error, recreating consumer: {:#}", e); + return Err(anyhow::anyhow!("Message stream error: {}", e)); } }; @@ -231,6 +231,7 @@ impl ToolInstallationMessageListener { deliver_subject, durable_name: Some(durable_name), ack_wait: Duration::from_secs(CONSUMER_ACK_WAIT_SECS), + idle_heartbeat: Duration::from_secs(CONSUMER_IDLE_HEARTBEAT_SECS), max_deliver: CONSUMER_MAX_DELIVER, ..Default::default() } diff --git a/clients/openframe-client/src/listener/tool_uninstall_message_listener.rs b/clients/openframe-client/src/listener/tool_uninstall_message_listener.rs index edbf95280..54bdc52bb 100644 --- a/clients/openframe-client/src/listener/tool_uninstall_message_listener.rs +++ b/clients/openframe-client/src/listener/tool_uninstall_message_listener.rs @@ -1,7 +1,7 @@ use crate::config::update_config::{ - CONSUMER_ACK_WAIT_SECS, CONSUMER_CYCLE_PAUSE_MS, CONSUMER_RETRY_ATTEMPTS_PER_CYCLE, - INITIAL_RETRY_DELAY_MS, MAX_RETRY_DELAY_MS, RECONNECTION_DELAY_MS, - UNINSTALL_CONSUMER_MAX_DELIVER, + CONSUMER_ACK_WAIT_SECS, CONSUMER_CYCLE_PAUSE_MS, CONSUMER_IDLE_HEARTBEAT_SECS, + CONSUMER_RETRY_ATTEMPTS_PER_CYCLE, INITIAL_RETRY_DELAY_MS, MAX_RETRY_DELAY_MS, + RECONNECTION_DELAY_MS, UNINSTALL_CONSUMER_MAX_DELIVER, }; use crate::models::ToolUninstallMessage; use crate::services::nats_connection_manager::NatsConnectionManager; @@ -83,8 +83,8 @@ impl ToolUninstallMessageListener { let message = match msg_result { Ok(msg) => msg, Err(e) => { - error!("Failed to receive message: {:#}", e); - continue; + error!("Message stream error, recreating consumer: {:#}", e); + return Err(anyhow::anyhow!("Message stream error: {}", e)); } }; @@ -256,6 +256,7 @@ impl ToolUninstallMessageListener { deliver_subject, durable_name: Some(durable_name), ack_wait: Duration::from_secs(CONSUMER_ACK_WAIT_SECS), + idle_heartbeat: Duration::from_secs(CONSUMER_IDLE_HEARTBEAT_SECS), max_deliver: UNINSTALL_CONSUMER_MAX_DELIVER, ..Default::default() } From 478483dad21329539c73fd08d19cd922da4f9b8a Mon Sep 17 00:00:00 2001 From: denys-gif Date: Mon, 6 Jul 2026 16:51:58 +0100 Subject: [PATCH 02/19] fix: revert heartbeat --- clients/openframe-client/src/config/update_config.rs | 1 - .../src/listener/openframe_client_update_listener.rs | 11 +++++------ .../src/listener/tool_agent_update_listener.rs | 11 +++++------ .../listener/tool_installation_message_listener.rs | 11 +++++------ .../src/listener/tool_uninstall_message_listener.rs | 11 +++++------ 5 files changed, 20 insertions(+), 25 deletions(-) diff --git a/clients/openframe-client/src/config/update_config.rs b/clients/openframe-client/src/config/update_config.rs index 63f76596b..f1dcc1657 100644 --- a/clients/openframe-client/src/config/update_config.rs +++ b/clients/openframe-client/src/config/update_config.rs @@ -18,6 +18,5 @@ pub const RECONNECTION_DELAY_MS: u64 = 5000; // 5 seconds // NATS message settings pub const CONSUMER_ACK_WAIT_SECS: u64 = 120; -pub const CONSUMER_IDLE_HEARTBEAT_SECS: u64 = 30; pub const CONSUMER_MAX_DELIVER: i64 = 10; // Maximum delivery attempts pub const UNINSTALL_CONSUMER_MAX_DELIVER: i64 = 20; // Larger budget: uninstall may defer behind a long install holding the tool lock diff --git a/clients/openframe-client/src/listener/openframe_client_update_listener.rs b/clients/openframe-client/src/listener/openframe_client_update_listener.rs index b2e2a7e0f..ba29c3148 100644 --- a/clients/openframe-client/src/listener/openframe_client_update_listener.rs +++ b/clients/openframe-client/src/listener/openframe_client_update_listener.rs @@ -1,7 +1,7 @@ use crate::config::update_config::{ - CONSUMER_ACK_WAIT_SECS, CONSUMER_CYCLE_PAUSE_MS, CONSUMER_IDLE_HEARTBEAT_SECS, - CONSUMER_MAX_DELIVER, CONSUMER_RETRY_ATTEMPTS_PER_CYCLE, INITIAL_RETRY_DELAY_MS, - MAX_RETRY_DELAY_MS, RECONNECTION_DELAY_MS, + CONSUMER_ACK_WAIT_SECS, CONSUMER_CYCLE_PAUSE_MS, CONSUMER_MAX_DELIVER, + CONSUMER_RETRY_ATTEMPTS_PER_CYCLE, INITIAL_RETRY_DELAY_MS, MAX_RETRY_DELAY_MS, + RECONNECTION_DELAY_MS, }; use crate::models::openframe_client_update_message::OpenFrameClientUpdateMessage; use crate::services::nats_connection_manager::NatsConnectionManager; @@ -80,8 +80,8 @@ impl OpenFrameClientUpdateListener { let message = match msg_result { Ok(msg) => msg, Err(e) => { - error!("Message stream error, recreating consumer: {:#}", e); - return Err(anyhow::anyhow!("Message stream error: {}", e)); + error!("Failed to receive message: {:#}", e); + continue; } }; @@ -231,7 +231,6 @@ impl OpenFrameClientUpdateListener { deliver_subject, durable_name: Some(durable_name), ack_wait: Duration::from_secs(CONSUMER_ACK_WAIT_SECS), - idle_heartbeat: Duration::from_secs(CONSUMER_IDLE_HEARTBEAT_SECS), deliver_policy: DeliverPolicy::New, max_deliver: CONSUMER_MAX_DELIVER, ..Default::default() diff --git a/clients/openframe-client/src/listener/tool_agent_update_listener.rs b/clients/openframe-client/src/listener/tool_agent_update_listener.rs index 743d19f74..2047b97e6 100644 --- a/clients/openframe-client/src/listener/tool_agent_update_listener.rs +++ b/clients/openframe-client/src/listener/tool_agent_update_listener.rs @@ -1,7 +1,7 @@ use crate::config::update_config::{ - CONSUMER_ACK_WAIT_SECS, CONSUMER_CYCLE_PAUSE_MS, CONSUMER_IDLE_HEARTBEAT_SECS, - CONSUMER_MAX_DELIVER, CONSUMER_RETRY_ATTEMPTS_PER_CYCLE, INITIAL_RETRY_DELAY_MS, - MAX_RETRY_DELAY_MS, RECONNECTION_DELAY_MS, + CONSUMER_ACK_WAIT_SECS, CONSUMER_CYCLE_PAUSE_MS, CONSUMER_MAX_DELIVER, + CONSUMER_RETRY_ATTEMPTS_PER_CYCLE, INITIAL_RETRY_DELAY_MS, MAX_RETRY_DELAY_MS, + RECONNECTION_DELAY_MS, }; use crate::models::tool_agent_update_message::ToolAgentUpdateMessage; use crate::services::nats_connection_manager::NatsConnectionManager; @@ -80,8 +80,8 @@ impl ToolAgentUpdateListener { let message = match msg_result { Ok(msg) => msg, Err(e) => { - error!("Message stream error, recreating consumer: {:#}", e); - return Err(anyhow::anyhow!("Message stream error: {}", e)); + error!("Failed to receive message: {:#}", e); + continue; } }; @@ -239,7 +239,6 @@ impl ToolAgentUpdateListener { deliver_subject, durable_name: Some(durable_name), ack_wait: Duration::from_secs(CONSUMER_ACK_WAIT_SECS), - idle_heartbeat: Duration::from_secs(CONSUMER_IDLE_HEARTBEAT_SECS), deliver_policy: DeliverPolicy::New, max_deliver: CONSUMER_MAX_DELIVER, ..Default::default() diff --git a/clients/openframe-client/src/listener/tool_installation_message_listener.rs b/clients/openframe-client/src/listener/tool_installation_message_listener.rs index 3c533ffcc..fe12981c0 100644 --- a/clients/openframe-client/src/listener/tool_installation_message_listener.rs +++ b/clients/openframe-client/src/listener/tool_installation_message_listener.rs @@ -1,7 +1,7 @@ use crate::config::update_config::{ - CONSUMER_ACK_WAIT_SECS, CONSUMER_CYCLE_PAUSE_MS, CONSUMER_IDLE_HEARTBEAT_SECS, - CONSUMER_MAX_DELIVER, CONSUMER_RETRY_ATTEMPTS_PER_CYCLE, INITIAL_RETRY_DELAY_MS, - MAX_RETRY_DELAY_MS, RECONNECTION_DELAY_MS, + CONSUMER_ACK_WAIT_SECS, CONSUMER_CYCLE_PAUSE_MS, CONSUMER_MAX_DELIVER, + CONSUMER_RETRY_ATTEMPTS_PER_CYCLE, INITIAL_RETRY_DELAY_MS, MAX_RETRY_DELAY_MS, + RECONNECTION_DELAY_MS, }; use crate::models::tool_installation_message::ToolInstallationMessage; use crate::services::nats_connection_manager::NatsConnectionManager; @@ -79,8 +79,8 @@ impl ToolInstallationMessageListener { let message = match msg_result { Ok(msg) => msg, Err(e) => { - error!("Message stream error, recreating consumer: {:#}", e); - return Err(anyhow::anyhow!("Message stream error: {}", e)); + error!("Failed to receive message: {:#}", e); + continue; } }; @@ -231,7 +231,6 @@ impl ToolInstallationMessageListener { deliver_subject, durable_name: Some(durable_name), ack_wait: Duration::from_secs(CONSUMER_ACK_WAIT_SECS), - idle_heartbeat: Duration::from_secs(CONSUMER_IDLE_HEARTBEAT_SECS), max_deliver: CONSUMER_MAX_DELIVER, ..Default::default() } diff --git a/clients/openframe-client/src/listener/tool_uninstall_message_listener.rs b/clients/openframe-client/src/listener/tool_uninstall_message_listener.rs index 54bdc52bb..edbf95280 100644 --- a/clients/openframe-client/src/listener/tool_uninstall_message_listener.rs +++ b/clients/openframe-client/src/listener/tool_uninstall_message_listener.rs @@ -1,7 +1,7 @@ use crate::config::update_config::{ - CONSUMER_ACK_WAIT_SECS, CONSUMER_CYCLE_PAUSE_MS, CONSUMER_IDLE_HEARTBEAT_SECS, - CONSUMER_RETRY_ATTEMPTS_PER_CYCLE, INITIAL_RETRY_DELAY_MS, MAX_RETRY_DELAY_MS, - RECONNECTION_DELAY_MS, UNINSTALL_CONSUMER_MAX_DELIVER, + CONSUMER_ACK_WAIT_SECS, CONSUMER_CYCLE_PAUSE_MS, CONSUMER_RETRY_ATTEMPTS_PER_CYCLE, + INITIAL_RETRY_DELAY_MS, MAX_RETRY_DELAY_MS, RECONNECTION_DELAY_MS, + UNINSTALL_CONSUMER_MAX_DELIVER, }; use crate::models::ToolUninstallMessage; use crate::services::nats_connection_manager::NatsConnectionManager; @@ -83,8 +83,8 @@ impl ToolUninstallMessageListener { let message = match msg_result { Ok(msg) => msg, Err(e) => { - error!("Message stream error, recreating consumer: {:#}", e); - return Err(anyhow::anyhow!("Message stream error: {}", e)); + error!("Failed to receive message: {:#}", e); + continue; } }; @@ -256,7 +256,6 @@ impl ToolUninstallMessageListener { deliver_subject, durable_name: Some(durable_name), ack_wait: Duration::from_secs(CONSUMER_ACK_WAIT_SECS), - idle_heartbeat: Duration::from_secs(CONSUMER_IDLE_HEARTBEAT_SECS), max_deliver: UNINSTALL_CONSUMER_MAX_DELIVER, ..Default::default() } From 25f2feb20e4bc9f274d82727dfc73ff0c9fe136e Mon Sep 17 00:00:00 2001 From: mikhailm-coder Date: Tue, 7 Jul 2026 21:35:52 +0300 Subject: [PATCH 03/19] feat: add token_refresh_run_manager (#2016) Co-authored-by: Claude Opus 4.8 Co-authored-by: denys-gif --- clients/openframe-client/src/lib.rs | 10 ++ .../services/agent_configuration_service.rs | 13 ++- clients/openframe-client/src/services/mod.rs | 2 + .../src/services/shared_token_service.rs | 12 +-- .../src/services/token_refresh_run_manager.rs | 102 ++++++++++++++++++ clients/openframe-client/src/utils.rs | 3 + clients/openframe-client/src/utils/fs.rs | 20 ++++ clients/openframe-client/src/utils/jwt.rs | 56 ++++++++++ 8 files changed, 207 insertions(+), 11 deletions(-) create mode 100644 clients/openframe-client/src/services/token_refresh_run_manager.rs create mode 100644 clients/openframe-client/src/utils/fs.rs create mode 100644 clients/openframe-client/src/utils/jwt.rs diff --git a/clients/openframe-client/src/lib.rs b/clients/openframe-client/src/lib.rs index f8d453859..1d2e1bfc8 100644 --- a/clients/openframe-client/src/lib.rs +++ b/clients/openframe-client/src/lib.rs @@ -65,6 +65,7 @@ use crate::services::openframe_client_info_service::OpenFrameClientInfoService; use crate::services::openframe_client_update_service::OpenFrameClientUpdateService; use crate::services::registration_processor::RegistrationProcessor; use crate::services::shared_token_service::SharedTokenService; +use crate::services::token_refresh_run_manager::TokenRefreshRunManager; use crate::services::tool_agent_update_service::ToolAgentUpdateService; use crate::services::tool_connection_message_publisher::ToolConnectionMessagePublisher; use crate::services::tool_connection_service::ToolConnectionService; @@ -152,6 +153,7 @@ pub struct Client { command_execution_listener: ExecutionListener, script_execution_listener: ExecutionListener, tool_run_manager: ToolRunManager, + token_refresh_run_manager: TokenRefreshRunManager, mesh_self_heal_service: MeshSelfHealService, tool_connection_processing_manager: ToolConnectionProcessingManager, machine_heartbeat_run_manager: MachineHeartbeatRunManager, @@ -251,6 +253,11 @@ impl Client { let auth_processor = InitialAuthenticationProcessor::new(auth_service.clone(), config_service.clone()); + // Initialize proactive token refresh run manager (keeps shared_token.enc valid + // independent of NATS reconnects) + let token_refresh_run_manager = + TokenRefreshRunManager::new(auth_service.clone(), config_service.clone()); + // Initialize NATS connection manager let ws_url = format!("wss://{}", initial_configuration_service.get_server_url()?); let tls_config_provider = @@ -469,6 +476,7 @@ impl Client { command_execution_listener, script_execution_listener, tool_run_manager, + token_refresh_run_manager, mesh_self_heal_service, tool_connection_processing_manager, machine_heartbeat_run_manager, @@ -498,6 +506,8 @@ impl Client { self.registration_processor.process().await?; self.auth_processor.process().await?; + self.token_refresh_run_manager.start(); + // Connect to NATS self.nats_connection_manager.connect().await?; diff --git a/clients/openframe-client/src/services/agent_configuration_service.rs b/clients/openframe-client/src/services/agent_configuration_service.rs index e90c46511..34a5e076c 100644 --- a/clients/openframe-client/src/services/agent_configuration_service.rs +++ b/clients/openframe-client/src/services/agent_configuration_service.rs @@ -1,6 +1,8 @@ use anyhow::{Context, Result}; use std::fs; use std::path::PathBuf; +use std::sync::Arc; +use tokio::sync::Mutex; use crate::models::AgentConfiguration; use crate::platform::directories::DirectoryManager; @@ -8,6 +10,8 @@ use crate::platform::directories::DirectoryManager; #[derive(Clone)] pub struct AgentConfigurationService { config_file_path: PathBuf, + // Serializes read-modify-write of the config so concurrent savers can't lose each other's fields. + write_lock: Arc>, } impl AgentConfigurationService { @@ -18,7 +22,10 @@ impl AgentConfigurationService { .ensure_directories() .with_context(|| "Failed to ensure secured directory exists")?; - Ok(Self { config_file_path }) + Ok(Self { + config_file_path, + write_lock: Arc::new(Mutex::new(())), + }) } pub async fn save_registration_data( @@ -27,6 +34,7 @@ impl AgentConfigurationService { client_id: String, client_secret: String, ) -> Result<()> { + let _guard = self.write_lock.lock().await; let mut config = self.get()?; config.machine_id = machine_id; config.client_id = client_id; @@ -38,6 +46,7 @@ impl AgentConfigurationService { } pub async fn update_tokens(&self, access_token: String, refresh_token: String) -> Result<()> { + let _guard = self.write_lock.lock().await; let mut config = self.get()?; config.access_token = access_token; config.refresh_token = refresh_token; @@ -85,7 +94,7 @@ impl AgentConfigurationService { let json_content = serde_json::to_string_pretty(config) .context("Failed to serialize agent configuration to JSON")?; - fs::write(&self.config_file_path, json_content) + crate::utils::fs::atomic_write(&self.config_file_path, json_content) .with_context(|| format!("Failed to write config file: {:?}", self.config_file_path))?; Ok(()) diff --git a/clients/openframe-client/src/services/mod.rs b/clients/openframe-client/src/services/mod.rs index 0bf1eac11..ac87a3802 100644 --- a/clients/openframe-client/src/services/mod.rs +++ b/clients/openframe-client/src/services/mod.rs @@ -20,6 +20,7 @@ pub mod openframe_client_info_service; pub mod openframe_client_update_service; pub mod registration_processor; pub mod shared_token_service; +pub mod token_refresh_run_manager; pub mod tool_agent_update_service; pub mod tool_command_params_resolver; pub mod tool_connection_message_publisher; @@ -51,6 +52,7 @@ pub use nats_message_publisher::NatsMessagePublisher; pub use openframe_client_info_service::OpenFrameClientInfoService; pub use openframe_client_update_service::OpenFrameClientUpdateService; pub use shared_token_service::SharedTokenService; +pub use token_refresh_run_manager::TokenRefreshRunManager; pub use tool_agent_update_service::ToolAgentUpdateService; pub use tool_command_params_resolver::ToolCommandParamsResolver; pub use tool_connection_message_publisher::ToolConnectionMessagePublisher; diff --git a/clients/openframe-client/src/services/shared_token_service.rs b/clients/openframe-client/src/services/shared_token_service.rs index 38301c011..91f6cb5bb 100644 --- a/clients/openframe-client/src/services/shared_token_service.rs +++ b/clients/openframe-client/src/services/shared_token_service.rs @@ -1,7 +1,7 @@ use crate::platform::directories::DirectoryManager; use crate::services::EncryptionService; +use crate::utils::fs::atomic_write; use anyhow::Result; -use std::fs; #[derive(Clone)] pub struct SharedTokenService { @@ -18,15 +18,9 @@ impl SharedTokenService { } pub fn update(&self, token: String) -> Result<()> { - let config_dir = self.dir_manager.secured_dir(); - let token_file_path = config_dir.join("shared_token.enc"); - - if let Some(parent) = token_file_path.parent() { - fs::create_dir_all(parent)?; - } - + let token_file_path = self.dir_manager.secured_dir().join("shared_token.enc"); let encrypted_token = self.encryption_service.encrypt(&token)?; - fs::write(token_file_path, encrypted_token)?; + atomic_write(&token_file_path, encrypted_token)?; Ok(()) } } diff --git a/clients/openframe-client/src/services/token_refresh_run_manager.rs b/clients/openframe-client/src/services/token_refresh_run_manager.rs new file mode 100644 index 000000000..644a53803 --- /dev/null +++ b/clients/openframe-client/src/services/token_refresh_run_manager.rs @@ -0,0 +1,102 @@ +use chrono::Utc; +use tokio::time::{sleep, timeout, Duration}; +use tracing::{debug, error, info, warn}; + +use crate::services::agent_configuration_service::AgentConfigurationService; +use crate::services::AgentAuthService; +use crate::utils::jwt; + +/// Refresh this long before `exp` under normal TTLs. +const REFRESH_MARGIN: Duration = Duration::from_secs(5 * 60); +/// Lead for a short-lived token (TTL <= margin) so it doesn't refresh every loop. +const MIN_LEAD: Duration = Duration::from_secs(15); +/// Used when the token's `exp` can't be decoded. +const FALLBACK_INTERVAL: Duration = Duration::from_secs(30 * 60); +/// Delay between refresh attempts after a failure. +const RETRY_INTERVAL: Duration = Duration::from_secs(60); +/// Cap on a single `reauthenticate()` call. +const REAUTH_TIMEOUT: Duration = Duration::from_secs(30); + +/// Proactively refreshes the access token before `exp` so `shared_token.enc` stays valid without a NATS reconnect. +#[derive(Clone)] +pub struct TokenRefreshRunManager { + auth_service: AgentAuthService, + config_service: AgentConfigurationService, +} + +impl TokenRefreshRunManager { + pub fn new(auth_service: AgentAuthService, config_service: AgentConfigurationService) -> Self { + Self { + auth_service, + config_service, + } + } + + pub fn start(&self) { + let auth_service = self.auth_service.clone(); + let config_service = self.config_service.clone(); + + info!("Starting proactive token refresh run manager"); + + tokio::spawn(async move { + loop { + let wait = next_refresh_delay(&config_service).await; + if !wait.is_zero() { + debug!("Next proactive token refresh in {}s", wait.as_secs()); + } + sleep(wait).await; + + // Retry on the short interval until a refresh succeeds. + loop { + match timeout(REAUTH_TIMEOUT, auth_service.reauthenticate()).await { + Ok(Ok(_)) => { + info!("Proactively refreshed access token; shared_token.enc updated"); + break; + } + Ok(Err(e)) => error!( + "Proactive token refresh failed: {e:#}; retrying in {}s", + RETRY_INTERVAL.as_secs() + ), + Err(_) => error!( + "Proactive token refresh timed out after {}s; retrying in {}s", + REAUTH_TIMEOUT.as_secs(), + RETRY_INTERVAL.as_secs() + ), + } + sleep(RETRY_INTERVAL).await; + } + } + }); + } +} + +/// Delay until the next refresh; zero when the token is at/near expiry, missing, or undecodable. +async fn next_refresh_delay(config_service: &AgentConfigurationService) -> Duration { + let token = match config_service.get_access_token().await { + Ok(t) if !t.is_empty() => t, + Ok(_) => return Duration::ZERO, + Err(e) => { + warn!("Token refresh: cannot read access token ({e:#}); using fallback interval"); + return FALLBACK_INTERVAL; + } + }; + + let Some(exp) = jwt::token_exp_unix(&token) else { + warn!("Token refresh: access token has no decodable exp; using fallback interval"); + return FALLBACK_INTERVAL; + }; + + // Full margin normally; MIN_LEAD for short-lived tokens. Saturating so a bad `exp` can't underflow. + let secs_to_exp = exp.saturating_sub(Utc::now().timestamp()); + let lead = if secs_to_exp > REFRESH_MARGIN.as_secs() as i64 { + REFRESH_MARGIN.as_secs() as i64 + } else { + MIN_LEAD.as_secs() as i64 + }; + let secs_until_refresh = secs_to_exp.saturating_sub(lead); + if secs_until_refresh <= 0 { + Duration::ZERO + } else { + Duration::from_secs(secs_until_refresh as u64) + } +} diff --git a/clients/openframe-client/src/utils.rs b/clients/openframe-client/src/utils.rs index 8e0ea38bf..91ce7b5b2 100644 --- a/clients/openframe-client/src/utils.rs +++ b/clients/openframe-client/src/utils.rs @@ -1,2 +1,5 @@ +pub mod fs; +pub mod jwt; + #[cfg(target_os = "windows")] pub mod windows_helpers; diff --git a/clients/openframe-client/src/utils/fs.rs b/clients/openframe-client/src/utils/fs.rs new file mode 100644 index 000000000..9ccdbaf2e --- /dev/null +++ b/clients/openframe-client/src/utils/fs.rs @@ -0,0 +1,20 @@ +use std::io::Write; +use std::path::Path; + +use anyhow::{Context, Result}; +use tempfile::NamedTempFile; + +/// Write `contents` to `path` atomically via a same-dir temp file + rename, so a concurrent +/// writer or reader never observes a torn file. +pub fn atomic_write(path: &Path, contents: impl AsRef<[u8]>) -> Result<()> { + let parent = path.parent().context("path has no parent directory")?; + std::fs::create_dir_all(parent)?; + + let mut tmp = NamedTempFile::new_in(parent) + .with_context(|| format!("failed to create temp file in {}", parent.display()))?; + tmp.write_all(contents.as_ref())?; + // persist() atomically replaces any existing file at `path` (MoveFileEx on Windows). + tmp.persist(path) + .with_context(|| format!("failed to persist temp file to {}", path.display()))?; + Ok(()) +} diff --git a/clients/openframe-client/src/utils/jwt.rs b/clients/openframe-client/src/utils/jwt.rs new file mode 100644 index 000000000..b5c172267 --- /dev/null +++ b/clients/openframe-client/src/utils/jwt.rs @@ -0,0 +1,56 @@ +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; +use serde::Deserialize; + +#[derive(Deserialize)] +struct ExpClaim { + exp: i64, +} + +/// Decode a JWT's `exp` claim (seconds since the Unix epoch) without verifying the signature. +/// Returns `None` if the token is malformed or carries no `exp`. +pub fn token_exp_unix(token: &str) -> Option { + // Require a well-formed `header.payload.signature` — reject tokens with missing or extra parts. + let mut parts = token.split('.'); + let (Some(_header), Some(payload), Some(_signature), None) = + (parts.next(), parts.next(), parts.next(), parts.next()) + else { + return None; + }; + let bytes = URL_SAFE_NO_PAD.decode(payload.trim_end_matches('=')).ok()?; + let claim: ExpClaim = serde_json::from_slice(&bytes).ok()?; + Some(claim.exp) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn make_token(payload: &str) -> String { + format!("header.{}.sig", URL_SAFE_NO_PAD.encode(payload.as_bytes())) + } + + #[test] + fn decodes_exp() { + let token = make_token(r#"{"exp":1700000000,"sub":"machine"}"#); + assert_eq!(token_exp_unix(&token), Some(1700000000)); + } + + #[test] + fn none_without_exp_claim() { + let token = make_token(r#"{"sub":"machine"}"#); + assert_eq!(token_exp_unix(&token), None); + } + + #[test] + fn none_when_malformed() { + assert_eq!(token_exp_unix("not-a-jwt"), None); + assert_eq!(token_exp_unix(""), None); + } + + #[test] + fn none_when_wrong_segment_count() { + let payload = URL_SAFE_NO_PAD.encode(r#"{"exp":1700000000}"#.as_bytes()); + assert_eq!(token_exp_unix(&format!("header.{payload}")), None); + assert_eq!(token_exp_unix(&format!("header.{payload}.sig.extra")), None); + } +} From 279d5a232bfbbeb268b51e549331883d1102e67f Mon Sep 17 00:00:00 2001 From: yaroslavmokflmg Date: Wed, 8 Jul 2026 09:43:02 -0400 Subject: [PATCH 04/19] docs(client): add trailing newline to PERMISSIONS.md (#2091) --- clients/openframe-client/docs/PERMISSIONS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/openframe-client/docs/PERMISSIONS.md b/clients/openframe-client/docs/PERMISSIONS.md index d40538b00..4d4525b06 100644 --- a/clients/openframe-client/docs/PERMISSIONS.md +++ b/clients/openframe-client/docs/PERMISSIONS.md @@ -98,4 +98,4 @@ If you encounter persistent permission issues: 1. Check the permissions log file 2. Run the agent with debug logging enabled 3. Contact support with the log files -4. Include the output of permission checks \ No newline at end of file +4. Include the output of permission checks From 80438571b3e98efd86fb4f1d5f98c4716b352efa Mon Sep 17 00:00:00 2001 From: mikhailm-coder Date: Wed, 8 Jul 2026 17:54:30 +0200 Subject: [PATCH 05/19] fix(client): restore 0o644 on atomic_write so chat can read shared_token.enc (#2092) Co-authored-by: Claude Opus 4.8 (1M context) --- clients/openframe-client/src/utils/fs.rs | 34 ++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/clients/openframe-client/src/utils/fs.rs b/clients/openframe-client/src/utils/fs.rs index 9ccdbaf2e..a5fffd85b 100644 --- a/clients/openframe-client/src/utils/fs.rs +++ b/clients/openframe-client/src/utils/fs.rs @@ -13,8 +13,42 @@ pub fn atomic_write(path: &Path, contents: impl AsRef<[u8]>) -> Result<()> { let mut tmp = NamedTempFile::new_in(parent) .with_context(|| format!("failed to create temp file in {}", parent.display()))?; tmp.write_all(contents.as_ref())?; + // NamedTempFile defaults to 0o600; restore fs::write's 0o644 so user processes can read. + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + tmp.as_file() + .set_permissions(std::fs::Permissions::from_mode(0o644))?; + } // persist() atomically replaces any existing file at `path` (MoveFileEx on Windows). tmp.persist(path) .with_context(|| format!("failed to persist temp file to {}", path.display()))?; Ok(()) } + +#[cfg(all(test, unix))] +mod tests { + use super::*; + use std::os::unix::fs::PermissionsExt; + + #[test] + fn atomic_write_sets_644_and_self_heals() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("token.enc"); + + atomic_write(&path, b"first").unwrap(); + assert_eq!( + std::fs::metadata(&path).unwrap().permissions().mode() & 0o777, + 0o644 + ); + + // A pre-existing owner-only file must be corrected on the next write, not preserved. + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).unwrap(); + atomic_write(&path, b"second").unwrap(); + assert_eq!( + std::fs::metadata(&path).unwrap().permissions().mode() & 0o777, + 0o644 + ); + assert_eq!(std::fs::read(&path).unwrap(), b"second"); + } +} From 4b295f023cd7ec9c77ec00e492702cae5a4cba19 Mon Sep 17 00:00:00 2001 From: denys-gif Date: Tue, 14 Jul 2026 12:56:57 +0100 Subject: [PATCH 06/19] feat: add script lock result --- .../src/executor/windows/mod.rs | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/clients/openframe-client/src/executor/windows/mod.rs b/clients/openframe-client/src/executor/windows/mod.rs index 0db81222d..3cb4281f0 100644 --- a/clients/openframe-client/src/executor/windows/mod.rs +++ b/clients/openframe-client/src/executor/windows/mod.rs @@ -2,7 +2,8 @@ mod job; mod process; mod run_as_user; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; +use std::time::Duration; use anyhow::{anyhow, Result}; @@ -36,6 +37,10 @@ pub async fn execute_script(params: ScriptParams<'_>) -> ExecResult { path: tmp_file.clone(), }; + if wait_until_readable(&tmp_file).await { + return spawn_error("Script file locked by another process".to_string()); + } + if wants_run_as { run_as_user::run_as_interactive(&interpreter, &tmp_file, ¶ms).await } else { @@ -43,6 +48,33 @@ pub async fn execute_script(params: ScriptParams<'_>) -> ExecResult { } } +const SCRIPT_READY_RETRIES: u32 = 3; +const SCRIPT_READY_DELAY_MS: u64 = 200; + +async fn wait_until_readable(path: &Path) -> bool { + for attempt in 0..SCRIPT_READY_RETRIES { + match std::fs::File::open(path) { + Ok(_) => return false, + Err(e) => { + let locked = matches!(e.raw_os_error(), Some(32) | Some(33)); + let transient = locked || e.raw_os_error() == Some(5); + if transient && attempt + 1 < SCRIPT_READY_RETRIES { + tracing::warn!( + attempt = attempt + 1, + error = %e, + path = %path.display(), + "script file not yet readable (antivirus scan lock?), waiting before launch" + ); + tokio::time::sleep(Duration::from_millis(SCRIPT_READY_DELAY_MS)).await; + continue; + } + return locked; + } + } + } + false +} + fn spawn_error(msg: String) -> ExecResult { ExecResult { stdout: String::new(), From e5e07fb225def562a7c9badb9dc60279df921664 Mon Sep 17 00:00:00 2001 From: denys-gif Date: Wed, 15 Jul 2026 15:23:18 +0100 Subject: [PATCH 07/19] feat: scheduled scripts parallel run --- .../src/config/update_config.rs | 3 + clients/openframe-client/src/lib.rs | 11 +- .../src/listener/execution_listener.rs | 160 +++++++++++++++++- .../openframe-client/src/models/execution.rs | 10 ++ 4 files changed, 174 insertions(+), 10 deletions(-) diff --git a/clients/openframe-client/src/config/update_config.rs b/clients/openframe-client/src/config/update_config.rs index f1dcc1657..4086b9ae1 100644 --- a/clients/openframe-client/src/config/update_config.rs +++ b/clients/openframe-client/src/config/update_config.rs @@ -16,6 +16,9 @@ pub const CONSUMER_CYCLE_PAUSE_MS: u64 = 30000; // 30 seconds pause between retr // Reconnection pub const RECONNECTION_DELAY_MS: u64 = 5000; // 5 seconds +// Execution concurrency +pub const EXECUTION_MIN_CONCURRENCY: usize = 4; + // NATS message settings pub const CONSUMER_ACK_WAIT_SECS: u64 = 120; pub const CONSUMER_MAX_DELIVER: i64 = 10; // Maximum delivery attempts diff --git a/clients/openframe-client/src/lib.rs b/clients/openframe-client/src/lib.rs index 1d2e1bfc8..2abcdeeaa 100644 --- a/clients/openframe-client/src/lib.rs +++ b/clients/openframe-client/src/lib.rs @@ -38,7 +38,9 @@ pub mod executor; use crate::clients::tool_agent_file_client::ToolAgentFileClient; use crate::clients::{AuthClient, RegistrationClient, ToolApiClient}; -use crate::config::update_config::{DOWNLOAD_CLIENT_TIMEOUT_SECS, HTTP_CLIENT_TIMEOUT_SECS}; +use crate::config::update_config::{ + DOWNLOAD_CLIENT_TIMEOUT_SECS, EXECUTION_MIN_CONCURRENCY, HTTP_CLIENT_TIMEOUT_SECS, +}; use crate::listener::execution_listener::ExecutionListener; use crate::listener::openframe_client_update_listener::OpenFrameClientUpdateListener; use crate::listener::tool_agent_update_listener::ToolAgentUpdateListener; @@ -435,17 +437,24 @@ impl Client { ); let execution_service = ExecutionService::new(); + let execution_concurrency = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(EXECUTION_MIN_CONCURRENCY) + .max(EXECUTION_MIN_CONCURRENCY); + let execution_semaphore = Arc::new(tokio::sync::Semaphore::new(execution_concurrency)); let command_execution_listener = ExecutionListener::::new( nats_connection_manager.clone(), nats_message_publisher.clone(), execution_service.clone(), config_service.clone(), + execution_semaphore.clone(), ); let script_execution_listener = ExecutionListener::::new( nats_connection_manager.clone(), nats_message_publisher.clone(), execution_service, config_service.clone(), + execution_semaphore, ); // Initialize machine heartbeat publisher and run manager diff --git a/clients/openframe-client/src/listener/execution_listener.rs b/clients/openframe-client/src/listener/execution_listener.rs index 542324791..985afeb5f 100644 --- a/clients/openframe-client/src/listener/execution_listener.rs +++ b/clients/openframe-client/src/listener/execution_listener.rs @@ -1,8 +1,10 @@ use std::marker::PhantomData; +use std::sync::Arc; use anyhow::{anyhow, Result}; use async_nats::Message; use futures::StreamExt; +use tokio::sync::Semaphore; use tokio::time::Duration; use tracing::{error, info, warn}; @@ -18,6 +20,7 @@ pub struct ExecutionListener { nats_message_publisher: NatsMessagePublisher, execution_service: ExecutionService, config_service: AgentConfigurationService, + semaphore: Arc, _marker: PhantomData M>, } @@ -28,6 +31,7 @@ impl Clone for ExecutionListener { nats_message_publisher: self.nats_message_publisher.clone(), execution_service: self.execution_service.clone(), config_service: self.config_service.clone(), + semaphore: self.semaphore.clone(), _marker: PhantomData, } } @@ -39,12 +43,14 @@ impl ExecutionListener { nats_message_publisher: NatsMessagePublisher, execution_service: ExecutionService, config_service: AgentConfigurationService, + semaphore: Arc, ) -> Self { Self { nats_connection_manager, nats_message_publisher, execution_service, config_service, + semaphore, _marker: PhantomData, } } @@ -77,21 +83,27 @@ impl ExecutionListener { let machine_id = self.config_service.get_machine_id()?; let subject = format!("machine.{}.{}", machine_id, M::KIND); - let mut subscriber = client + let subscriber = client .subscribe(subject.clone()) .await .map_err(|e| anyhow!("failed to subscribe to {}: {}", subject, e))?; info!(subject = %subject, "Execution listener active"); - while let Some(message) = subscriber.next().await { - if let Err(e) = self.handle_message(message, &machine_id).await { - error!( - kind = M::KIND, - "Failed to handle execution message: {:#}", e - ); + let listener = self.clone(); + run_bounded(subscriber, self.semaphore.clone(), move |message| { + let listener = listener.clone(); + let machine_id = machine_id.clone(); + async move { + if let Err(e) = listener.handle_message(message, &machine_id).await { + error!( + kind = M::KIND, + "Failed to handle execution message: {:#}", e + ); + } } - } + }) + .await; Ok(()) } @@ -106,7 +118,8 @@ impl ExecutionListener { } }; let execution_id = parsed.execution_id().to_string(); - info!(kind = M::KIND, execution_id = %execution_id, "Execution request received"); + let schedule_id = parsed.schedule_id().unwrap_or("-").to_string(); + info!(kind = M::KIND, execution_id = %execution_id, schedule_id = %schedule_id, "Execution request received"); let request = parsed.to_request(); let result = self.execution_service.execute(&request, machine_id).await; @@ -114,6 +127,7 @@ impl ExecutionListener { info!( kind = M::KIND, execution_id = %execution_id, + schedule_id = %schedule_id, exit_code = result.exit_code, timed_out = result.timed_out, execution_time_ms = result.execution_time_ms, @@ -132,3 +146,131 @@ impl ExecutionListener { Ok(()) } } + +async fn run_bounded( + stream: impl futures::Stream, + semaphore: Arc, + handler: F, +) where + T: Send + 'static, + F: Fn(T) -> Fut + Clone + Send + 'static, + Fut: std::future::Future + Send + 'static, +{ + tokio::pin!(stream); + while let Some(item) = stream.next().await { + let permit = match semaphore.clone().acquire_owned().await { + Ok(permit) => permit, + Err(_) => break, + }; + let handler = handler.clone(); + tokio::spawn(async move { + let _permit = permit; + handler(item).await; + }); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::time::{Duration as StdDuration, Instant}; + + async fn wait_for(counter: &AtomicUsize, target: usize) { + while counter.load(Ordering::SeqCst) < target { + tokio::time::sleep(StdDuration::from_millis(5)).await; + } + } + + #[tokio::test] + async fn runs_up_to_k_in_parallel() { + let k = 4; + let semaphore = Arc::new(Semaphore::new(k)); + let active = Arc::new(AtomicUsize::new(0)); + let max_active = Arc::new(AtomicUsize::new(0)); + let done = Arc::new(AtomicUsize::new(0)); + + let (a, m, d) = (active.clone(), max_active.clone(), done.clone()); + let start = Instant::now(); + run_bounded(futures::stream::iter(0..k), semaphore, move |_| { + let (a, m, d) = (a.clone(), m.clone(), d.clone()); + async move { + let now = a.fetch_add(1, Ordering::SeqCst) + 1; + m.fetch_max(now, Ordering::SeqCst); + tokio::time::sleep(StdDuration::from_millis(200)).await; + a.fetch_sub(1, Ordering::SeqCst); + d.fetch_add(1, Ordering::SeqCst); + } + }) + .await; + wait_for(&done, k).await; + + assert_eq!( + max_active.load(Ordering::SeqCst), + k, + "all K should run at once" + ); + assert!( + start.elapsed() < StdDuration::from_millis(600), + "K parallel sleeps should take ~one duration, took {:?}", + start.elapsed() + ); + } + + #[tokio::test] + async fn concurrency_never_exceeds_k() { + let k = 2; + let n = 10; + let semaphore = Arc::new(Semaphore::new(k)); + let active = Arc::new(AtomicUsize::new(0)); + let max_active = Arc::new(AtomicUsize::new(0)); + let done = Arc::new(AtomicUsize::new(0)); + + let (a, m, d) = (active.clone(), max_active.clone(), done.clone()); + run_bounded(futures::stream::iter(0..n), semaphore, move |_| { + let (a, m, d) = (a.clone(), m.clone(), d.clone()); + async move { + let now = a.fetch_add(1, Ordering::SeqCst) + 1; + m.fetch_max(now, Ordering::SeqCst); + tokio::time::sleep(StdDuration::from_millis(30)).await; + a.fetch_sub(1, Ordering::SeqCst); + d.fetch_add(1, Ordering::SeqCst); + } + }) + .await; + wait_for(&done, n).await; + + assert!( + max_active.load(Ordering::SeqCst) <= k, + "observed {} concurrent, cap is {}", + max_active.load(Ordering::SeqCst), + k + ); + assert_eq!(done.load(Ordering::SeqCst), n, "every item must complete"); + } + + #[tokio::test] + async fn permit_released_on_panic() { + let semaphore = Arc::new(Semaphore::new(1)); + let done = Arc::new(AtomicUsize::new(0)); + + let d = done.clone(); + run_bounded(futures::stream::iter(0..3usize), semaphore, move |i| { + let d = d.clone(); + async move { + if i == 0 { + panic!("intentional panic in first task"); + } + d.fetch_add(1, Ordering::SeqCst); + } + }) + .await; + wait_for(&done, 2).await; + + assert_eq!( + done.load(Ordering::SeqCst), + 2, + "a panicking task must release its permit so the rest still run" + ); + } +} diff --git a/clients/openframe-client/src/models/execution.rs b/clients/openframe-client/src/models/execution.rs index ca0a05493..241aee89c 100644 --- a/clients/openframe-client/src/models/execution.rs +++ b/clients/openframe-client/src/models/execution.rs @@ -47,6 +47,8 @@ pub struct ScriptMessage { pub execution_id: String, #[serde(default)] pub machine_id: Option, + #[serde(default)] + pub schedule_id: Option, pub code: String, pub shell: ScriptShell, #[serde(default)] @@ -104,6 +106,10 @@ pub trait ExecutionMessage: Sized + Send { fn from_payload(payload: &str) -> Result; fn execution_id(&self) -> &str; fn to_request(&self) -> ExecutionRequest<'_>; + + fn schedule_id(&self) -> Option<&str> { + None + } } impl ExecutionMessage for CommandMessage { @@ -141,6 +147,10 @@ impl ExecutionMessage for ScriptMessage { &self.execution_id } + fn schedule_id(&self) -> Option<&str> { + self.schedule_id.as_deref() + } + fn to_request(&self) -> ExecutionRequest<'_> { ExecutionRequest { execution_id: &self.execution_id, From 6b674dcb338357858e1ed682cce42a38bff46ddf Mon Sep 17 00:00:00 2001 From: denys-gif Date: Fri, 17 Jul 2026 13:21:50 +0100 Subject: [PATCH 08/19] feat: add script id --- .../openframe-client/src/models/execution.rs | 31 ++++++++++++++++++- .../src/services/execution_service.rs | 8 +++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/clients/openframe-client/src/models/execution.rs b/clients/openframe-client/src/models/execution.rs index 241aee89c..06e43e24c 100644 --- a/clients/openframe-client/src/models/execution.rs +++ b/clients/openframe-client/src/models/execution.rs @@ -49,6 +49,8 @@ pub struct ScriptMessage { pub machine_id: Option, #[serde(default)] pub schedule_id: Option, + #[serde(default)] + pub script_id: Option, pub code: String, pub shell: ScriptShell, #[serde(default)] @@ -88,6 +90,10 @@ pub struct RmmResult { pub timed_out: bool, #[serde(skip_serializing_if = "Option::is_none")] pub error: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub script_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub schedule_id: Option, } pub struct ExecutionRequest<'a> { @@ -98,6 +104,8 @@ pub struct ExecutionRequest<'a> { pub args: &'a [String], pub timeout_secs: u64, pub env_vars: Vec, + pub script_id: Option<&'a str>, + pub schedule_id: Option<&'a str>, } pub trait ExecutionMessage: Sized + Send { @@ -132,6 +140,8 @@ impl ExecutionMessage for CommandMessage { args: &[], timeout_secs: self.timeout, env_vars: Vec::new(), + script_id: None, + schedule_id: None, } } } @@ -164,6 +174,8 @@ impl ExecutionMessage for ScriptMessage { .iter() .map(|e| format!("{}={}", e.name, e.value)) .collect(), + script_id: self.script_id.as_deref(), + schedule_id: self.schedule_id.as_deref(), } } } @@ -186,7 +198,7 @@ mod tests { #[test] fn parses_script_message_with_env_and_args() { let m = ScriptMessage::from_payload( - r#"{"executionId":"e","machineId":"mac","code":"x","shell":"POWERSHELL","privilegeLevel":"USER","args":["-v"],"timeoutSeconds":60,"envVars":[{"name":"FOO","value":"bar"}]}"#, + r#"{"executionId":"e","machineId":"mac","scheduleId":"sch-1","scriptId":"scr-1","code":"x","shell":"POWERSHELL","privilegeLevel":"USER","args":["-v"],"timeoutSeconds":60,"envVars":[{"name":"FOO","value":"bar"}]}"#, ) .unwrap(); let req = m.to_request(); @@ -194,6 +206,16 @@ mod tests { assert_eq!(req.args, &["-v".to_string()]); assert_eq!(req.env_vars, vec!["FOO=bar".to_string()]); assert!(matches!(req.privilege, PrivilegeLevel::User)); + assert_eq!(req.script_id, Some("scr-1")); + assert_eq!(req.schedule_id, Some("sch-1")); + } + + #[test] + fn script_message_without_script_id_still_parses() { + let m = ScriptMessage::from_payload(r#"{"executionId":"e","code":"x","shell":"BASH"}"#) + .unwrap(); + assert!(m.script_id.is_none()); + assert!(m.to_request().script_id.is_none()); } #[test] @@ -215,6 +237,8 @@ mod tests { execution_time_ms: 1, timed_out: false, error: None, + script_id: Some("scr-1".into()), + schedule_id: None, }; let v = serde_json::to_value(&r).unwrap(); assert!(v.get("execution_id").is_some()); @@ -222,6 +246,11 @@ mod tests { assert!(v.get("execution_time_ms").is_some()); assert!(v.get("timed_out").is_some()); assert!(v.get("error").is_none(), "None error must be omitted"); + assert_eq!(v.get("script_id").and_then(|s| s.as_str()), Some("scr-1")); + assert!( + v.get("schedule_id").is_none(), + "None schedule_id must be omitted" + ); } #[test] diff --git a/clients/openframe-client/src/services/execution_service.rs b/clients/openframe-client/src/services/execution_service.rs index 04c61ee28..2ebea4344 100644 --- a/clients/openframe-client/src/services/execution_service.rs +++ b/clients/openframe-client/src/services/execution_service.rs @@ -52,6 +52,8 @@ impl ExecutionService { execution_time_ms: start.elapsed().as_millis() as u64, timed_out: result.timed_out, error, + script_id: req.script_id.map(str::to_string), + schedule_id: req.schedule_id.map(str::to_string), } } @@ -66,6 +68,8 @@ impl ExecutionService { execution_time_ms: start.elapsed().as_millis() as u64, timed_out: false, error: Some("unsupported platform".to_string()), + script_id: req.script_id.map(str::to_string), + schedule_id: req.schedule_id.map(str::to_string), } } } @@ -85,6 +89,8 @@ mod tests { args: &[], timeout_secs: 30, env_vars: Vec::new(), + script_id: None, + schedule_id: None, } } @@ -135,6 +141,8 @@ mod windows_tests { args: &[], timeout_secs: 30, env_vars: Vec::new(), + script_id: None, + schedule_id: None, } } From 1f090c43b8d38650535e544f5b5026c785ebf167 Mon Sep 17 00:00:00 2001 From: mikhailm-coder Date: Fri, 17 Jul 2026 18:56:49 +0200 Subject: [PATCH 09/19] fix(mesh-self-heal): version-proof detection, real restart via shared tool-restart flow, silence detection (#2138) Co-authored-by: Claude Fable 5 Co-authored-by: denys-gif --- .../src/config/update_config.rs | 2 + clients/openframe-client/src/lib.rs | 21 ++ clients/openframe-client/src/listener/mod.rs | 2 + .../listener/tool_restart_message_listener.rs | 272 ++++++++++++++++++ clients/openframe-client/src/models/mod.rs | 2 + .../src/models/tool_restart_message.rs | 7 + .../src/services/mesh_self_heal_service.rs | 257 +++++++++++++---- clients/openframe-client/src/services/mod.rs | 2 + .../src/services/tool_kill_service.rs | 83 ++++-- .../src/services/tool_restart_service.rs | 145 ++++++++++ .../src/services/tool_run_manager.rs | 2 + 11 files changed, 718 insertions(+), 77 deletions(-) create mode 100644 clients/openframe-client/src/listener/tool_restart_message_listener.rs create mode 100644 clients/openframe-client/src/models/tool_restart_message.rs create mode 100644 clients/openframe-client/src/services/tool_restart_service.rs diff --git a/clients/openframe-client/src/config/update_config.rs b/clients/openframe-client/src/config/update_config.rs index 4086b9ae1..9614bd17f 100644 --- a/clients/openframe-client/src/config/update_config.rs +++ b/clients/openframe-client/src/config/update_config.rs @@ -23,3 +23,5 @@ pub const EXECUTION_MIN_CONCURRENCY: usize = 4; pub const CONSUMER_ACK_WAIT_SECS: u64 = 120; pub const CONSUMER_MAX_DELIVER: i64 = 10; // Maximum delivery attempts pub const UNINSTALL_CONSUMER_MAX_DELIVER: i64 = 20; // Larger budget: uninstall may defer behind a long install holding the tool lock +pub const RESTART_CONSUMER_MAX_DELIVER: i64 = 20; // Larger budget: restart may defer behind a long install holding the tool lock +pub const RESTART_CONSUMER_QUIET_PAUSE_MS: u64 = 300_000; // Quiet retry cadence once consumer creation keeps failing (subject/grants not provisioned yet) diff --git a/clients/openframe-client/src/lib.rs b/clients/openframe-client/src/lib.rs index 2abcdeeaa..efb87fda8 100644 --- a/clients/openframe-client/src/lib.rs +++ b/clients/openframe-client/src/lib.rs @@ -45,6 +45,7 @@ use crate::listener::execution_listener::ExecutionListener; use crate::listener::openframe_client_update_listener::OpenFrameClientUpdateListener; use crate::listener::tool_agent_update_listener::ToolAgentUpdateListener; use crate::listener::tool_installation_message_listener::ToolInstallationMessageListener; +use crate::listener::tool_restart_message_listener::ToolRestartMessageListener; use crate::listener::tool_uninstall_message_listener::ToolUninstallMessageListener; use crate::logging::nats_streaming::LogStreamingRunManager; use crate::models::{CommandMessage, ScriptMessage}; @@ -72,6 +73,7 @@ use crate::services::tool_agent_update_service::ToolAgentUpdateService; use crate::services::tool_connection_message_publisher::ToolConnectionMessagePublisher; use crate::services::tool_connection_service::ToolConnectionService; use crate::services::tool_installation_service::ToolInstallationService; +use crate::services::tool_restart_service::ToolRestartService; use crate::services::tool_uninstall_service::ToolUninstallService; use crate::services::InstalledToolsService; use crate::services::{ @@ -150,6 +152,7 @@ pub struct Client { nats_connection_manager: NatsConnectionManager, tool_installation_message_listener: ToolInstallationMessageListener, tool_uninstall_message_listener: ToolUninstallMessageListener, + tool_restart_message_listener: ToolRestartMessageListener, openframe_client_update_listener: OpenFrameClientUpdateListener, tool_agent_update_listener: ToolAgentUpdateListener, command_execution_listener: ExecutionListener, @@ -326,11 +329,19 @@ impl Client { tool_kill_service.clone(), ); + // Initialize tool restart service + let tool_restart_service = ToolRestartService::new( + installed_tools_service.clone(), + tool_kill_service.clone(), + tool_run_manager.clone(), + ); + // Initialize mesh self-heal service let mesh_self_heal_service = MeshSelfHealService::new( directory_manager.clone(), installed_tools_service.clone(), tool_kill_service.clone(), + tool_restart_service.clone(), initial_configuration_service.clone(), config_service.clone(), tool_run_manager.clone(), @@ -422,6 +433,13 @@ impl Client { config_service.clone(), ); + // Initialize tool restart listener (shares the restart service with mesh self-heal) + let tool_restart_message_listener = ToolRestartMessageListener::new( + nats_connection_manager.clone(), + tool_restart_service, + config_service.clone(), + ); + // Initialize OpenFrame client update listener let openframe_client_update_listener = OpenFrameClientUpdateListener::new( nats_connection_manager.clone(), @@ -480,6 +498,7 @@ impl Client { nats_connection_manager, tool_installation_message_listener, tool_uninstall_message_listener, + tool_restart_message_listener, openframe_client_update_listener, tool_agent_update_listener, command_execution_listener, @@ -534,6 +553,8 @@ impl Client { self.tool_uninstall_message_listener.start().await?; + self.tool_restart_message_listener.start().await?; + // Start OpenFrame client update listener in background self.openframe_client_update_listener.start().await?; diff --git a/clients/openframe-client/src/listener/mod.rs b/clients/openframe-client/src/listener/mod.rs index f6ad2c663..731eebc7e 100644 --- a/clients/openframe-client/src/listener/mod.rs +++ b/clients/openframe-client/src/listener/mod.rs @@ -2,10 +2,12 @@ pub mod execution_listener; pub mod openframe_client_update_listener; pub mod tool_agent_update_listener; pub mod tool_installation_message_listener; +pub mod tool_restart_message_listener; pub mod tool_uninstall_message_listener; pub use execution_listener::ExecutionListener; pub use openframe_client_update_listener::OpenFrameClientUpdateListener; pub use tool_agent_update_listener::ToolAgentUpdateListener; pub use tool_installation_message_listener::ToolInstallationMessageListener; +pub use tool_restart_message_listener::ToolRestartMessageListener; pub use tool_uninstall_message_listener::ToolUninstallMessageListener; diff --git a/clients/openframe-client/src/listener/tool_restart_message_listener.rs b/clients/openframe-client/src/listener/tool_restart_message_listener.rs new file mode 100644 index 000000000..deba0b62c --- /dev/null +++ b/clients/openframe-client/src/listener/tool_restart_message_listener.rs @@ -0,0 +1,272 @@ +use crate::config::update_config::{ + CONSUMER_ACK_WAIT_SECS, CONSUMER_CYCLE_PAUSE_MS, CONSUMER_RETRY_ATTEMPTS_PER_CYCLE, + INITIAL_RETRY_DELAY_MS, MAX_RETRY_DELAY_MS, RECONNECTION_DELAY_MS, + RESTART_CONSUMER_MAX_DELIVER, RESTART_CONSUMER_QUIET_PAUSE_MS, +}; +use crate::models::ToolRestartMessage; +use crate::services::nats_connection_manager::NatsConnectionManager; +use crate::services::tool_restart_service::RestartOutcome; +use crate::services::tool_restart_service::ToolRestartService; +use crate::services::AgentConfigurationService; +use anyhow::Result; +use async_nats::jetstream; +use async_nats::jetstream::consumer::push; +use async_nats::jetstream::consumer::PushConsumer; +use async_nats::jetstream::Message; +use futures::StreamExt; +use tokio::time::Duration; +use tracing::{debug, error, info, warn}; + +#[derive(Clone)] +pub struct ToolRestartMessageListener { + nats_connection_manager: NatsConnectionManager, + tool_restart_service: ToolRestartService, + config_service: AgentConfigurationService, +} + +impl ToolRestartMessageListener { + const STREAM_NAME: &'static str = "TOOL_INSTALLATION"; + + pub fn new( + nats_connection_manager: NatsConnectionManager, + tool_restart_service: ToolRestartService, + config_service: AgentConfigurationService, + ) -> Self { + Self { + nats_connection_manager, + tool_restart_service, + config_service, + } + } + + pub async fn start(&self) -> Result> { + let listener = self.clone(); + let handle = tokio::spawn(async move { + loop { + info!("Starting tool restart message listener..."); + match listener.listen().await { + Ok(_) => { + warn!("Tool restart message listener exited normally (unexpected)"); + } + Err(e) => { + error!("Tool restart message listener error: {:#}", e); + } + } + + info!( + "Reconnecting tool restart message listener in {} seconds...", + RECONNECTION_DELAY_MS / 1000 + ); + tokio::time::sleep(Duration::from_millis(RECONNECTION_DELAY_MS)).await; + } + }); + Ok(handle) + } + + async fn listen(&self) -> Result<()> { + info!("Run tool restart message listener"); + let client = self.nats_connection_manager.get_client().await?; + let js = jetstream::new((*client).clone()); + + let machine_id = self.config_service.get_machine_id()?; + + let consumer = self.create_consumer(&js, &machine_id).await; + + info!("Start listening for tool restart messages"); + let mut messages = consumer.messages().await?; + + while let Some(msg_result) = messages.next().await { + let message = match msg_result { + Ok(msg) => msg, + Err(e) => { + error!("Failed to receive message: {:#}", e); + continue; + } + }; + + if let Err(e) = self.handle_message(message).await { + error!("Failed to handle message: {:#}", e); + } + } + + Ok(()) + } + + async fn handle_message(&self, message: Message) -> Result<()> { + let payload = String::from_utf8_lossy(&message.payload); + info!("Received tool restart message: {:?}", payload); + + let restart_message: ToolRestartMessage = match serde_json::from_str(&payload) { + Ok(msg) => msg, + Err(e) => { + error!("Failed to parse tool restart message: {:#}", e); + if let Err(ack_err) = message.ack().await { + warn!("Failed to ack malformed message: {}", ack_err); + } + return Ok(()); + } + }; + + let tool_agent_id = restart_message.tool_agent_id; + + let ack_message = match self + .tool_restart_service + .restart_guarded(&tool_agent_id) + .await + { + Ok(RestartOutcome::Busy) => { + info!( + "Tool {} busy with another operation, deferring restart for redelivery", + tool_agent_id + ); + return Ok(()); + } + Ok(RestartOutcome::Restarted) | Ok(RestartOutcome::NotInstalled) => true, + Err(e) => { + error!("Failed to restart tool {}: {:#}", tool_agent_id, e); + false + } + }; + + if ack_message { + message + .ack() + .await + .map_err(|e| anyhow::anyhow!("Failed to ack message: {}", e))?; + info!("Restart message acknowledged for tool: {}", tool_agent_id); + } else { + info!( + "Leaving restart message unacked for potential redelivery: tool {}", + tool_agent_id + ); + } + + Ok(()) + } + + async fn create_consumer(&self, js: &jetstream::Context, machine_id: &str) -> PushConsumer { + let consumer_configuration = Self::build_consumer_configuration(machine_id); + let mut cycle = 0u32; + + loop { + cycle += 1; + let mut delay_ms = INITIAL_RETRY_DELAY_MS; + // First cycle logs loudly; later cycles go quiet with a long pause so a server missing the tool-restart subject/grants can't spam the fleet's logs. + let loud = cycle == 1; + + for attempt in 1..=CONSUMER_RETRY_ATTEMPTS_PER_CYCLE { + if loud { + info!( + "Creating restart consumer for stream {} (cycle {}, attempt {}/{})", + Self::STREAM_NAME, + cycle, + attempt, + CONSUMER_RETRY_ATTEMPTS_PER_CYCLE + ); + } else { + debug!( + "Creating restart consumer for stream {} (cycle {}, attempt {}/{})", + Self::STREAM_NAME, + cycle, + attempt, + CONSUMER_RETRY_ATTEMPTS_PER_CYCLE + ); + } + + match js + .create_consumer_on_stream(consumer_configuration.clone(), Self::STREAM_NAME) + .await + { + Ok(consumer) => { + info!("Restart consumer created for stream: {}", Self::STREAM_NAME); + return consumer; + } + Err(e) => { + let error_msg = format!("{:?}", e); + if error_msg.contains("consumer name already in use") + || error_msg.contains("10013") + { + warn!("Restart consumer already exists, attempting to get existing consumer"); + let durable_name = Self::build_durable_name(machine_id); + if let Ok(existing_consumer) = js + .get_consumer_from_stream(Self::STREAM_NAME, &durable_name) + .await + { + info!( + "Retrieved existing restart consumer for stream: {}", + Self::STREAM_NAME + ); + return existing_consumer; + } + } + + if loud { + warn!( + "Failed to create restart consumer (cycle {}, attempt {}/{}): {:#}", + cycle, attempt, CONSUMER_RETRY_ATTEMPTS_PER_CYCLE, e + ); + } else { + debug!( + "Failed to create restart consumer (cycle {}, attempt {}/{}): {:#}", + cycle, attempt, CONSUMER_RETRY_ATTEMPTS_PER_CYCLE, e + ); + } + if attempt < CONSUMER_RETRY_ATTEMPTS_PER_CYCLE { + tokio::time::sleep(Duration::from_millis(delay_ms)).await; + delay_ms = (delay_ms * 2).min(MAX_RETRY_DELAY_MS); + } + } + } + } + + let pause_ms = if loud { + CONSUMER_CYCLE_PAUSE_MS + } else { + RESTART_CONSUMER_QUIET_PAUSE_MS + }; + if loud { + warn!( + "All {} attempts in cycle {} failed (tool-restart stream subject/permissions may not be provisioned yet). Retrying quietly every {} seconds...", + CONSUMER_RETRY_ATTEMPTS_PER_CYCLE, cycle, RESTART_CONSUMER_QUIET_PAUSE_MS / 1000 + ); + } else { + debug!( + "All {} attempts in cycle {} failed. Pausing {} seconds before next cycle...", + CONSUMER_RETRY_ATTEMPTS_PER_CYCLE, + cycle, + pause_ms / 1000 + ); + } + tokio::time::sleep(Duration::from_millis(pause_ms)).await; + } + } + + fn build_consumer_configuration(machine_id: &str) -> push::Config { + let filter_subject = Self::build_filter_subject(machine_id); + let deliver_subject = Self::build_deliver_subject(machine_id); + let durable_name = Self::build_durable_name(machine_id); + + info!("Restart consumer configuration - filter subject: {}, deliver subject: {}, durable name: {}", filter_subject, deliver_subject, durable_name); + + push::Config { + filter_subject, + deliver_subject, + durable_name: Some(durable_name), + ack_wait: Duration::from_secs(CONSUMER_ACK_WAIT_SECS), + max_deliver: RESTART_CONSUMER_MAX_DELIVER, + ..Default::default() + } + } + + fn build_filter_subject(machine_id: &str) -> String { + format!("machine.{}.tool-restart", machine_id) + } + + fn build_deliver_subject(machine_id: &str) -> String { + format!("machine.{}.tool-restart.inbox", machine_id) + } + + fn build_durable_name(machine_id: &str) -> String { + format!("machine_{}_tool-restart_consumer", machine_id) + } +} diff --git a/clients/openframe-client/src/models/mod.rs b/clients/openframe-client/src/models/mod.rs index 3aaedcb66..e09ea1ea3 100644 --- a/clients/openframe-client/src/models/mod.rs +++ b/clients/openframe-client/src/models/mod.rs @@ -16,6 +16,7 @@ pub mod tool_connection; pub mod tool_connection_message; pub mod tool_installation_message; pub mod tool_installation_result; +pub mod tool_restart_message; pub mod tool_uninstall_message; pub mod tool_version_overrides; pub mod update_state; @@ -42,5 +43,6 @@ pub use tool_connection::ToolConnection; pub use tool_connection_message::ToolConnectionMessage; pub use tool_installation_message::ToolInstallationMessage; pub use tool_installation_result::ToolInstallationResult; +pub use tool_restart_message::ToolRestartMessage; pub use tool_uninstall_message::ToolUninstallMessage; pub use update_state::{UpdatePhase, UpdateState}; diff --git a/clients/openframe-client/src/models/tool_restart_message.rs b/clients/openframe-client/src/models/tool_restart_message.rs new file mode 100644 index 000000000..ad6d12ee5 --- /dev/null +++ b/clients/openframe-client/src/models/tool_restart_message.rs @@ -0,0 +1,7 @@ +use serde::Deserialize; + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ToolRestartMessage { + pub tool_agent_id: String, +} diff --git a/clients/openframe-client/src/services/mesh_self_heal_service.rs b/clients/openframe-client/src/services/mesh_self_heal_service.rs index 833520b28..fb56c81ef 100644 --- a/clients/openframe-client/src/services/mesh_self_heal_service.rs +++ b/clients/openframe-client/src/services/mesh_self_heal_service.rs @@ -5,9 +5,9 @@ use anyhow::{anyhow, Result}; use tokio::time::sleep; use tracing::{debug, error, info, warn}; -use crate::models::Installation; -use crate::platform::{system_service, DirectoryManager}; +use crate::platform::DirectoryManager; use crate::services::tool_kill_service::ToolKillService; +use crate::services::tool_restart_service::{RestartOutcome, ToolRestartService}; use crate::services::tool_run_manager::ToolRunManager; use crate::services::{ AgentConfigurationService, InitialConfigurationService, InstalledToolsService, @@ -15,25 +15,30 @@ use crate::services::{ const MESH_TOOL_ID: &str = "meshcentral-agent"; -/// Agent log line for a control channel that can't connect. -const FAILURE_MARKER: &str = "Connection FAILED: No HTTP response"; -/// Printed only after a successful server connect. +/// Stable prefix shared by every control-channel failure line the agent emits ("Connection FAILED: ..." and "Connection FAILED (latest attempt): ..."). +const FAILURE_MARKER: &str = "Connection FAILED"; +/// Printed only after a successful server connect, identically across agent versions. const HEALTHY_MARKER: &str = "Received CoreOk from server"; /// How often we scan the agent log. const POLL_INTERVAL: Duration = Duration::from_secs(30); -/// How long continuously stuck before we act. +/// How long an unresolved failure (no healthy connect logged since) may stand before we act. const STUCK_DURATION: Duration = Duration::from_secs(10 * 60); -/// Minimum wait between heal attempts (success or no-op), so a server-side outage can't spin. -const NOOP_HEAL_COOLDOWN: Duration = Duration::from_secs(60 * 60); +/// A healthy agent logs roughly hourly, so this much total silence means it is wedged or dead. +const SILENCE_DURATION: Duration = Duration::from_secs(90 * 60); +/// Minimum wait between heal attempts (restart, no-op, or failure), so a server-side outage can't spin. +const ACTION_COOLDOWN: Duration = Duration::from_secs(60 * 60); /// Timeout for the /generate-msh fetch so an unresponsive server can't block the heal loop. const HTTP_TIMEOUT: Duration = Duration::from_secs(30); +/// How far back to look for markers when seeding health state at startup. +const TAIL_BYTES: u64 = 64 * 1024; #[derive(Clone)] pub struct MeshSelfHealService { directory_manager: DirectoryManager, installed_tools: InstalledToolsService, tool_kill: ToolKillService, + tool_restart: ToolRestartService, initial_config: InitialConfigurationService, agent_config: AgentConfigurationService, tool_run_manager: ToolRunManager, @@ -45,6 +50,7 @@ impl MeshSelfHealService { directory_manager: DirectoryManager, installed_tools: InstalledToolsService, tool_kill: ToolKillService, + tool_restart: ToolRestartService, initial_config: InitialConfigurationService, agent_config: AgentConfigurationService, tool_run_manager: ToolRunManager, @@ -53,6 +59,7 @@ impl MeshSelfHealService { directory_manager, installed_tools, tool_kill, + tool_restart, initial_config, agent_config, tool_run_manager, @@ -84,23 +91,43 @@ impl MeshSelfHealService { log_path.display() ); - let mut offset: u64 = 0; - let mut stuck_since: Option = None; - let mut last_heal_attempt: Option = None; + // Start at EOF so stale history can't arm detection, but seed health from the tail to catch an already-wedged agent. + let mut offset: u64 = tokio::fs::metadata(&log_path) + .await + .map(|m| m.len()) + .unwrap_or(0); + let mut last_marker_healthy = last_marker_in_tail(&log_path).await.unwrap_or(true); + let mut stuck_since: Option = if last_marker_healthy { + None + } else { + Some(Instant::now()) + }; + let mut last_action: Option = None; + let mut last_activity = seed_last_activity(&log_path).await; loop { + let sleep_started = Instant::now(); sleep(POLL_INTERVAL).await; - let msh_missing_serverid = self.current_msh_missing_serverid().await; + // The sleep alone overran by far ⇒ the host was suspended (Instant counts suspend on Windows) — discard timers measured across it. + if sleep_started.elapsed() > POLL_INTERVAL * 5 { + stuck_since = None; + last_activity = Instant::now(); + } match read_new_lines(&log_path, &mut offset).await { Ok(lines) => { + if !lines.is_empty() { + last_activity = Instant::now(); + } for line in &lines { if line.contains(HEALTHY_MARKER) { stuck_since = None; - last_heal_attempt = None; + last_action = None; + last_marker_healthy = true; } else if line.contains(FAILURE_MARKER) { stuck_since.get_or_insert_with(Instant::now); + last_marker_healthy = false; } } } @@ -109,42 +136,79 @@ impl MeshSelfHealService { } } - let stuck = stuck_since.is_some_and(|t| t.elapsed() >= STUCK_DURATION); - if !msh_missing_serverid && !stuck { + // Agent is being replaced by an update — drop stale history so we don't restart it post-update. + if self.tool_run_manager.is_updating(MESH_TOOL_ID).await { + stuck_since = None; + last_activity = Instant::now(); continue; } - if let Some(t) = last_heal_attempt { - if t.elapsed() < NOOP_HEAL_COOLDOWN { + if let Some(t) = last_action { + if t.elapsed() < ACTION_COOLDOWN { continue; } } - if self.tool_run_manager.is_updating(MESH_TOOL_ID).await { - info!("meshcentral-agent is updating — skipping .msh self-heal this cycle"); + let stuck = stuck_since.is_some_and(|t| t.elapsed() >= STUCK_DURATION); + let silent = last_activity.elapsed() >= SILENCE_DURATION; + let msh_missing_serverid = self.current_msh_missing_serverid().await; + + if msh_missing_serverid || stuck { + let reason = if msh_missing_serverid { + "current .msh has no ServerID (agent cannot authenticate the server)" + .to_string() + } else { + format!("no successful connect within {}s", STUCK_DURATION.as_secs()) + }; + warn!("meshcentral-agent unhealthy: {reason} — refreshing .msh and restarting the agent"); + + // Arm the cooldown before acting so no outcome (busy, error, no-op) can spin the loop. + last_action = Some(Instant::now()); + match self.try_refresh_msh().await { + Ok(true) => info!("mesh self-heal: refreshed .msh (NodeID preserved)"), + Ok(false) => debug!("mesh self-heal: .msh already current"), + Err(e) => { + error!("mesh self-heal: .msh refresh failed (restarting anyway): {e:#}") + } + } + self.restart_agent().await; + stuck_since = None; - continue; - } + last_activity = Instant::now(); + } else if silent { + warn!( + "meshcentral-agent silent for {}s (last_marker_healthy={last_marker_healthy}) — restarting the agent", + last_activity.elapsed().as_secs() + ); - let reason = if msh_missing_serverid { - "current .msh has no ServerID (agent cannot authenticate the server)".to_string() - } else { - format!("no successful connect within {}s", STUCK_DURATION.as_secs()) - }; - warn!("meshcentral-agent unhealthy: {reason} — attempting .msh self-heal"); - - match self.try_heal().await { - Ok(true) => info!("mesh self-heal: refreshed .msh and restarted the agent (NodeID preserved)"), - Ok(false) => debug!("mesh self-heal: .msh already current — likely a server-side issue, no action taken"), - Err(e) => error!("mesh self-heal failed: {e:#}"), + last_action = Some(Instant::now()); + self.restart_agent().await; + + stuck_since = None; + last_activity = Instant::now(); } + } + } - stuck_since = None; - last_heal_attempt = Some(Instant::now()); + /// Restart through the shared guarded flow; a missing registry entry degrades to a process kill so the OS supervisor can relaunch. + async fn restart_agent(&self) { + match self.tool_restart.restart_guarded(MESH_TOOL_ID).await { + Ok(RestartOutcome::Restarted) => info!("mesh self-heal: agent restarted"), + Ok(RestartOutcome::Busy) => info!( + "mesh self-heal: meshcentral-agent busy with another operation — skipping restart" + ), + Ok(RestartOutcome::NotInstalled) => { + warn!("mesh self-heal: meshcentral-agent not in registry — falling back to a process kill"); + if let Err(e) = self.tool_kill.stop_tool(MESH_TOOL_ID).await { + error!("mesh self-heal: fallback kill failed: {e:#}"); + } + } + Err(e) => error!("mesh self-heal: agent restart failed: {e:#}"), } } - async fn try_heal(&self) -> Result { + /// Refresh the .msh from /generate-msh; returns true when it was rewritten. + async fn try_refresh_msh(&self) -> Result { let host = self.initial_config.get_server_url()?; let url = format!("https://{host}/tools/agent/meshcentral-server/generate-msh?host={host}"); @@ -183,7 +247,7 @@ impl MeshSelfHealService { .as_deref() .and_then(|s| parse_msh_field(s, "MeshServer")) .unwrap_or_else(|| "".to_string()); - info!("mesh self-heal: .msh already current (MeshServer={server}) — no action"); + info!("mesh self-heal: .msh already current (MeshServer={server})"); return Ok(false); } @@ -206,15 +270,6 @@ impl MeshSelfHealService { let tmp_path = msh_path.with_extension("msh.tmp"); tokio::fs::write(&tmp_path, body.as_bytes()).await?; tokio::fs::rename(&tmp_path, &msh_path).await?; - - self.tool_kill.stop_tool(MESH_TOOL_ID).await?; - if let Some(service_name) = self.mesh_service_name().await? { - if let Err(e) = system_service::start_service(&service_name).await { - debug!( - "mesh self-heal: start_service({service_name}) — likely already running: {e}" - ); - } - } Ok(true) } @@ -244,18 +299,6 @@ impl MeshSelfHealService { } Err(anyhow!("no .msh found in {}", dir.display())) } - - async fn mesh_service_name(&self) -> Result> { - let tool = self - .installed_tools - .get_by_tool_agent_id(MESH_TOOL_ID) - .await? - .ok_or_else(|| anyhow!("{MESH_TOOL_ID} is not installed"))?; - Ok(match tool.installation { - Installation::Service { service_name, .. } => Some(service_name), - _ => None, - }) - } } fn parse_msh_field(msh: &str, key: &str) -> Option { @@ -269,6 +312,43 @@ fn parse_msh_field(msh: &str, key: &str) -> Option { .filter(|v| !v.is_empty()) } +/// Seed the silence timer from the log's mtime so an already-silent agent isn't granted a fresh window on client restart; a recent boot (Instant underflow) falls back to now. +/// Staleness credit is capped below SILENCE_DURATION so at least STUCK_DURATION of live, monotonically-measured silence is observed before the branch can fire (also bounds wall-clock/NTP skew in mtime). +async fn seed_last_activity(path: &Path) -> Instant { + let now = Instant::now(); + let max_credit = SILENCE_DURATION.saturating_sub(STUCK_DURATION); + tokio::fs::metadata(path) + .await + .ok() + .and_then(|m| m.modified().ok()) + .and_then(|mtime| mtime.elapsed().ok()) + .and_then(|stale_for| now.checked_sub(stale_for.min(max_credit))) + .unwrap_or(now) +} + +/// Health of the last marker within the log tail: Some(true)=healthy, Some(false)=failing, None=no marker found. +async fn last_marker_in_tail(path: &Path) -> Option { + use tokio::io::{AsyncReadExt, AsyncSeekExt, SeekFrom}; + + let mut file = tokio::fs::File::open(path).await.ok()?; + let len = file.metadata().await.ok()?.len(); + file.seek(SeekFrom::Start(len.saturating_sub(TAIL_BYTES))) + .await + .ok()?; + let mut buf = Vec::new(); + file.read_to_end(&mut buf).await.ok()?; + + let mut last = None; + for line in String::from_utf8_lossy(&buf).lines() { + if line.contains(HEALTHY_MARKER) { + last = Some(true); + } else if line.contains(FAILURE_MARKER) { + last = Some(false); + } + } + last +} + async fn read_new_lines(path: &Path, offset: &mut u64) -> Result> { use tokio::io::{AsyncReadExt, AsyncSeekExt, SeekFrom}; @@ -296,3 +376,66 @@ async fn read_new_lines(path: &Path, offset: &mut u64) -> Result> { .map(|s| s.to_string()) .collect()) } + +#[cfg(test)] +mod tests { + use super::*; + + const FAILED_0_0_22_NO_HTTP: &str = "Connection FAILED: No HTTP response (fd=0, status=Complete/Disconnected, authState=0, connState=0, tls=down, elapsedMs=20016, attempt=ABCD1234-2100)"; + const FAILED_0_0_22_TIMEOUT: &str = "Connection FAILED: Network timeout - server unreachable or gateway blocking (tls=down, elapsedMs=21016, attempt=ABCD1234-2101)"; + const FAILED_0_0_23_PLUS: &str = "Connection FAILED (latest attempt): No HTTP response (fd=0, status=Complete/Disconnected, authState=0, connState=0, tls=down, elapsedMs=20016, attempt=ABCD1234-2102)"; + const CORE_OK: &str = "Received CoreOk from server (coreTimeout=0x0)"; + + #[test] + fn failure_marker_matches_0_0_22_formats() { + assert!(FAILED_0_0_22_NO_HTTP.contains(FAILURE_MARKER)); + assert!(FAILED_0_0_22_TIMEOUT.contains(FAILURE_MARKER)); + } + + #[test] + fn failure_marker_matches_0_0_23_plus_format() { + assert!(FAILED_0_0_23_PLUS.contains(FAILURE_MARKER)); + } + + #[test] + fn markers_ignore_unrelated_lines() { + for line in [ + "Connection: dialing uri=wss://x.openframe.ai/ws/tools/agent/meshcentral-server/agent.ashx host=x.openframe.ai port=443 family=IPv4 ip=1.2.3.4 useproxy=0 proxy=DIRECT attempt=ABCD1234-2103 suppressed=2", + "AutoRetry Connect in 299066 milliseconds", + ] { + assert!(!line.contains(FAILURE_MARKER)); + assert!(!line.contains(HEALTHY_MARKER)); + } + } + + #[test] + fn healthy_marker_matches_core_ok() { + assert!(CORE_OK.contains(HEALTHY_MARKER)); + } + + #[tokio::test] + async fn tail_seed_reports_last_marker() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("meshcentral-agent.log"); + + assert_eq!(last_marker_in_tail(&path).await, None); + + tokio::fs::write(&path, "startup\nno markers here\n") + .await + .unwrap(); + assert_eq!(last_marker_in_tail(&path).await, None); + + tokio::fs::write(&path, format!("{FAILED_0_0_22_NO_HTTP}\n{CORE_OK}\n")) + .await + .unwrap(); + assert_eq!(last_marker_in_tail(&path).await, Some(true)); + + tokio::fs::write( + &path, + format!("{CORE_OK}\n{FAILED_0_0_23_PLUS}\n{FAILED_0_0_22_TIMEOUT}\n"), + ) + .await + .unwrap(); + assert_eq!(last_marker_in_tail(&path).await, Some(false)); + } +} diff --git a/clients/openframe-client/src/services/mod.rs b/clients/openframe-client/src/services/mod.rs index ac87a3802..4a2f52c14 100644 --- a/clients/openframe-client/src/services/mod.rs +++ b/clients/openframe-client/src/services/mod.rs @@ -28,6 +28,7 @@ pub mod tool_connection_processing_manager; pub mod tool_connection_service; pub mod tool_installation_service; pub mod tool_kill_service; +pub mod tool_restart_service; pub mod tool_run_manager; pub mod tool_uninstall_service; pub mod tool_url_params_resolver; @@ -60,6 +61,7 @@ pub use tool_connection_processing_manager::ToolConnectionProcessingManager; pub use tool_connection_service::ToolConnectionService; pub use tool_installation_service::ToolInstallationService; pub use tool_kill_service::ToolKillService; +pub use tool_restart_service::{RestartOutcome, ToolRestartService}; pub use tool_run_manager::ToolRunManager; pub use tool_uninstall_service::{ToolUninstallService, UninstallOutcome}; pub use tool_url_params_resolver::ToolUrlParamsResolver; diff --git a/clients/openframe-client/src/services/tool_kill_service.rs b/clients/openframe-client/src/services/tool_kill_service.rs index 13aa46dc2..1e9a1b81e 100644 --- a/clients/openframe-client/src/services/tool_kill_service.rs +++ b/clients/openframe-client/src/services/tool_kill_service.rs @@ -5,7 +5,7 @@ use crate::config::service_stop::{ use crate::models::{Installation, InstalledTool}; use crate::platform::system_service; use anyhow::Result; -use sysinfo::{Pid, Signal, System}; +use sysinfo::{Pid, ProcessRefreshKind, Signal, System, UpdateKind}; use tokio::time::{sleep, Duration}; use tracing::{error, info, warn}; @@ -49,6 +49,56 @@ impl ToolKillService { .await } + /// Collect (pid, exe) of processes whose cmdline or exe path contains any of the patterns. + fn collect_matching_processes(patterns: &[String]) -> Vec<(Pid, String)> { + let mut sys = System::new(); + sys.refresh_processes_specifics( + ProcessRefreshKind::new() + .with_cmd(UpdateKind::Always) + .with_exe(UpdateKind::Always), + ); + sys.processes() + .iter() + .filter_map(|(pid, process)| { + let cmdline = process.cmd().join(" ").to_lowercase(); + let exe_path = process + .exe() + .map(|p| p.to_string_lossy().to_lowercase()) + .unwrap_or_default(); + patterns + .iter() + .any(|p| cmdline.contains(p.as_str()) || exe_path.contains(p.as_str())) + .then(|| (*pid, exe_path.clone())) + }) + .collect() + } + + /// Check whether the installed tool's process is running; the pattern set mirrors stop_for_installation's kill targets. + pub async fn is_installed_tool_running(&self, tool: &InstalledTool) -> bool { + let mut patterns: Vec = Vec::new(); + match &tool.installation { + Installation::GuiApp { + executable_path, .. + } => patterns.push(executable_path.to_lowercase()), + Installation::Standard { executable_path } => { + if let Some(path) = executable_path { + patterns.push(path.to_lowercase()); + } + patterns.push(Self::build_tool_cmd_pattern(&tool.tool_agent_id)); + } + Installation::Service { + executable_path, .. + } => match executable_path { + Some(path) => patterns.push(path.to_lowercase()), + // No registered path to mirror: fall back to the tool pattern rather than reporting a blind false. + None => patterns.push(Self::build_tool_cmd_pattern(&tool.tool_agent_id)), + }, + } + tokio::task::spawn_blocking(move || !Self::collect_matching_processes(&patterns).is_empty()) + .await + .unwrap_or(false) + } + /// Generic method to stop processes matching a command pattern /// /// This method will search for any running processes that match the given @@ -57,27 +107,20 @@ impl ToolKillService { info!("Attempting to stop {}", description); info!("Using pattern to stop: {}", pattern); - let mut sys = System::new_all(); - sys.refresh_all(); + let pattern_string = pattern.to_string(); + let matches = tokio::task::spawn_blocking(move || { + Self::collect_matching_processes(&[pattern_string]) + }) + .await + .unwrap_or_default(); let mut pids_to_stop = Vec::new(); - - // Find all matching processes by cmdline OR executable path - for (pid, process) in sys.processes() { - let cmd_items = process.cmd(); - let cmdline = cmd_items.join(" ").to_lowercase(); - let exe_path = process - .exe() - .map(|p| p.to_string_lossy().to_lowercase()) - .unwrap_or_default(); - - if cmdline.contains(pattern) || exe_path.contains(pattern) { - info!( - "Found process for {} with pid {} (exe: {})", - description, pid, exe_path - ); - pids_to_stop.push(*pid); - } + for (pid, exe_path) in matches { + info!( + "Found process for {} with pid {} (exe: {})", + description, pid, exe_path + ); + pids_to_stop.push(pid); } if pids_to_stop.is_empty() { diff --git a/clients/openframe-client/src/services/tool_restart_service.rs b/clients/openframe-client/src/services/tool_restart_service.rs new file mode 100644 index 000000000..bd6bf39da --- /dev/null +++ b/clients/openframe-client/src/services/tool_restart_service.rs @@ -0,0 +1,145 @@ +use crate::models::{Installation, InstalledTool}; +use crate::platform::system_service; +use crate::services::tool_run_manager::ToolRunManager; +use crate::services::InstalledToolsService; +use crate::services::ToolKillService; +use anyhow::{Context, Result}; +use futures::FutureExt; +use std::panic::AssertUnwindSafe; +use tracing::{info, warn}; + +pub enum RestartOutcome { + Restarted, + NotInstalled, + Busy, +} + +/// Clears the updating flag on drop (surviving cancellation and panic), releasing the tool lock only after the flag clears. +struct UpdatingGuard { + tool_run_manager: ToolRunManager, + tool_agent_id: String, + lock_guard: Option>, +} + +impl Drop for UpdatingGuard { + fn drop(&mut self) { + if let Ok(handle) = tokio::runtime::Handle::try_current() { + let manager = self.tool_run_manager.clone(); + let tool_agent_id = self.tool_agent_id.clone(); + let lock_guard = self.lock_guard.take(); + handle.spawn(async move { + manager.clear_updating(&tool_agent_id).await; + drop(lock_guard); + }); + } + } +} + +#[derive(Clone)] +pub struct ToolRestartService { + installed_tools_service: InstalledToolsService, + tool_kill_service: ToolKillService, + tool_run_manager: ToolRunManager, +} + +impl ToolRestartService { + pub fn new( + installed_tools_service: InstalledToolsService, + tool_kill_service: ToolKillService, + tool_run_manager: ToolRunManager, + ) -> Self { + Self { + installed_tools_service, + tool_kill_service, + tool_run_manager, + } + } + + /// Restart under the tool lock with the updating flag held; the flag is cleared exactly once on return, panic, or cancellation, and the lock is held until then. + pub async fn restart_guarded(&self, tool_agent_id: &str) -> Result { + let tool_lock = self.tool_run_manager.tool_lock(tool_agent_id).await; + let lock_guard = match tool_lock.try_lock_owned() { + Ok(guard) => guard, + Err(_) => return Ok(RestartOutcome::Busy), + }; + self.tool_run_manager.mark_updating(tool_agent_id).await; + let _updating = UpdatingGuard { + tool_run_manager: self.tool_run_manager.clone(), + tool_agent_id: tool_agent_id.to_string(), + lock_guard: Some(lock_guard), + }; + let outcome = AssertUnwindSafe(self.restart_by_tool_agent_id(tool_agent_id)) + .catch_unwind() + .await; + match outcome { + Ok(result) => result, + Err(_) => Err(anyhow::anyhow!( + "Restart panicked for tool {}", + tool_agent_id + )), + } + } + + async fn restart_by_tool_agent_id(&self, tool_agent_id: &str) -> Result { + match self + .installed_tools_service + .get_by_tool_agent_id(tool_agent_id) + .await? + { + None => { + info!( + "Tool {} not present in registry, nothing to restart", + tool_agent_id + ); + Ok(RestartOutcome::NotInstalled) + } + Some(tool) => { + self.restart_tool(&tool) + .await + .with_context(|| format!("Failed to restart tool: {}", tool_agent_id))?; + Ok(RestartOutcome::Restarted) + } + } + } + + async fn restart_tool(&self, tool: &InstalledTool) -> Result<()> { + let tool_agent_id = &tool.tool_agent_id; + + // Stop the tool: for Service installs this stops the OS service and kills detached children. + info!("Stopping tool for restart: {}", tool_agent_id); + self.tool_kill_service + .stop_installed_tool(tool, false) + .await + .with_context(|| format!("Failed to stop tool for restart: {}", tool_agent_id))?; + + match &tool.installation { + Installation::Service { service_name, .. } => { + // Services aren't supervised by the run manager, so start them back explicitly. + info!(service_name = %service_name, "Starting service tool back up"); + if let Err(e) = system_service::start_service(service_name).await { + // A start error can mean "already running" (e.g. relaunched by the OS supervisor) — verify before failing. + if self.tool_kill_service.is_installed_tool_running(tool).await { + info!(service_name = %service_name, "start_service failed but the tool is running — treating as restarted: {e:#}"); + } else { + warn!(service_name = %service_name, "start_service failed, retrying once: {e:#}"); + system_service::start_service(service_name) + .await + .with_context(|| format!("Failed to start service {}", service_name))?; + } + } + } + _ => { + // Supervised process: the run-manager loop relaunches it once the update flag clears. + self.tool_run_manager + .run_new_tool(tool.clone()) + .await + .with_context(|| { + format!("Failed to ensure supervision for tool: {}", tool_agent_id) + })?; + } + } + + info!("Tool {} restart triggered", tool_agent_id); + Ok(()) + } +} diff --git a/clients/openframe-client/src/services/tool_run_manager.rs b/clients/openframe-client/src/services/tool_run_manager.rs index 0790e6912..c5289acf7 100644 --- a/clients/openframe-client/src/services/tool_run_manager.rs +++ b/clients/openframe-client/src/services/tool_run_manager.rs @@ -699,6 +699,7 @@ impl ToolRunManager { if is_process_running(&command_path).await { info!(tool_id = %tool.tool_agent_id, "Already running, skipping launch"); + running_tools.write().await.remove(&tool.tool_agent_id); return; } @@ -754,6 +755,7 @@ impl ToolRunManager { sleep(Duration::from_secs(3)).await; if is_process_running(&command_path).await { info!(tool_id = %tool.tool_agent_id, "GuiApp verified running"); + running_tools.write().await.remove(&tool.tool_agent_id); return; } From c17eda840fdcebab3ac4abb50983c0a40b2e5e28 Mon Sep 17 00:00:00 2001 From: denys-gif Date: Mon, 20 Jul 2026 17:52:58 +0100 Subject: [PATCH 10/19] feat: listeners reconnect --- .../openframe_client_update_listener.rs | 49 ++++++++++++------- .../listener/tool_agent_update_listener.rs | 49 ++++++++++++------- .../tool_installation_message_listener.rs | 49 ++++++++++++------- .../tool_uninstall_message_listener.rs | 49 ++++++++++++------- .../src/services/nats_connection_manager.rs | 21 ++++++-- 5 files changed, 142 insertions(+), 75 deletions(-) diff --git a/clients/openframe-client/src/listener/openframe_client_update_listener.rs b/clients/openframe-client/src/listener/openframe_client_update_listener.rs index ba29c3148..31a499d3c 100644 --- a/clients/openframe-client/src/listener/openframe_client_update_listener.rs +++ b/clients/openframe-client/src/listener/openframe_client_update_listener.rs @@ -66,31 +66,44 @@ impl OpenFrameClientUpdateListener { async fn listen(&self) -> Result<()> { info!("Run OpenFrame client update message listener"); - let client = self.nats_connection_manager.get_client().await?; - let js = jetstream::new((*client).clone()); - let machine_id = self.config_service.get_machine_id()?; - let consumer = self.create_consumer(&js, &machine_id).await; + loop { + let client = self.nats_connection_manager.get_client().await?; + let mut reconnect_rx = self.nats_connection_manager.subscribe_reconnect(); + let js = jetstream::new((*client).clone()); - info!("Start listening for OpenFrame client update messages"); - let mut messages = consumer.messages().await?; + let consumer = self.create_consumer(&js, &machine_id).await; - while let Some(msg_result) = messages.next().await { - let message = match msg_result { - Ok(msg) => msg, - Err(e) => { - error!("Failed to receive message: {:#}", e); - continue; - } - }; + info!("Start listening for OpenFrame client update messages"); + let mut messages = consumer.messages().await?; - if let Err(e) = self.handle_message(message).await { - error!("Failed to handle message: {:#}", e); + loop { + tokio::select! { + msg_result = messages.next() => { + match msg_result { + Some(Ok(message)) => { + if let Err(e) = self.handle_message(message).await { + error!("Failed to handle message: {:#}", e); + } + } + Some(Err(e)) => { + error!("Message stream error, recreating consumer: {:#}", e); + return Err(anyhow::anyhow!("Message stream error: {}", e)); + } + None => { + warn!("Message stream ended, rebinding consumer"); + break; + } + } + } + _ = reconnect_rx.recv() => { + info!("NATS reconnected, rebinding OpenFrame client update consumer"); + break; + } + } } } - - Ok(()) } async fn handle_message(&self, message: Message) -> Result<()> { diff --git a/clients/openframe-client/src/listener/tool_agent_update_listener.rs b/clients/openframe-client/src/listener/tool_agent_update_listener.rs index 2047b97e6..647073e81 100644 --- a/clients/openframe-client/src/listener/tool_agent_update_listener.rs +++ b/clients/openframe-client/src/listener/tool_agent_update_listener.rs @@ -66,31 +66,44 @@ impl ToolAgentUpdateListener { async fn listen(&self) -> Result<()> { info!("Run tool agent update message listener"); - let client = self.nats_connection_manager.get_client().await?; - let js = jetstream::new((*client).clone()); - let machine_id = self.config_service.get_machine_id()?; - let consumer = self.create_consumer(&js, &machine_id).await; + loop { + let client = self.nats_connection_manager.get_client().await?; + let mut reconnect_rx = self.nats_connection_manager.subscribe_reconnect(); + let js = jetstream::new((*client).clone()); - info!("Start listening for tool agent update messages"); - let mut messages = consumer.messages().await?; + let consumer = self.create_consumer(&js, &machine_id).await; - while let Some(msg_result) = messages.next().await { - let message = match msg_result { - Ok(msg) => msg, - Err(e) => { - error!("Failed to receive message: {:#}", e); - continue; - } - }; + info!("Start listening for tool agent update messages"); + let mut messages = consumer.messages().await?; - if let Err(e) = self.handle_message(message).await { - error!("Failed to handle message: {:#}", e); + loop { + tokio::select! { + msg_result = messages.next() => { + match msg_result { + Some(Ok(message)) => { + if let Err(e) = self.handle_message(message).await { + error!("Failed to handle message: {:#}", e); + } + } + Some(Err(e)) => { + error!("Message stream error, recreating consumer: {:#}", e); + return Err(anyhow::anyhow!("Message stream error: {}", e)); + } + None => { + warn!("Message stream ended, rebinding consumer"); + break; + } + } + } + _ = reconnect_rx.recv() => { + info!("NATS reconnected, rebinding tool agent update consumer"); + break; + } + } } } - - Ok(()) } async fn handle_message(&self, message: Message) -> Result<()> { diff --git a/clients/openframe-client/src/listener/tool_installation_message_listener.rs b/clients/openframe-client/src/listener/tool_installation_message_listener.rs index fe12981c0..e9f4a0aef 100644 --- a/clients/openframe-client/src/listener/tool_installation_message_listener.rs +++ b/clients/openframe-client/src/listener/tool_installation_message_listener.rs @@ -65,31 +65,44 @@ impl ToolInstallationMessageListener { async fn listen(&self) -> Result<()> { info!("Run tool installation message listener"); - let client = self.nats_connection_manager.get_client().await?; - let js = jetstream::new((*client).clone()); - let machine_id = self.config_service.get_machine_id()?; - let consumer = self.create_consumer(&js, &machine_id).await; + loop { + let client = self.nats_connection_manager.get_client().await?; + let mut reconnect_rx = self.nats_connection_manager.subscribe_reconnect(); + let js = jetstream::new((*client).clone()); - info!("Start listening for tool installation messages"); - let mut messages = consumer.messages().await?; + let consumer = self.create_consumer(&js, &machine_id).await; - while let Some(msg_result) = messages.next().await { - let message = match msg_result { - Ok(msg) => msg, - Err(e) => { - error!("Failed to receive message: {:#}", e); - continue; - } - }; + info!("Start listening for tool installation messages"); + let mut messages = consumer.messages().await?; - if let Err(e) = self.handle_message(message).await { - error!("Failed to handle message: {:#}", e); + loop { + tokio::select! { + msg_result = messages.next() => { + match msg_result { + Some(Ok(message)) => { + if let Err(e) = self.handle_message(message).await { + error!("Failed to handle message: {:#}", e); + } + } + Some(Err(e)) => { + error!("Message stream error, recreating consumer: {:#}", e); + return Err(anyhow::anyhow!("Message stream error: {}", e)); + } + None => { + warn!("Message stream ended, rebinding consumer"); + break; + } + } + } + _ = reconnect_rx.recv() => { + info!("NATS reconnected, rebinding tool installation consumer"); + break; + } + } } } - - Ok(()) } async fn handle_message(&self, message: Message) -> Result<()> { diff --git a/clients/openframe-client/src/listener/tool_uninstall_message_listener.rs b/clients/openframe-client/src/listener/tool_uninstall_message_listener.rs index edbf95280..52bdfd81a 100644 --- a/clients/openframe-client/src/listener/tool_uninstall_message_listener.rs +++ b/clients/openframe-client/src/listener/tool_uninstall_message_listener.rs @@ -69,31 +69,44 @@ impl ToolUninstallMessageListener { async fn listen(&self) -> Result<()> { info!("Run tool uninstall message listener"); - let client = self.nats_connection_manager.get_client().await?; - let js = jetstream::new((*client).clone()); - let machine_id = self.config_service.get_machine_id()?; - let consumer = self.create_consumer(&js, &machine_id).await; + loop { + let client = self.nats_connection_manager.get_client().await?; + let mut reconnect_rx = self.nats_connection_manager.subscribe_reconnect(); + let js = jetstream::new((*client).clone()); - info!("Start listening for tool uninstall messages"); - let mut messages = consumer.messages().await?; + let consumer = self.create_consumer(&js, &machine_id).await; - while let Some(msg_result) = messages.next().await { - let message = match msg_result { - Ok(msg) => msg, - Err(e) => { - error!("Failed to receive message: {:#}", e); - continue; - } - }; + info!("Start listening for tool uninstall messages"); + let mut messages = consumer.messages().await?; - if let Err(e) = self.handle_message(message).await { - error!("Failed to handle message: {:#}", e); + loop { + tokio::select! { + msg_result = messages.next() => { + match msg_result { + Some(Ok(message)) => { + if let Err(e) = self.handle_message(message).await { + error!("Failed to handle message: {:#}", e); + } + } + Some(Err(e)) => { + error!("Message stream error, recreating consumer: {:#}", e); + return Err(anyhow::anyhow!("Message stream error: {}", e)); + } + None => { + warn!("Message stream ended, rebinding consumer"); + break; + } + } + } + _ = reconnect_rx.recv() => { + info!("NATS reconnected, rebinding tool uninstall consumer"); + break; + } + } } } - - Ok(()) } async fn handle_message(&self, message: Message) -> Result<()> { diff --git a/clients/openframe-client/src/services/nats_connection_manager.rs b/clients/openframe-client/src/services/nats_connection_manager.rs index efccfa243..7959b75d1 100644 --- a/clients/openframe-client/src/services/nats_connection_manager.rs +++ b/clients/openframe-client/src/services/nats_connection_manager.rs @@ -2,15 +2,17 @@ use crate::services::agent_configuration_service::AgentConfigurationService; use crate::services::local_tls_config_provider::LocalTlsConfigProvider; use crate::services::{AgentAuthService, InitialConfigurationService}; use anyhow::{Context, Result}; -use async_nats::Client; +use async_nats::{Client, Event}; use log::error; use std::sync::Arc; +use tokio::sync::broadcast; use tokio::sync::RwLock; use tracing::{info, warn}; #[derive(Clone)] pub struct NatsConnectionManager { client: Arc>>>, + reconnect_tx: broadcast::Sender<()>, nats_server_url: String, config_service: AgentConfigurationService, tls_config_provider: LocalTlsConfigProvider, @@ -29,8 +31,10 @@ impl NatsConnectionManager { auth_service: AgentAuthService, tls_config_provider: LocalTlsConfigProvider, ) -> Self { + let (reconnect_tx, _) = broadcast::channel(16); Self { client: Arc::new(RwLock::new(None)), + reconnect_tx, nats_server_url: nats_server_url.to_string(), config_service, tls_config_provider, @@ -39,6 +43,10 @@ impl NatsConnectionManager { } } + pub fn subscribe_reconnect(&self) -> broadcast::Receiver<()> { + self.reconnect_tx.subscribe() + } + pub async fn connect(&self) -> Result<()> { let machine_id = self.config_service.get_machine_id()?; @@ -54,6 +62,7 @@ impl NatsConnectionManager { let config_service = self.config_service.clone(); let nats_server_url = self.nats_server_url.clone(); let nats_server_url_for_reconnect = self.nats_server_url.clone(); + let reconnect_tx = self.reconnect_tx.clone(); // TODO: token fallback and connection retry let mut connect_options = async_nats::ConnectOptions::new() @@ -73,8 +82,14 @@ impl NatsConnectionManager { std::time::Duration::from_secs(5) }) .ping_interval(std::time::Duration::from_secs(10)) - .event_callback(|event| async move { - info!("Nats event: {:?}", event); + .event_callback(move |event| { + let reconnect_tx = reconnect_tx.clone(); + async move { + info!("Nats event: {:?}", event); + if matches!(event, Event::Connected) { + let _ = reconnect_tx.send(()); + } + } }) .auth_url_callback(move |()| { info!("Starting reauthentication"); From 2b2e2fa6c5bd6c867debe33a07ecae4fabff4425 Mon Sep 17 00:00:00 2001 From: denys-gif Date: Tue, 21 Jul 2026 11:39:10 +0100 Subject: [PATCH 11/19] fix: nats message lost during rebind --- .../src/listener/openframe_client_update_listener.rs | 4 ++-- .../src/listener/tool_agent_update_listener.rs | 4 ++-- .../src/listener/tool_installation_message_listener.rs | 4 ++-- .../src/listener/tool_uninstall_message_listener.rs | 4 ++-- .../src/services/nats_connection_manager.rs | 7 ++++++- 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/clients/openframe-client/src/listener/openframe_client_update_listener.rs b/clients/openframe-client/src/listener/openframe_client_update_listener.rs index 31a499d3c..0d76410ba 100644 --- a/clients/openframe-client/src/listener/openframe_client_update_listener.rs +++ b/clients/openframe-client/src/listener/openframe_client_update_listener.rs @@ -98,8 +98,8 @@ impl OpenFrameClientUpdateListener { } } _ = reconnect_rx.recv() => { - info!("NATS reconnected, rebinding OpenFrame client update consumer"); - break; + info!("NATS reconnected, re-provisioning OpenFrame client update consumer"); + self.create_consumer(&js, &machine_id).await; } } } diff --git a/clients/openframe-client/src/listener/tool_agent_update_listener.rs b/clients/openframe-client/src/listener/tool_agent_update_listener.rs index 647073e81..ef824a3b1 100644 --- a/clients/openframe-client/src/listener/tool_agent_update_listener.rs +++ b/clients/openframe-client/src/listener/tool_agent_update_listener.rs @@ -98,8 +98,8 @@ impl ToolAgentUpdateListener { } } _ = reconnect_rx.recv() => { - info!("NATS reconnected, rebinding tool agent update consumer"); - break; + info!("NATS reconnected, re-provisioning tool agent update consumer"); + self.create_consumer(&js, &machine_id).await; } } } diff --git a/clients/openframe-client/src/listener/tool_installation_message_listener.rs b/clients/openframe-client/src/listener/tool_installation_message_listener.rs index e9f4a0aef..616207d24 100644 --- a/clients/openframe-client/src/listener/tool_installation_message_listener.rs +++ b/clients/openframe-client/src/listener/tool_installation_message_listener.rs @@ -97,8 +97,8 @@ impl ToolInstallationMessageListener { } } _ = reconnect_rx.recv() => { - info!("NATS reconnected, rebinding tool installation consumer"); - break; + info!("NATS reconnected, re-provisioning tool installation consumer"); + self.create_consumer(&js, &machine_id).await; } } } diff --git a/clients/openframe-client/src/listener/tool_uninstall_message_listener.rs b/clients/openframe-client/src/listener/tool_uninstall_message_listener.rs index 52bdfd81a..225b96557 100644 --- a/clients/openframe-client/src/listener/tool_uninstall_message_listener.rs +++ b/clients/openframe-client/src/listener/tool_uninstall_message_listener.rs @@ -101,8 +101,8 @@ impl ToolUninstallMessageListener { } } _ = reconnect_rx.recv() => { - info!("NATS reconnected, rebinding tool uninstall consumer"); - break; + info!("NATS reconnected, re-provisioning tool uninstall consumer"); + self.create_consumer(&js, &machine_id).await; } } } diff --git a/clients/openframe-client/src/services/nats_connection_manager.rs b/clients/openframe-client/src/services/nats_connection_manager.rs index 7959b75d1..040790f38 100644 --- a/clients/openframe-client/src/services/nats_connection_manager.rs +++ b/clients/openframe-client/src/services/nats_connection_manager.rs @@ -4,6 +4,7 @@ use crate::services::{AgentAuthService, InitialConfigurationService}; use anyhow::{Context, Result}; use async_nats::{Client, Event}; use log::error; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use tokio::sync::broadcast; use tokio::sync::RwLock; @@ -63,6 +64,7 @@ impl NatsConnectionManager { let nats_server_url = self.nats_server_url.clone(); let nats_server_url_for_reconnect = self.nats_server_url.clone(); let reconnect_tx = self.reconnect_tx.clone(); + let connected_once = Arc::new(AtomicBool::new(false)); // TODO: token fallback and connection retry let mut connect_options = async_nats::ConnectOptions::new() @@ -84,9 +86,12 @@ impl NatsConnectionManager { .ping_interval(std::time::Duration::from_secs(10)) .event_callback(move |event| { let reconnect_tx = reconnect_tx.clone(); + let connected_once = connected_once.clone(); async move { info!("Nats event: {:?}", event); - if matches!(event, Event::Connected) { + if matches!(event, Event::Connected) + && connected_once.swap(true, Ordering::SeqCst) + { let _ = reconnect_tx.send(()); } } From 882e18dce46fc7c2496cd1198c7cb71715b31e23 Mon Sep 17 00:00:00 2001 From: mikhailm-coder Date: Tue, 21 Jul 2026 15:05:48 +0200 Subject: [PATCH 12/19] client: disable tool-restart listener until backend support, align it with sibling listeners (#2163) Co-authored-by: Claude Fable 5 --- .../src/config/update_config.rs | 1 - clients/openframe-client/src/lib.rs | 4 +- .../listener/tool_restart_message_listener.rs | 119 ++++++++---------- 3 files changed, 56 insertions(+), 68 deletions(-) diff --git a/clients/openframe-client/src/config/update_config.rs b/clients/openframe-client/src/config/update_config.rs index 9614bd17f..79bd55df8 100644 --- a/clients/openframe-client/src/config/update_config.rs +++ b/clients/openframe-client/src/config/update_config.rs @@ -24,4 +24,3 @@ pub const CONSUMER_ACK_WAIT_SECS: u64 = 120; pub const CONSUMER_MAX_DELIVER: i64 = 10; // Maximum delivery attempts pub const UNINSTALL_CONSUMER_MAX_DELIVER: i64 = 20; // Larger budget: uninstall may defer behind a long install holding the tool lock pub const RESTART_CONSUMER_MAX_DELIVER: i64 = 20; // Larger budget: restart may defer behind a long install holding the tool lock -pub const RESTART_CONSUMER_QUIET_PAUSE_MS: u64 = 300_000; // Quiet retry cadence once consumer creation keeps failing (subject/grants not provisioned yet) diff --git a/clients/openframe-client/src/lib.rs b/clients/openframe-client/src/lib.rs index efb87fda8..7cf358e23 100644 --- a/clients/openframe-client/src/lib.rs +++ b/clients/openframe-client/src/lib.rs @@ -152,6 +152,7 @@ pub struct Client { nats_connection_manager: NatsConnectionManager, tool_installation_message_listener: ToolInstallationMessageListener, tool_uninstall_message_listener: ToolUninstallMessageListener, + #[allow(dead_code)] // TODO: remove when tool-restart is implemented on backend tool_restart_message_listener: ToolRestartMessageListener, openframe_client_update_listener: OpenFrameClientUpdateListener, tool_agent_update_listener: ToolAgentUpdateListener, @@ -553,7 +554,8 @@ impl Client { self.tool_uninstall_message_listener.start().await?; - self.tool_restart_message_listener.start().await?; + // TODO: uncomment when implemented on backend + // self.tool_restart_message_listener.start().await?; // Start OpenFrame client update listener in background self.openframe_client_update_listener.start().await?; diff --git a/clients/openframe-client/src/listener/tool_restart_message_listener.rs b/clients/openframe-client/src/listener/tool_restart_message_listener.rs index deba0b62c..ded3620ad 100644 --- a/clients/openframe-client/src/listener/tool_restart_message_listener.rs +++ b/clients/openframe-client/src/listener/tool_restart_message_listener.rs @@ -1,7 +1,7 @@ use crate::config::update_config::{ CONSUMER_ACK_WAIT_SECS, CONSUMER_CYCLE_PAUSE_MS, CONSUMER_RETRY_ATTEMPTS_PER_CYCLE, INITIAL_RETRY_DELAY_MS, MAX_RETRY_DELAY_MS, RECONNECTION_DELAY_MS, - RESTART_CONSUMER_MAX_DELIVER, RESTART_CONSUMER_QUIET_PAUSE_MS, + RESTART_CONSUMER_MAX_DELIVER, }; use crate::models::ToolRestartMessage; use crate::services::nats_connection_manager::NatsConnectionManager; @@ -15,7 +15,7 @@ use async_nats::jetstream::consumer::PushConsumer; use async_nats::jetstream::Message; use futures::StreamExt; use tokio::time::Duration; -use tracing::{debug, error, info, warn}; +use tracing::{error, info, warn}; #[derive(Clone)] pub struct ToolRestartMessageListener { @@ -65,31 +65,44 @@ impl ToolRestartMessageListener { async fn listen(&self) -> Result<()> { info!("Run tool restart message listener"); - let client = self.nats_connection_manager.get_client().await?; - let js = jetstream::new((*client).clone()); - let machine_id = self.config_service.get_machine_id()?; - let consumer = self.create_consumer(&js, &machine_id).await; + loop { + let client = self.nats_connection_manager.get_client().await?; + let mut reconnect_rx = self.nats_connection_manager.subscribe_reconnect(); + let js = jetstream::new((*client).clone()); - info!("Start listening for tool restart messages"); - let mut messages = consumer.messages().await?; + let consumer = self.create_consumer(&js, &machine_id).await; - while let Some(msg_result) = messages.next().await { - let message = match msg_result { - Ok(msg) => msg, - Err(e) => { - error!("Failed to receive message: {:#}", e); - continue; - } - }; + info!("Start listening for tool restart messages"); + let mut messages = consumer.messages().await?; - if let Err(e) = self.handle_message(message).await { - error!("Failed to handle message: {:#}", e); + loop { + tokio::select! { + msg_result = messages.next() => { + match msg_result { + Some(Ok(message)) => { + if let Err(e) = self.handle_message(message).await { + error!("Failed to handle message: {:#}", e); + } + } + Some(Err(e)) => { + error!("Message stream error, recreating consumer: {:#}", e); + return Err(anyhow::anyhow!("Message stream error: {}", e)); + } + None => { + warn!("Message stream ended, rebinding consumer"); + break; + } + } + } + _ = reconnect_rx.recv() => { + info!("NATS reconnected, re-provisioning tool restart consumer"); + self.create_consumer(&js, &machine_id).await; + } + } } } - - Ok(()) } async fn handle_message(&self, message: Message) -> Result<()> { @@ -151,27 +164,15 @@ impl ToolRestartMessageListener { loop { cycle += 1; let mut delay_ms = INITIAL_RETRY_DELAY_MS; - // First cycle logs loudly; later cycles go quiet with a long pause so a server missing the tool-restart subject/grants can't spam the fleet's logs. - let loud = cycle == 1; for attempt in 1..=CONSUMER_RETRY_ATTEMPTS_PER_CYCLE { - if loud { - info!( - "Creating restart consumer for stream {} (cycle {}, attempt {}/{})", - Self::STREAM_NAME, - cycle, - attempt, - CONSUMER_RETRY_ATTEMPTS_PER_CYCLE - ); - } else { - debug!( - "Creating restart consumer for stream {} (cycle {}, attempt {}/{})", - Self::STREAM_NAME, - cycle, - attempt, - CONSUMER_RETRY_ATTEMPTS_PER_CYCLE - ); - } + info!( + "Creating restart consumer for stream {} (cycle {}, attempt {}/{})", + Self::STREAM_NAME, + cycle, + attempt, + CONSUMER_RETRY_ATTEMPTS_PER_CYCLE + ); match js .create_consumer_on_stream(consumer_configuration.clone(), Self::STREAM_NAME) @@ -200,44 +201,30 @@ impl ToolRestartMessageListener { } } - if loud { + if attempt < CONSUMER_RETRY_ATTEMPTS_PER_CYCLE { warn!( - "Failed to create restart consumer (cycle {}, attempt {}/{}): {:#}", - cycle, attempt, CONSUMER_RETRY_ATTEMPTS_PER_CYCLE, e + "Failed to create restart consumer (cycle {}, attempt {}/{}): {:#}. Retrying in {} ms...", + cycle, attempt, CONSUMER_RETRY_ATTEMPTS_PER_CYCLE, e, delay_ms ); + tokio::time::sleep(Duration::from_millis(delay_ms)).await; + delay_ms = (delay_ms * 2).min(MAX_RETRY_DELAY_MS); } else { - debug!( + warn!( "Failed to create restart consumer (cycle {}, attempt {}/{}): {:#}", cycle, attempt, CONSUMER_RETRY_ATTEMPTS_PER_CYCLE, e ); } - if attempt < CONSUMER_RETRY_ATTEMPTS_PER_CYCLE { - tokio::time::sleep(Duration::from_millis(delay_ms)).await; - delay_ms = (delay_ms * 2).min(MAX_RETRY_DELAY_MS); - } } } } - let pause_ms = if loud { - CONSUMER_CYCLE_PAUSE_MS - } else { - RESTART_CONSUMER_QUIET_PAUSE_MS - }; - if loud { - warn!( - "All {} attempts in cycle {} failed (tool-restart stream subject/permissions may not be provisioned yet). Retrying quietly every {} seconds...", - CONSUMER_RETRY_ATTEMPTS_PER_CYCLE, cycle, RESTART_CONSUMER_QUIET_PAUSE_MS / 1000 - ); - } else { - debug!( - "All {} attempts in cycle {} failed. Pausing {} seconds before next cycle...", - CONSUMER_RETRY_ATTEMPTS_PER_CYCLE, - cycle, - pause_ms / 1000 - ); - } - tokio::time::sleep(Duration::from_millis(pause_ms)).await; + info!( + "All {} attempts in cycle {} failed. Pausing {} seconds before next cycle...", + CONSUMER_RETRY_ATTEMPTS_PER_CYCLE, + cycle, + CONSUMER_CYCLE_PAUSE_MS / 1000 + ); + tokio::time::sleep(Duration::from_millis(CONSUMER_CYCLE_PAUSE_MS)).await; } } From 47006c6ac811a3ee0b46f29065dba75a438e902c Mon Sep 17 00:00:00 2001 From: Danylo Date: Tue, 21 Jul 2026 19:29:14 +0300 Subject: [PATCH 13/19] Hotfix/client update ordering and lkg ratchet v2 (#2169) --- .../src/config/update_config.rs | 16 ++ clients/openframe-client/src/lib.rs | 35 ++- .../src/listener/client_update_gate.rs | 45 ++++ clients/openframe-client/src/listener/mod.rs | 1 + .../listener/tool_agent_update_listener.rs | 39 ++- .../tool_installation_message_listener.rs | 39 ++- .../listener/tool_restart_message_listener.rs | 37 ++- .../tool_uninstall_message_listener.rs | 34 ++- .../src/models/update_state.rs | 10 + .../src/platform/installation_detector.rs | 26 +- .../src/platform/system_service.rs | 14 + .../src/platform/update_scripts/macos.rs | 250 ++++++++++++++---- .../src/platform/update_scripts/windows.rs | 203 ++++++++++++-- .../src/platform/updater_launcher/macos.rs | 21 +- .../src/platform/updater_launcher/mod.rs | 5 + .../src/platform/updater_launcher/windows.rs | 20 +- clients/openframe-client/src/service.rs | 105 +++++++- .../openframe-client/src/service_adapter.rs | 17 +- .../src/services/last_known_good_service.rs | 222 ++++++++++++++++ clients/openframe-client/src/services/mod.rs | 2 + .../services/openframe_client_info_service.rs | 14 + .../openframe_client_update_service.rs | 71 ++++- .../src/services/tool_run_manager.rs | 94 +++++++ .../src/services/update_cleanup_service.rs | 48 +++- .../src/services/update_handler_service.rs | 160 +++++++++-- .../src/services/update_state_service.rs | 7 +- 26 files changed, 1327 insertions(+), 208 deletions(-) create mode 100644 clients/openframe-client/src/listener/client_update_gate.rs create mode 100644 clients/openframe-client/src/services/last_known_good_service.rs diff --git a/clients/openframe-client/src/config/update_config.rs b/clients/openframe-client/src/config/update_config.rs index 79bd55df8..d3902a648 100644 --- a/clients/openframe-client/src/config/update_config.rs +++ b/clients/openframe-client/src/config/update_config.rs @@ -19,8 +19,24 @@ pub const RECONNECTION_DELAY_MS: u64 = 5000; // 5 seconds // Execution concurrency pub const EXECUTION_MIN_CONCURRENCY: usize = 4; +// Last-known-good update ratchet +/// How long the updater waits for the new binary's boot marker. +pub const BOOT_MARKER_WAIT_SECS: u64 = 90; +/// Unverified boots tolerated before an update is treated as failed. +pub const CRASH_LOOP_MAX_BOOT_ATTEMPTS: u32 = 3; +/// Refuse update messages below the LKG anchor (flip to force a downgrade). +pub const ALLOW_DOWNGRADE: bool = false; +/// Updater transcripts kept after pruning; one is written per update attempt. +pub const UPDATER_TRANSCRIPTS_KEPT: usize = 5; +/// Minimum age before a temp update leftover is swept (a live updater's files are younger). +pub const TEMP_LEFTOVER_MIN_AGE_SECS: u64 = 3600; + // NATS message settings pub const CONSUMER_ACK_WAIT_SECS: u64 = 120; pub const CONSUMER_MAX_DELIVER: i64 = 10; // Maximum delivery attempts pub const UNINSTALL_CONSUMER_MAX_DELIVER: i64 = 20; // Larger budget: uninstall may defer behind a long install holding the tool lock pub const RESTART_CONSUMER_MAX_DELIVER: i64 = 20; // Larger budget: restart may defer behind a long install holding the tool lock + +// Client-before-tool update ordering +pub const CLIENT_UPDATE_PENDING_TTL_SECS: u64 = 300; // > ack_wait (120s) so the flag survives redelivery gaps of a deferred client update +pub const PROGRESS_ACK_INTERVAL_SECS: u64 = 60; // well inside ack_wait (120s) diff --git a/clients/openframe-client/src/lib.rs b/clients/openframe-client/src/lib.rs index 7cf358e23..3d70402d5 100644 --- a/clients/openframe-client/src/lib.rs +++ b/clients/openframe-client/src/lib.rs @@ -82,7 +82,8 @@ use crate::services::{ ToolUrlParamsResolver, }; use crate::services::{ - InitialKeyService, UpdateCleanupService, UpdateHandlerService, UpdateStateService, + InitialKeyService, LastKnownGoodService, UpdateCleanupService, UpdateHandlerService, + UpdateStateService, }; #[derive(Debug, Clone, Serialize, Deserialize)] @@ -164,6 +165,8 @@ pub struct Client { tool_connection_processing_manager: ToolConnectionProcessingManager, machine_heartbeat_run_manager: MachineHeartbeatRunManager, update_handler_service: UpdateHandlerService, + openframe_client_info_service: OpenFrameClientInfoService, + last_known_good_service: LastKnownGoodService, // Services needed for log streaming initialization initial_configuration_service: InitialConfigurationService, agent_configuration_service: AgentConfigurationService, @@ -377,6 +380,9 @@ impl Client { let update_cleanup_service = UpdateCleanupService::new().context("Failed to initialize update cleanup service")?; + let last_known_good_service = LastKnownGoodService::new(directory_manager.clone()) + .context("Failed to initialize last-known-good service")?; + // Initialize tool installation service let tool_installation_service = ToolInstallationService::new( github_download_service.clone(), @@ -398,6 +404,7 @@ impl Client { openframe_client_info_service.clone(), github_download_service.clone(), update_state_service.clone(), + last_known_good_service.clone(), tool_run_manager.clone(), ); @@ -419,6 +426,7 @@ impl Client { nats_connection_manager.clone(), tool_installation_service, config_service.clone(), + tool_run_manager.clone(), ); let tool_uninstall_service = ToolUninstallService::new( @@ -439,6 +447,7 @@ impl Client { nats_connection_manager.clone(), tool_restart_service, config_service.clone(), + tool_run_manager.clone(), ); // Initialize OpenFrame client update listener @@ -453,6 +462,7 @@ impl Client { nats_connection_manager.clone(), tool_agent_update_service, config_service.clone(), + tool_run_manager.clone(), ); let execution_service = ExecutionService::new(); @@ -487,6 +497,7 @@ impl Client { update_state_service.clone(), openframe_client_info_service.clone(), update_cleanup_service.clone(), + last_known_good_service.clone(), installed_agent_message_publisher.clone(), config_service.clone(), ); @@ -510,6 +521,8 @@ impl Client { tool_connection_processing_manager, machine_heartbeat_run_manager, update_handler_service, + openframe_client_info_service, + last_known_good_service, initial_configuration_service, agent_configuration_service: config_service, installed_tools_service, @@ -520,6 +533,26 @@ impl Client { pub async fn start(&self) -> Result<()> { info!("Starting OpenFrame Client"); + if let Err(e) = self + .openframe_client_info_service + .reconcile_version(env!("OPENFRAME_VERSION")) + .await + { + error!("Failed to reconcile client version at startup: {:#}", e); + } + + if let Err(e) = self.last_known_good_service.write_boot_marker().await { + error!("Failed to write boot marker: {:#}", e); + } + + if let Err(e) = self.last_known_good_service.seed_if_missing().await { + error!("Failed to seed last-known-good anchor: {:#}", e); + } + + if let Err(e) = self.update_handler_service.record_boot_attempt().await { + error!("Failed to record boot attempt: {:#}", e); + } + self.initial_key_service.clone().ensure_initial_key().await; LogStreamingRunManager::new( diff --git a/clients/openframe-client/src/listener/client_update_gate.rs b/clients/openframe-client/src/listener/client_update_gate.rs new file mode 100644 index 000000000..d5943ec41 --- /dev/null +++ b/clients/openframe-client/src/listener/client_update_gate.rs @@ -0,0 +1,45 @@ +use crate::config::update_config::PROGRESS_ACK_INTERVAL_SECS; +use crate::services::tool_run_manager::ToolRunManager; +use async_nats::jetstream::{AckKind, Message}; +use std::future::Future; +use tokio::time::Duration; +use tracing::{info, warn}; + +pub async fn park_or_dispatch( + manager: ToolRunManager, + message: Message, + label: String, + dispatch: F, +) where + F: FnOnce(Message) -> Fut + Send + 'static, + Fut: Future + Send + 'static, +{ + if !manager.is_client_update_pending().await { + dispatch(message).await; + return; + } + + info!( + "Client update pending: parking {} (keeping it alive with Progress acks every {}s)", + label, PROGRESS_ACK_INTERVAL_SECS + ); + + tokio::spawn(async move { + while manager.is_client_update_pending().await { + if let Err(e) = message.ack_with(AckKind::Progress).await { + warn!( + "Failed to send Progress ack for parked {}: {} — abandoning park, redelivery takes over", + label, e + ); + return; + } + tokio::time::sleep(Duration::from_secs(PROGRESS_ACK_INTERVAL_SECS)).await; + } + + info!( + "Client update no longer pending: dispatching parked {}", + label + ); + dispatch(message).await; + }); +} diff --git a/clients/openframe-client/src/listener/mod.rs b/clients/openframe-client/src/listener/mod.rs index 731eebc7e..258588ff7 100644 --- a/clients/openframe-client/src/listener/mod.rs +++ b/clients/openframe-client/src/listener/mod.rs @@ -1,3 +1,4 @@ +pub mod client_update_gate; pub mod execution_listener; pub mod openframe_client_update_listener; pub mod tool_agent_update_listener; diff --git a/clients/openframe-client/src/listener/tool_agent_update_listener.rs b/clients/openframe-client/src/listener/tool_agent_update_listener.rs index ef824a3b1..607606630 100644 --- a/clients/openframe-client/src/listener/tool_agent_update_listener.rs +++ b/clients/openframe-client/src/listener/tool_agent_update_listener.rs @@ -3,9 +3,11 @@ use crate::config::update_config::{ CONSUMER_RETRY_ATTEMPTS_PER_CYCLE, INITIAL_RETRY_DELAY_MS, MAX_RETRY_DELAY_MS, RECONNECTION_DELAY_MS, }; +use crate::listener::client_update_gate::park_or_dispatch; use crate::models::tool_agent_update_message::ToolAgentUpdateMessage; use crate::services::nats_connection_manager::NatsConnectionManager; use crate::services::tool_agent_update_service::ToolAgentUpdateService; +use crate::services::tool_run_manager::ToolRunManager; use crate::services::AgentConfigurationService; use anyhow::Result; use async_nats::jetstream; @@ -22,6 +24,7 @@ pub struct ToolAgentUpdateListener { pub nats_connection_manager: NatsConnectionManager, pub tool_agent_update_service: ToolAgentUpdateService, pub config_service: AgentConfigurationService, + pub tool_run_manager: ToolRunManager, } impl ToolAgentUpdateListener { @@ -31,11 +34,13 @@ impl ToolAgentUpdateListener { nats_connection_manager: NatsConnectionManager, tool_agent_update_service: ToolAgentUpdateService, config_service: AgentConfigurationService, + tool_run_manager: ToolRunManager, ) -> Self { Self { nats_connection_manager, tool_agent_update_service, config_service, + tool_run_manager, } } @@ -124,6 +129,23 @@ impl ToolAgentUpdateListener { let tool_agent_id = tool_agent_update_message.tool_agent_id.clone(); + let listener = self.clone(); + park_or_dispatch( + self.tool_run_manager.clone(), + message, + format!("tool-update:{}", tool_agent_id), + move |msg| async move { + listener.dispatch(msg, tool_agent_update_message).await; + }, + ) + .await; + + Ok(()) + } + + async fn dispatch(&self, message: Message, tool_agent_update_message: ToolAgentUpdateMessage) { + let tool_agent_id = tool_agent_update_message.tool_agent_id.clone(); + match self .tool_agent_update_service .process_update(tool_agent_update_message) @@ -134,14 +156,13 @@ impl ToolAgentUpdateListener { "Acknowledging tool agent update message for tool: {}", tool_agent_id ); - message - .ack() - .await - .map_err(|e| anyhow::anyhow!("Failed to ack message: {}", e))?; - info!( - "Tool agent update message acknowledged for tool: {}", - tool_agent_id - ); + match message.ack().await { + Ok(_) => info!( + "Tool agent update message acknowledged for tool: {}", + tool_agent_id + ), + Err(e) => error!("Failed to ack message for tool {}: {}", tool_agent_id, e), + } } Err(e) => { error!( @@ -154,8 +175,6 @@ impl ToolAgentUpdateListener { ); } } - - Ok(()) } async fn create_consumer(&self, js: &jetstream::Context, machine_id: &str) -> PushConsumer { diff --git a/clients/openframe-client/src/listener/tool_installation_message_listener.rs b/clients/openframe-client/src/listener/tool_installation_message_listener.rs index 616207d24..42ac2e75e 100644 --- a/clients/openframe-client/src/listener/tool_installation_message_listener.rs +++ b/clients/openframe-client/src/listener/tool_installation_message_listener.rs @@ -3,9 +3,11 @@ use crate::config::update_config::{ CONSUMER_RETRY_ATTEMPTS_PER_CYCLE, INITIAL_RETRY_DELAY_MS, MAX_RETRY_DELAY_MS, RECONNECTION_DELAY_MS, }; +use crate::listener::client_update_gate::park_or_dispatch; use crate::models::tool_installation_message::ToolInstallationMessage; use crate::services::nats_connection_manager::NatsConnectionManager; use crate::services::tool_installation_service::ToolInstallationService; +use crate::services::tool_run_manager::ToolRunManager; use crate::services::AgentConfigurationService; use anyhow::Result; use async_nats::jetstream; @@ -21,6 +23,7 @@ pub struct ToolInstallationMessageListener { pub nats_connection_manager: NatsConnectionManager, pub tool_installation_service: ToolInstallationService, pub config_service: AgentConfigurationService, + pub tool_run_manager: ToolRunManager, } impl ToolInstallationMessageListener { @@ -30,11 +33,13 @@ impl ToolInstallationMessageListener { nats_connection_manager: NatsConnectionManager, tool_installation_service: ToolInstallationService, config_service: AgentConfigurationService, + tool_run_manager: ToolRunManager, ) -> Self { Self { nats_connection_manager, tool_installation_service, config_service, + tool_run_manager, } } @@ -124,6 +129,23 @@ impl ToolInstallationMessageListener { let tool_agent_id = tool_installation_message.tool_agent_id.clone(); + let listener = self.clone(); + park_or_dispatch( + self.tool_run_manager.clone(), + message, + format!("tool-installation:{}", tool_agent_id), + move |msg| async move { + listener.dispatch(msg, tool_installation_message).await; + }, + ) + .await; + + Ok(()) + } + + async fn dispatch(&self, message: Message, tool_installation_message: ToolInstallationMessage) { + let tool_agent_id = tool_installation_message.tool_agent_id.clone(); + match self .tool_installation_service .install(tool_installation_message) @@ -134,14 +156,13 @@ impl ToolInstallationMessageListener { "Acknowledging installation message for tool: {}", tool_agent_id ); - message - .ack() - .await - .map_err(|e| anyhow::anyhow!("Failed to ack message: {}", e))?; - info!( - "Installation message acknowledged for tool: {}", - tool_agent_id - ); + match message.ack().await { + Ok(_) => info!( + "Installation message acknowledged for tool: {}", + tool_agent_id + ), + Err(e) => error!("Failed to ack message for tool {}: {}", tool_agent_id, e), + } } Err(e) => { error!( @@ -154,8 +175,6 @@ impl ToolInstallationMessageListener { ); } } - - Ok(()) } async fn create_consumer(&self, js: &jetstream::Context, machine_id: &str) -> PushConsumer { diff --git a/clients/openframe-client/src/listener/tool_restart_message_listener.rs b/clients/openframe-client/src/listener/tool_restart_message_listener.rs index ded3620ad..a0e3c7dcc 100644 --- a/clients/openframe-client/src/listener/tool_restart_message_listener.rs +++ b/clients/openframe-client/src/listener/tool_restart_message_listener.rs @@ -3,10 +3,12 @@ use crate::config::update_config::{ INITIAL_RETRY_DELAY_MS, MAX_RETRY_DELAY_MS, RECONNECTION_DELAY_MS, RESTART_CONSUMER_MAX_DELIVER, }; +use crate::listener::client_update_gate::park_or_dispatch; use crate::models::ToolRestartMessage; use crate::services::nats_connection_manager::NatsConnectionManager; use crate::services::tool_restart_service::RestartOutcome; use crate::services::tool_restart_service::ToolRestartService; +use crate::services::tool_run_manager::ToolRunManager; use crate::services::AgentConfigurationService; use anyhow::Result; use async_nats::jetstream; @@ -22,6 +24,7 @@ pub struct ToolRestartMessageListener { nats_connection_manager: NatsConnectionManager, tool_restart_service: ToolRestartService, config_service: AgentConfigurationService, + tool_run_manager: ToolRunManager, } impl ToolRestartMessageListener { @@ -31,11 +34,13 @@ impl ToolRestartMessageListener { nats_connection_manager: NatsConnectionManager, tool_restart_service: ToolRestartService, config_service: AgentConfigurationService, + tool_run_manager: ToolRunManager, ) -> Self { Self { nats_connection_manager, tool_restart_service, config_service, + tool_run_manager, } } @@ -122,6 +127,22 @@ impl ToolRestartMessageListener { let tool_agent_id = restart_message.tool_agent_id; + let listener = self.clone(); + let label = format!("tool-restart:{}", tool_agent_id); + park_or_dispatch( + self.tool_run_manager.clone(), + message, + label, + move |msg| async move { + listener.dispatch(msg, tool_agent_id).await; + }, + ) + .await; + + Ok(()) + } + + async fn dispatch(&self, message: Message, tool_agent_id: String) { let ack_message = match self .tool_restart_service .restart_guarded(&tool_agent_id) @@ -132,7 +153,7 @@ impl ToolRestartMessageListener { "Tool {} busy with another operation, deferring restart for redelivery", tool_agent_id ); - return Ok(()); + return; } Ok(RestartOutcome::Restarted) | Ok(RestartOutcome::NotInstalled) => true, Err(e) => { @@ -142,19 +163,19 @@ impl ToolRestartMessageListener { }; if ack_message { - message - .ack() - .await - .map_err(|e| anyhow::anyhow!("Failed to ack message: {}", e))?; - info!("Restart message acknowledged for tool: {}", tool_agent_id); + match message.ack().await { + Ok(_) => info!("Restart message acknowledged for tool: {}", tool_agent_id), + Err(e) => error!( + "Failed to ack restart message for tool {}: {}", + tool_agent_id, e + ), + } } else { info!( "Leaving restart message unacked for potential redelivery: tool {}", tool_agent_id ); } - - Ok(()) } async fn create_consumer(&self, js: &jetstream::Context, machine_id: &str) -> PushConsumer { diff --git a/clients/openframe-client/src/listener/tool_uninstall_message_listener.rs b/clients/openframe-client/src/listener/tool_uninstall_message_listener.rs index 225b96557..170a01493 100644 --- a/clients/openframe-client/src/listener/tool_uninstall_message_listener.rs +++ b/clients/openframe-client/src/listener/tool_uninstall_message_listener.rs @@ -3,6 +3,7 @@ use crate::config::update_config::{ INITIAL_RETRY_DELAY_MS, MAX_RETRY_DELAY_MS, RECONNECTION_DELAY_MS, UNINSTALL_CONSUMER_MAX_DELIVER, }; +use crate::listener::client_update_gate::park_or_dispatch; use crate::models::ToolUninstallMessage; use crate::services::nats_connection_manager::NatsConnectionManager; use crate::services::tool_run_manager::ToolRunManager; @@ -126,6 +127,23 @@ impl ToolUninstallMessageListener { let tool_agent_id = uninstall_message.tool_agent_id.clone(); + let listener = self.clone(); + park_or_dispatch( + self.tool_run_manager.clone(), + message, + format!("tool-uninstall:{}", tool_agent_id), + move |msg| async move { + listener.dispatch(msg, uninstall_message).await; + }, + ) + .await; + + Ok(()) + } + + async fn dispatch(&self, message: Message, uninstall_message: ToolUninstallMessage) { + let tool_agent_id = uninstall_message.tool_agent_id.clone(); + let tool_lock = self.tool_run_manager.tool_lock(&tool_agent_id).await; let _guard = match tool_lock.try_lock() { Ok(guard) => guard, @@ -134,7 +152,7 @@ impl ToolUninstallMessageListener { "Tool {} busy with another operation, deferring uninstall for redelivery", tool_agent_id ); - return Ok(()); + return; } }; @@ -168,19 +186,19 @@ impl ToolUninstallMessageListener { self.tool_run_manager.clear_updating(&tool_agent_id).await; if ack_message { - message - .ack() - .await - .map_err(|e| anyhow::anyhow!("Failed to ack message: {}", e))?; - info!("Uninstall message acknowledged for tool: {}", tool_agent_id); + match message.ack().await { + Ok(_) => info!("Uninstall message acknowledged for tool: {}", tool_agent_id), + Err(e) => error!( + "Failed to ack uninstall message for tool {}: {}", + tool_agent_id, e + ), + } } else { info!( "Leaving uninstall message unacked for potential redelivery: tool {}", tool_agent_id ); } - - Ok(()) } async fn create_consumer(&self, js: &jetstream::Context, machine_id: &str) -> PushConsumer { diff --git a/clients/openframe-client/src/models/update_state.rs b/clients/openframe-client/src/models/update_state.rs index 047f79502..7f96f8a70 100644 --- a/clients/openframe-client/src/models/update_state.rs +++ b/clients/openframe-client/src/models/update_state.rs @@ -9,6 +9,8 @@ pub enum UpdatePhase { PreparingUpdater, UpdaterLaunched, Completed, + Verifying, + RolledBack, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -17,6 +19,12 @@ pub struct UpdateState { /// Current phase pub phase: UpdatePhase, + + #[serde(default)] + pub boot_attempts: u32, + + #[serde(default)] + pub started_at: Option, } impl UpdateState { @@ -24,6 +32,8 @@ impl UpdateState { Self { target_version, phase: UpdatePhase::Validating, + boot_attempts: 0, + started_at: Some(chrono::Utc::now().to_rfc3339()), } } diff --git a/clients/openframe-client/src/platform/installation_detector.rs b/clients/openframe-client/src/platform/installation_detector.rs index 65e206970..7733b04d4 100644 --- a/clients/openframe-client/src/platform/installation_detector.rs +++ b/clients/openframe-client/src/platform/installation_detector.rs @@ -44,24 +44,18 @@ fn detect_service( #[cfg(windows)] { - use std::process::Command; + if crate::platform::system_service::service_exists(service_name) { + info!(tool_id = %tool_id, "Detected existing Windows service: {}", service_name); - let output = Command::new("sc").args(["query", service_name]).output(); - - if let Ok(out) = output { - if out.status.success() { - info!(tool_id = %tool_id, "Detected existing Windows service: {}", service_name); - - let exec_path = directory_manager - .get_tool_executable_path(tool_id, Some(&config.target_file_name)) - .to_string_lossy() - .to_string(); + let exec_path = directory_manager + .get_tool_executable_path(tool_id, Some(&config.target_file_name)) + .to_string_lossy() + .to_string(); - return Some(Installation::Service { - service_name: service_name.clone(), - executable_path: Some(exec_path), - }); - } + return Some(Installation::Service { + service_name: service_name.clone(), + executable_path: Some(exec_path), + }); } } diff --git a/clients/openframe-client/src/platform/system_service.rs b/clients/openframe-client/src/platform/system_service.rs index 92f6af5a5..dc641ad79 100644 --- a/clients/openframe-client/src/platform/system_service.rs +++ b/clients/openframe-client/src/platform/system_service.rs @@ -554,6 +554,20 @@ fn service_stopped_or_missing( } } +#[cfg(target_os = "windows")] +pub fn service_exists(service_name: &str) -> bool { + query_service_status_windows(service_name).is_ok() +} + +#[cfg(target_os = "windows")] +pub fn service_not_stopped(service_name: &str) -> bool { + use windows_service::service::ServiceState; + match query_service_status_windows(service_name) { + Ok(status) => status.current_state != ServiceState::Stopped, + Err(_) => false, + } +} + /// True only if SCM reports the service does not exist. #[cfg(target_os = "windows")] fn service_missing_windows(service_name: &str) -> bool { diff --git a/clients/openframe-client/src/platform/update_scripts/macos.rs b/clients/openframe-client/src/platform/update_scripts/macos.rs index 89c5382fb..0a0d3de5a 100644 --- a/clients/openframe-client/src/platform/update_scripts/macos.rs +++ b/clients/openframe-client/src/platform/update_scripts/macos.rs @@ -13,13 +13,18 @@ pub const UPDATER_PLIST_TEMPLATE: &str = r#"{SERVICE_LABEL} {TARGET_EXE} {UPDATE_STATE_PATH} + {TARGET_VERSION} + {BOOT_MARKER_PATH} + {LKG_PATH} + {BOOT_MARKER_WAIT_SECS} + {ROLLBACK_ONLY} RunAtLoad StandardOutPath - /tmp/openframe-update-debug.log + {TRANSCRIPT_PATH} StandardErrorPath - /tmp/openframe-update-debug.log + {TRANSCRIPT_PATH} "#; @@ -29,109 +34,238 @@ BINARY_PATH="$1" SERVICE_LABEL="$2" TARGET_EXE="$3" UPDATE_STATE_PATH="$4" +TARGET_VERSION="$5" +BOOT_MARKER_PATH="$6" +LKG_PATH="$7" +BOOT_MARKER_WAIT_SECS="${8:-90}" +ROLLBACK_ONLY="${9:-0}" -BACKUP_PATH="" +PLIST_PATH="/Library/LaunchDaemons/${SERVICE_LABEL}.plist" +PREV_PATH="${TARGET_EXE}.prev" + +log() { + echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) $*" +} + +set_update_phase() { + PHASE="$1" + if [ -n "$UPDATE_STATE_PATH" ] && [ -f "$UPDATE_STATE_PATH" ]; then + STATE_TMP="${UPDATE_STATE_PATH}.tmp" + if sed 's/"phase"[[:space:]]*:[[:space:]]*"[^"]*"/"phase": "'"$PHASE"'"/' "$UPDATE_STATE_PATH" > "$STATE_TMP" 2>/dev/null \ + && [ -f "$UPDATE_STATE_PATH" ] \ + && mv "$STATE_TMP" "$UPDATE_STATE_PATH" 2>/dev/null; then + log "Update state phase set to '$PHASE'" + else + rm -f "$STATE_TMP" 2>/dev/null + log "Failed to stamp update phase '$PHASE'" + fi + fi +} + +agent_uninstalled() { + [ "$ROLLBACK_ONLY" != "1" ] && [ -n "$UPDATE_STATE_PATH" ] && [ ! -f "$UPDATE_STATE_PATH" ] +} + +restore_reserve() { + if [ -n "$LKG_PATH" ] && [ -f "$LKG_PATH" ]; then + RESTORE_SOURCE="$LKG_PATH" + log "Restoring from last-known-good reserve: $LKG_PATH" + elif [ -f "$PREV_PATH" ]; then + RESTORE_SOURCE="$PREV_PATH" + log "Restoring from pre-swap copy: $PREV_PATH" + else + log "No reserve available for rollback (checked '$LKG_PATH' and '$PREV_PATH')" + return 1 + fi + rm -f "$TARGET_EXE" 2>/dev/null + cp "$RESTORE_SOURCE" "$TARGET_EXE" || return 1 + chmod 755 "$TARGET_EXE" 2>/dev/null + return 0 +} cleanup() { - if [ -f "$BINARY_PATH" ]; then + if [ -n "$BINARY_PATH" ] && [ -f "$BINARY_PATH" ]; then rm -f "$BINARY_PATH" 2>/dev/null fi - # Also cleanup the updater plist + # launchctl remove may terminate this script - keep it last UPDATER_PLIST="/tmp/com.openframe.updater.plist" if [ -f "$UPDATER_PLIST" ]; then - launchctl remove "com.openframe.updater" 2>/dev/null rm -f "$UPDATER_PLIST" 2>/dev/null + launchctl remove "com.openframe.updater" 2>/dev/null fi } -rollback() { - if [ -n "$BACKUP_PATH" ] && [ -f "$BACKUP_PATH" ]; then - cp "$BACKUP_PATH" "$TARGET_EXE" 2>/dev/null - chmod 755 "$TARGET_EXE" 2>/dev/null - launchctl load "/Library/LaunchDaemons/${SERVICE_LABEL}.plist" 2>/dev/null +service_loaded() { + launchctl list "$SERVICE_LABEL" >/dev/null 2>&1 +} + +stop_service() { + if service_loaded; then + launchctl unload "$PLIST_PATH" 2>/dev/null fi + STOP_ELAPSED=0 + while service_loaded && [ $STOP_ELAPSED -lt 30 ]; do + sleep 1 + STOP_ELAPSED=$((STOP_ELAPSED + 1)) + done + if service_loaded; then + return 1 + fi + sleep 2 + return 0 } -# Validate inputs -if [ ! -f "$BINARY_PATH" ]; then +fail_rollback() { + log "Updater failed: $1" + + if agent_uninstalled; then + log "Update state file is gone (agent uninstalled mid-update) - standing down without touching the service" + cleanup + exit 1 + fi + + stop_service + + RESTORED=0 + for RESTORE_ATTEMPT in 1 2 3; do + if restore_reserve; then + RESTORED=1 + break + fi + log "Restore attempt $RESTORE_ATTEMPT failed" + sleep 2 + done + + if [ $RESTORED -eq 1 ]; then + if launchctl load "$PLIST_PATH" 2>/dev/null; then + RB_ELAPSED=0 + while ! service_loaded && [ $RB_ELAPSED -lt 30 ]; do + sleep 1 + RB_ELAPSED=$((RB_ELAPSED + 1)) + done + if service_loaded; then + set_update_phase "rolled_back" + log "Rollback complete, service restarted" + else + log "Service did not come up after rollback" + fi + else + log "Failed to load service after rollback" + fi + else + log "All restore attempts failed" + launchctl load "$PLIST_PATH" 2>/dev/null + fi + + cleanup exit 1 +} + +fail_preswap() { + log "Updater failed before the binary swap: $1" + + if agent_uninstalled; then + log "Update state file is gone (agent uninstalled mid-update) - standing down without touching the service" + cleanup + exit 1 + fi + + if ! service_loaded; then + launchctl load "$PLIST_PATH" 2>/dev/null && log "Service restarted with the untouched binary" + fi + cleanup + exit 1 +} + +log "Updater starting: target version '$TARGET_VERSION', target exe '$TARGET_EXE'" + +if [ "$ROLLBACK_ONLY" = "1" ]; then + if [ ! -f "$PLIST_PATH" ]; then + log "Service plist not found: $PLIST_PATH - nothing to roll back" + cleanup + exit 1 + fi + fail_rollback "Rollback-only mode requested" fi +# Validate inputs +if [ ! -f "$BINARY_PATH" ]; then + fail_preswap "New binary not found: $BINARY_PATH" +fi if [ ! -f "$TARGET_EXE" ]; then - exit 1 + fail_preswap "Target executable not found: $TARGET_EXE" fi BINARY_SIZE=$(stat -f%z "$BINARY_PATH" 2>/dev/null || stat -c%s "$BINARY_PATH" 2>/dev/null) if [ "$BINARY_SIZE" -lt 102400 ]; then - exit 1 + fail_preswap "New binary too small ($BINARY_SIZE bytes), likely corrupted" fi -PLIST_PATH="/Library/LaunchDaemons/${SERVICE_LABEL}.plist" if [ ! -f "$PLIST_PATH" ]; then - exit 1 + fail_preswap "Service plist not found: $PLIST_PATH" fi # Stop the service -if launchctl list "$SERVICE_LABEL" >/dev/null 2>&1; then - launchctl unload "$PLIST_PATH" 2>/dev/null -fi - -# Wait for service to fully stop -TIMEOUT=30 -ELAPSED=0 -while launchctl list "$SERVICE_LABEL" >/dev/null 2>&1 && [ $ELAPSED -lt $TIMEOUT ]; do - sleep 1 - ELAPSED=$((ELAPSED + 1)) -done - -if [ $ELAPSED -ge $TIMEOUT ]; then - launchctl load "$PLIST_PATH" 2>/dev/null - exit 1 +if ! stop_service; then + fail_preswap "Service did not stop within 30 seconds" fi -sleep 2 - -# Create backup -BACKUP_PATH="${TARGET_EXE}.backup.$(date +%Y%m%d%H%M%S)" -if ! cp "$TARGET_EXE" "$BACKUP_PATH"; then - launchctl load "$PLIST_PATH" 2>/dev/null - exit 1 +if ! mv "$TARGET_EXE" "$PREV_PATH"; then + fail_preswap "Failed to move current binary to $PREV_PATH" fi # Replace binary if ! cp "$BINARY_PATH" "$TARGET_EXE"; then - rollback - cleanup - exit 1 + fail_rollback "Failed to copy new binary into place" fi - -# Set executable permissions if ! chmod 755 "$TARGET_EXE"; then - rollback - cleanup - exit 1 + fail_rollback "Failed to set executable permissions" fi -# Mark update as completed -if [ -n "$UPDATE_STATE_PATH" ] && [ -f "$UPDATE_STATE_PATH" ]; then - sed -i '' 's/"phase"[[:space:]]*:[[:space:]]*"[^"]*"/"phase": "completed"/' "$UPDATE_STATE_PATH" 2>/dev/null +if [ -n "$BOOT_MARKER_PATH" ]; then + rm -f "$BOOT_MARKER_PATH" 2>/dev/null fi # Start service if ! launchctl load "$PLIST_PATH"; then - rollback - cleanup - exit 1 + fail_rollback "Failed to load service" fi -# Verify service started sleep 3 -if ! launchctl list "$SERVICE_LABEL" >/dev/null 2>&1; then - rollback - cleanup - exit 1 +if ! service_loaded; then + fail_rollback "Service failed to start" +fi + +MARKER_OK=0 +if [ -n "$BOOT_MARKER_PATH" ] && [ -n "$TARGET_VERSION" ]; then + ELAPSED=0 + while [ $ELAPSED -lt "$BOOT_MARKER_WAIT_SECS" ]; do + if [ -f "$BOOT_MARKER_PATH" ]; then + MARKER_VERSION=$(cat "$BOOT_MARKER_PATH" 2>/dev/null | tr -d '[:space:]') + if [ "$MARKER_VERSION" = "$TARGET_VERSION" ]; then + MARKER_OK=1 + break + fi + if [ -n "$MARKER_VERSION" ]; then + log "Boot marker reports '$MARKER_VERSION', expected '$TARGET_VERSION' - wrong binary booted" + break + fi + fi + sleep 2 + ELAPSED=$((ELAPSED + 2)) + done +else + log "No boot marker path/target version provided, skipping boot check" + MARKER_OK=1 +fi + +if [ $MARKER_OK -ne 1 ]; then + fail_rollback "New binary did not report target version '$TARGET_VERSION' within $BOOT_MARKER_WAIT_SECS seconds" fi +log "Boot marker matched target version '$TARGET_VERSION'" + +set_update_phase "verifying" # Cleanup (removes temp binary and updater plist) cleanup diff --git a/clients/openframe-client/src/platform/update_scripts/windows.rs b/clients/openframe-client/src/platform/update_scripts/windows.rs index d2c46d90a..0cd00aea7 100644 --- a/clients/openframe-client/src/platform/update_scripts/windows.rs +++ b/clients/openframe-client/src/platform/update_scripts/windows.rs @@ -5,15 +5,73 @@ param( [string]$ArchivePath, [string]$ServiceName, [string]$TargetExe, - [string]$UpdateStatePath + [string]$UpdateStatePath, + [string]$TargetVersion, + [string]$BootMarkerPath, + [string]$LkgPath, + [string]$TranscriptPath, + [int]$BootMarkerWaitSecs = 90, + [switch]$RollbackOnly ) $ErrorActionPreference = 'Stop' -$BackupPath = $null +if ($TranscriptPath) { + try { Start-Transcript -Path $TranscriptPath -Force | Out-Null } catch { } +} + +$PrevPath = "$TargetExe.prev" $TempExtract = $null +$SwapReached = $false + +function Set-UpdatePhase { + param([string]$Phase) + if ($UpdateStatePath -and (Test-Path $UpdateStatePath)) { + try { + $stateContent = Get-Content -Path $UpdateStatePath -Raw | ConvertFrom-Json + $stateContent.phase = $Phase + $stateTmp = "$UpdateStatePath.tmp" + $stateContent | ConvertTo-Json -Depth 10 | Set-Content -Path $stateTmp -Force + Move-Item -Path $stateTmp -Destination $UpdateStatePath -Force + Write-Output "Update state phase set to '$Phase'" + } + catch { + Write-Output "Failed to stamp update phase '$Phase': $_" + } + } +} + +function Restore-Reserve { + if ($LkgPath -and (Test-Path $LkgPath)) { + $restoreSource = $LkgPath + Write-Output "Restoring from last-known-good reserve: $LkgPath" + } + elseif (Test-Path $PrevPath) { + $restoreSource = $PrevPath + Write-Output "Restoring from pre-swap copy: $PrevPath" + } + else { + throw "No reserve available for rollback (checked '$LkgPath' and '$PrevPath')" + } + if (Test-Path $TargetExe) { + try { Move-Item -Path $TargetExe -Destination "$TargetExe.bad" -Force -ErrorAction Stop } catch { } + } + Copy-Item -Path $restoreSource -Destination $TargetExe -Force -ErrorAction Stop + Remove-Item -Path "$TargetExe.bad" -Force -ErrorAction SilentlyContinue +} + +function Test-AgentUninstalled { + return (-not $RollbackOnly) -and $UpdateStatePath -and (-not (Test-Path $UpdateStatePath)) +} try { + Write-Output "Updater starting: target version '$TargetVersion', target exe '$TargetExe'" + + if ($RollbackOnly) { + $SwapReached = $true + throw "Rollback-only mode requested" + } + # Validate inputs if (-not (Test-Path $ArchivePath)) { throw "Archive file not found: $ArchivePath" @@ -51,10 +109,6 @@ try { Start-Sleep -Seconds 2 - # Create backup - $BackupPath = "$TargetExe.backup.$(Get-Date -Format 'yyyyMMddHHmmss')" - Copy-Item -Path $TargetExe -Destination $BackupPath -Force -ErrorAction Stop - # Extract archive $TempExtract = Join-Path $env:TEMP "openframe-update-$(New-Guid)" Expand-Archive -Path $ArchivePath -DestinationPath $TempExtract -Force -ErrorAction Stop @@ -71,18 +125,13 @@ try { } # Replace binary + Move-Item -Path $TargetExe -Destination $PrevPath -Force -ErrorAction Stop + $SwapReached = $true + Copy-Item -Path $NewExe.FullName -Destination $TargetExe -Force -ErrorAction Stop - # Mark update as completed - if ($UpdateStatePath -and (Test-Path $UpdateStatePath)) { - try { - $stateContent = Get-Content -Path $UpdateStatePath -Raw | ConvertFrom-Json - $stateContent.phase = "completed" - $stateContent | ConvertTo-Json -Depth 10 | Set-Content -Path $UpdateStatePath -Force - } - catch { - # Ignore state update errors - } + if ($BootMarkerPath -and (Test-Path $BootMarkerPath)) { + Remove-Item -Path $BootMarkerPath -Force -ErrorAction Stop } # Start service @@ -96,6 +145,39 @@ try { throw "Service failed to start" } + $markerOk = $false + if ($BootMarkerPath -and $TargetVersion) { + $elapsed = 0 + while ($elapsed -lt $BootMarkerWaitSecs) { + if (Test-Path $BootMarkerPath) { + $markerVersion = Get-Content -Path $BootMarkerPath -Raw -ErrorAction SilentlyContinue + if ($markerVersion) { $markerVersion = $markerVersion.Trim() } + if ($markerVersion -eq $TargetVersion) { + $markerOk = $true + break + } + if ($markerVersion) { + Write-Output "Boot marker reports '$markerVersion', expected '$TargetVersion' — wrong binary booted" + break + } + } + Start-Sleep -Seconds 2 + $elapsed += 2 + } + } + else { + Write-Output "No boot marker path/target version provided, skipping boot check" + $markerOk = $true + } + + if (-not $markerOk) { + throw "New binary did not report target version '$TargetVersion' within $BootMarkerWaitSecs seconds" + } + + Write-Output "Boot marker matched target version '$TargetVersion'" + + Set-UpdatePhase -Phase "verifying" + # Cleanup Remove-Item -Path $ArchivePath -Force -ErrorAction SilentlyContinue Remove-Item -Path $TempExtract -Recurse -Force -ErrorAction SilentlyContinue @@ -103,14 +185,85 @@ try { exit 0 } catch { - # Attempt rollback if backup exists - if ($BackupPath -and (Test-Path $BackupPath)) { + Write-Output "Updater failed: $_" + + if (Test-AgentUninstalled) { + Write-Output "Update state file is gone (agent uninstalled mid-update) — standing down without touching the service" + if ($TempExtract -and (Test-Path $TempExtract)) { + Remove-Item -Path $TempExtract -Recurse -Force -ErrorAction SilentlyContinue + } + if ($ArchivePath -and (Test-Path $ArchivePath)) { + Remove-Item -Path $ArchivePath -Force -ErrorAction SilentlyContinue + } + exit 1 + } + + if ($SwapReached) { try { - Copy-Item -Path $BackupPath -Destination $TargetExe -Force -ErrorAction Stop - Start-Service -Name $ServiceName -ErrorAction SilentlyContinue + $service = Get-Service -Name $ServiceName -ErrorAction SilentlyContinue + if ($service -and $service.Status -ne 'Stopped') { + Stop-Service -Name $ServiceName -Force -ErrorAction SilentlyContinue + $rbStop = 0 + while ((Get-Service -Name $ServiceName).Status -ne 'Stopped' -and $rbStop -lt 30) { + Start-Sleep -Seconds 1 + $rbStop++ + } + Start-Sleep -Seconds 2 + } + + $restored = $false + for ($restoreAttempt = 1; $restoreAttempt -le 3; $restoreAttempt++) { + try { + Restore-Reserve + $restored = $true + break + } + catch { + Write-Output "Restore attempt $restoreAttempt failed: $_" + Start-Sleep -Seconds 2 + } + } + if (-not $restored) { + throw "All restore attempts failed" + } + + Start-Service -Name $ServiceName -ErrorAction Stop + $rbElapsed = 0 + while ((Get-Service -Name $ServiceName).Status -ne 'Running' -and $rbElapsed -lt 30) { + Start-Sleep -Seconds 1 + $rbElapsed++ + } + if ((Get-Service -Name $ServiceName).Status -ne 'Running') { + throw "Service did not reach Running state after rollback" + } + + Set-UpdatePhase -Phase "rolled_back" + Write-Output "Rollback complete, service restarted" } catch { - # Rollback failed + Write-Output "Rollback failed: $_" + try { + if ((Get-Service -Name $ServiceName -ErrorAction Stop).Status -ne 'Running') { + Start-Service -Name $ServiceName -ErrorAction Stop + Write-Output "Service restarted with the binary currently in place" + } + } + catch { + Write-Output "Failed to restart service after failed rollback: $_" + } + } + } + else { + Write-Output "Failure happened before the binary swap, no rollback needed" + try { + $service = Get-Service -Name $ServiceName -ErrorAction SilentlyContinue + if ($service -and $service.Status -ne 'Running') { + Start-Service -Name $ServiceName -ErrorAction Stop + Write-Output "Service restarted with the untouched binary" + } + } + catch { + Write-Output "Failed to restart service after pre-swap failure: $_" } } @@ -118,7 +271,15 @@ catch { if ($TempExtract -and (Test-Path $TempExtract)) { Remove-Item -Path $TempExtract -Recurse -Force -ErrorAction SilentlyContinue } + if ($ArchivePath -and (Test-Path $ArchivePath)) { + Remove-Item -Path $ArchivePath -Force -ErrorAction SilentlyContinue + } exit 1 } +finally { + if ($TranscriptPath) { + try { Stop-Transcript | Out-Null } catch { } + } +} "#; diff --git a/clients/openframe-client/src/platform/updater_launcher/macos.rs b/clients/openframe-client/src/platform/updater_launcher/macos.rs index 370d092a5..be99f24ad 100644 --- a/clients/openframe-client/src/platform/updater_launcher/macos.rs +++ b/clients/openframe-client/src/platform/updater_launcher/macos.rs @@ -5,6 +5,7 @@ use tracing::info; use uuid::Uuid; use super::UpdaterParams; +use crate::config::update_config::BOOT_MARKER_WAIT_SECS; use crate::platform::update_scripts::{UPDATER_PLIST_TEMPLATE, UPDATE_SCRIPT_MACOS}; /// Launch bash updater script on macOS @@ -49,7 +50,25 @@ pub async fn launch_updater(params: UpdaterParams) -> Result<()> { .replace("{BINARY_PATH}", ¶ms.binary_path.to_string_lossy()) .replace("{SERVICE_LABEL}", ¶ms.service_name) .replace("{TARGET_EXE}", ¶ms.target_exe.to_string_lossy()) - .replace("{UPDATE_STATE_PATH}", ¶ms.update_state_path); + .replace("{UPDATE_STATE_PATH}", ¶ms.update_state_path) + .replace("{TARGET_VERSION}", ¶ms.target_version) + .replace( + "{BOOT_MARKER_PATH}", + ¶ms.boot_marker_path.to_string_lossy(), + ) + .replace("{LKG_PATH}", ¶ms.lkg_path.to_string_lossy()) + .replace( + "{BOOT_MARKER_WAIT_SECS}", + &BOOT_MARKER_WAIT_SECS.to_string(), + ) + .replace( + "{ROLLBACK_ONLY}", + if params.rollback_only { "1" } else { "0" }, + ) + .replace( + "{TRANSCRIPT_PATH}", + ¶ms.transcript_path.to_string_lossy(), + ); std::fs::write(&plist_path, &plist_content).context("Failed to write updater plist")?; diff --git a/clients/openframe-client/src/platform/updater_launcher/mod.rs b/clients/openframe-client/src/platform/updater_launcher/mod.rs index 5ebd03f4d..69992242c 100644 --- a/clients/openframe-client/src/platform/updater_launcher/mod.rs +++ b/clients/openframe-client/src/platform/updater_launcher/mod.rs @@ -6,6 +6,11 @@ pub struct UpdaterParams { pub target_exe: PathBuf, pub service_name: String, pub update_state_path: String, + pub target_version: String, + pub boot_marker_path: PathBuf, + pub lkg_path: PathBuf, + pub transcript_path: PathBuf, + pub rollback_only: bool, } #[cfg(windows)] diff --git a/clients/openframe-client/src/platform/updater_launcher/windows.rs b/clients/openframe-client/src/platform/updater_launcher/windows.rs index e60a181b1..fba8120c7 100644 --- a/clients/openframe-client/src/platform/updater_launcher/windows.rs +++ b/clients/openframe-client/src/platform/updater_launcher/windows.rs @@ -5,6 +5,7 @@ use tracing::info; use uuid::Uuid; use super::UpdaterParams; +use crate::config::update_config::BOOT_MARKER_WAIT_SECS; use crate::platform::get_powershell_path; use crate::platform::update_scripts::UPDATE_SCRIPT_WINDOWS; @@ -26,7 +27,8 @@ pub async fn launch_updater(params: UpdaterParams) -> Result<()> { let ps_path = get_powershell_path().map_err(|e| anyhow!(e))?; info!("Using PowerShell: {}", ps_path); - let child = Command::new(&ps_path) + let mut command = Command::new(&ps_path); + command .arg("-ExecutionPolicy") .arg("Bypass") .arg("-NoProfile") @@ -40,7 +42,21 @@ pub async fn launch_updater(params: UpdaterParams) -> Result<()> { .arg(¶ms.target_exe) .arg("-UpdateStatePath") .arg(¶ms.update_state_path) - .creation_flags(0x08000000) // CREATE_NO_WINDOW + .arg("-TargetVersion") + .arg(¶ms.target_version) + .arg("-BootMarkerPath") + .arg(¶ms.boot_marker_path) + .arg("-LkgPath") + .arg(¶ms.lkg_path) + .arg("-TranscriptPath") + .arg(¶ms.transcript_path) + .arg("-BootMarkerWaitSecs") + .arg(BOOT_MARKER_WAIT_SECS.to_string()) + .creation_flags(0x08000000); // CREATE_NO_WINDOW + if params.rollback_only { + command.arg("-RollbackOnly"); + } + let child = command .spawn() .context("Failed to spawn PowerShell updater")?; diff --git a/clients/openframe-client/src/service.rs b/clients/openframe-client/src/service.rs index 1418e968f..26271975c 100644 --- a/clients/openframe-client/src/service.rs +++ b/clients/openframe-client/src/service.rs @@ -156,17 +156,7 @@ impl Service { pub fn is_installed() -> bool { #[cfg(target_os = "windows")] { - use std::process::Command; - - // Check if Windows service exists using sc query - let output = Command::new("sc") - .args(["query", FULL_SERVICE_NAME]) - .output(); - - match output { - Ok(output) => output.status.success(), - Err(_) => false, - } + crate::platform::system_service::service_exists(FULL_SERVICE_NAME) } #[cfg(target_os = "macos")] @@ -276,8 +266,46 @@ impl Service { } // Copy the binary - std::fs::copy(¤t_exe_path, &install_path) - .with_context(|| format!("Failed to copy binary to {}", install_path.display()))?; + if let Err(copy_err) = std::fs::copy(¤t_exe_path, &install_path) { + warn!( + "Target binary is in use ({}); killing leftover processes running from {}", + copy_err, + install_path.display() + ); + Self::kill_processes_running_from(&install_path).await; + + if let Err(retry_err) = std::fs::copy(¤t_exe_path, &install_path) { + let aside_path = install_path.with_extension("exe.old"); + let _ = std::fs::remove_file(&aside_path); + std::fs::rename(&install_path, &aside_path) + .with_context(|| format!( + "Failed to copy binary to {} ({}), and could not move the existing binary aside", + install_path.display(), retry_err + ))?; + warn!( + "Target binary still in use ({}); moved it aside to {} and retrying copy", + retry_err, + aside_path.display() + ); + match std::fs::copy(¤t_exe_path, &install_path) { + Ok(_) => { + let _ = std::fs::remove_file(&aside_path); + } + Err(final_err) => { + if let Err(restore_err) = std::fs::rename(&aside_path, &install_path) { + warn!( + "Could not restore original binary from {}: {}", + aside_path.display(), + restore_err + ); + } + return Err(final_err).with_context(|| { + format!("Failed to copy binary to {}", install_path.display()) + }); + } + } + } + } // Set executable permissions on Unix #[cfg(unix)] @@ -362,6 +390,57 @@ impl Service { Ok(()) } + async fn kill_processes_running_from(target: &std::path::Path) { + use sysinfo::{ProcessRefreshKind, Signal, System, UpdateKind}; + + let target = target.to_string_lossy().to_lowercase(); + let own_pid = std::process::id(); + + let killed = tokio::task::spawn_blocking(move || { + let mut sys = System::new(); + sys.refresh_processes_specifics(ProcessRefreshKind::new().with_exe(UpdateKind::Always)); + + let mut killed = 0usize; + for (pid, process) in sys.processes() { + if pid.as_u32() == own_pid { + continue; + } + let exe = process + .exe() + .map(|p| p.to_string_lossy().to_lowercase()) + .unwrap_or_default(); + if exe != target { + continue; + } + warn!( + "Killing leftover process {} running from target binary", + pid + ); + if process + .kill_with(Signal::Kill) + .unwrap_or_else(|| process.kill()) + { + killed += 1; + } else { + warn!("Failed to kill leftover process {}", pid); + } + } + killed + }) + .await + .unwrap_or(0); + + if killed > 0 { + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + info!( + "Killed {} leftover process(es) holding the target binary", + killed + ); + } else { + info!("No leftover processes found holding the target binary"); + } + } + /// Uninstall the service on the current platform pub async fn uninstall() -> Result<()> { // Check if we have admin privileges diff --git a/clients/openframe-client/src/service_adapter.rs b/clients/openframe-client/src/service_adapter.rs index 9e6e1ebb9..7726dab8e 100644 --- a/clients/openframe-client/src/service_adapter.rs +++ b/clients/openframe-client/src/service_adapter.rs @@ -622,22 +622,7 @@ impl CrossPlatformServiceManager { /// Check if a Windows service process is still running #[cfg(target_os = "windows")] fn is_service_process_running(service_name: &str) -> bool { - use std::process::Command; - - // Use sc query to check service status - let output = Command::new("sc").args(["query", service_name]).output(); - - match output { - Ok(output) => { - let stdout = String::from_utf8_lossy(&output.stdout); - // If service is STOPPED or doesn't exist, it's not running - !stdout.contains("STOPPED") && output.status.success() - } - Err(_) => { - // If we can't check, assume it's not running - false - } - } + crate::platform::system_service::service_not_stopped(service_name) } } diff --git a/clients/openframe-client/src/services/last_known_good_service.rs b/clients/openframe-client/src/services/last_known_good_service.rs new file mode 100644 index 000000000..df4ca629a --- /dev/null +++ b/clients/openframe-client/src/services/last_known_good_service.rs @@ -0,0 +1,222 @@ +use crate::platform::directories::DirectoryManager; +use anyhow::{Context, Result}; +use serde::{Deserialize, Serialize}; +use std::fs; +use std::path::{Path, PathBuf}; +use tracing::{debug, info, warn}; + +#[derive(Debug, Serialize, Deserialize)] +struct LastKnownGood { + version: String, +} + +#[derive(Clone)] +pub struct LastKnownGoodService { + anchor_file_path: PathBuf, + boot_marker_path: PathBuf, + logs_dir: PathBuf, + current_exe: PathBuf, + reserve_path: PathBuf, +} + +impl LastKnownGoodService { + pub fn new(directory_manager: DirectoryManager) -> Result { + let anchor_file_path = directory_manager.secured_dir().join("last_known_good.json"); + let boot_marker_path = directory_manager.secured_dir().join("boot.marker"); + let logs_dir = directory_manager.logs_dir().to_path_buf(); + + directory_manager + .ensure_directories() + .with_context(|| "Failed to ensure secured directory exists")?; + + let current_exe = + std::env::current_exe().context("Failed to get current executable path")?; + + let mut reserve = current_exe.clone().into_os_string(); + reserve.push(".lkg"); + let reserve_path = PathBuf::from(reserve); + + Ok(Self { + anchor_file_path, + boot_marker_path, + logs_dir, + current_exe, + reserve_path, + }) + } + + pub async fn load(&self) -> Result> { + if !self.anchor_file_path.exists() { + debug!( + "No last-known-good file found at: {}", + self.anchor_file_path.display() + ); + return Ok(None); + } + + let json_content = fs::read_to_string(&self.anchor_file_path).with_context(|| { + format!( + "Failed to read last-known-good file: {:?}", + self.anchor_file_path + ) + })?; + + let anchor: LastKnownGood = serde_json::from_str(&json_content) + .context("Failed to deserialize last-known-good from JSON")?; + + Ok(Some(anchor.version)) + } + + fn copy_running_to_reserve(&self) -> Result<()> { + let temp_reserve = self.reserve_path.with_extension("lkg.tmp"); + fs::copy(&self.current_exe, &temp_reserve).with_context(|| { + format!( + "Failed to copy running binary {} to temp reserve {}", + self.current_exe.display(), + temp_reserve.display() + ) + })?; + fs::rename(&temp_reserve, &self.reserve_path).with_context(|| { + format!( + "Failed to move temp reserve into place: {}", + self.reserve_path.display() + ) + })?; + Ok(()) + } + + pub async fn promote(&self, version: &str) -> Result<()> { + self.copy_running_to_reserve()?; + + let json_content = serde_json::to_string_pretty(&LastKnownGood { + version: version.to_string(), + }) + .context("Failed to serialize last-known-good to JSON")?; + let temp_anchor = self.anchor_file_path.with_extension("json.tmp"); + fs::write(&temp_anchor, json_content).with_context(|| { + format!( + "Failed to write temp last-known-good file: {:?}", + temp_anchor + ) + })?; + fs::rename(&temp_anchor, &self.anchor_file_path).with_context(|| { + format!( + "Failed to move last-known-good file into place: {:?}", + self.anchor_file_path + ) + })?; + + info!( + "Last-known-good anchor set to {} (reserve: {})", + version, + self.reserve_path.display() + ); + Ok(()) + } + + pub async fn seed_if_missing(&self) -> Result<()> { + let running_version = env!("OPENFRAME_VERSION"); + let anchor = self.load().await.unwrap_or(None); + + match anchor { + Some(ref anchor_version) if self.reserve_path.exists() => { + debug!( + "Last-known-good anchor {} and reserve present, nothing to seed", + anchor_version + ); + Ok(()) + } + Some(ref anchor_version) if anchor_version == running_version => { + info!( + "Reserve missing; rebuilding it from running binary (matches anchor {})", + anchor_version + ); + self.promote(running_version).await + } + Some(anchor_version) => { + warn!( + "Rollback protection degraded: reserve missing, running {} below anchor {} — rebuilding reserve from running binary, anchor unchanged", + running_version, anchor_version + ); + self.copy_running_to_reserve() + } + None => { + info!( + "Seeding last-known-good anchor from running binary version {}", + running_version + ); + self.promote(running_version).await + } + } + } + + pub fn reserve_path(&self) -> &Path { + &self.reserve_path + } + + pub fn boot_marker_path(&self) -> &Path { + &self.boot_marker_path + } + + pub async fn write_boot_marker(&self) -> Result<()> { + let temp_path = self.boot_marker_path.with_extension("marker.tmp"); + fs::write(&temp_path, env!("OPENFRAME_VERSION")) + .with_context(|| format!("Failed to write temp boot marker: {:?}", temp_path))?; + fs::rename(&temp_path, &self.boot_marker_path).with_context(|| { + format!( + "Failed to move boot marker into place: {:?}", + self.boot_marker_path + ) + })?; + debug!("Boot marker written: {}", self.boot_marker_path.display()); + Ok(()) + } + + pub fn new_transcript_path(&self, target_version: &str) -> PathBuf { + self.logs_dir.join(format!( + "updater-{}-{}.log", + target_version, + chrono::Utc::now().format("%Y%m%d%H%M%S") + )) + } + + pub async fn prune_transcripts(&self, keep: usize) { + let entries = match fs::read_dir(&self.logs_dir) { + Ok(entries) => entries, + Err(e) => { + warn!("Failed to read logs dir for transcript pruning: {}", e); + return; + } + }; + + let mut transcripts: Vec<(std::time::SystemTime, PathBuf)> = entries + .filter_map(|e| e.ok()) + .filter(|e| { + let name = e.file_name().to_string_lossy().to_string(); + name.starts_with("updater-") && name.ends_with(".log") + }) + .filter_map(|e| { + e.metadata() + .ok() + .and_then(|m| m.modified().ok()) + .map(|modified| (modified, e.path())) + }) + .collect(); + + if transcripts.len() <= keep { + return; + } + + transcripts.sort_by(|a, b| b.0.cmp(&a.0)); // newest first + for (_, path) in transcripts.into_iter().skip(keep) { + match fs::remove_file(&path) { + Ok(_) => info!("Removed old updater transcript: {}", path.display()), + Err(e) => warn!( + "Failed to remove old updater transcript {}: {}", + path.display(), + e + ), + } + } + } +} diff --git a/clients/openframe-client/src/services/mod.rs b/clients/openframe-client/src/services/mod.rs index 4a2f52c14..b43057fd6 100644 --- a/clients/openframe-client/src/services/mod.rs +++ b/clients/openframe-client/src/services/mod.rs @@ -10,6 +10,7 @@ pub mod initial_configuration_service; pub mod initial_key_service; pub mod installed_agent_message_publisher; pub mod installed_tools_service; +pub mod last_known_good_service; pub mod local_tls_config_provider; pub mod machine_heartbeat_publisher; pub mod machine_heartbeat_run_manager; @@ -45,6 +46,7 @@ pub use initial_configuration_service::InitialConfigurationService; pub use initial_key_service::InitialKeyService; pub use installed_agent_message_publisher::InstalledAgentMessagePublisher; pub use installed_tools_service::InstalledToolsService; +pub use last_known_good_service::LastKnownGoodService; pub use local_tls_config_provider::LocalTlsConfigProvider; pub use machine_heartbeat_publisher::MachineHeartbeatPublisher; pub use machine_heartbeat_run_manager::MachineHeartbeatRunManager; diff --git a/clients/openframe-client/src/services/openframe_client_info_service.rs b/clients/openframe-client/src/services/openframe_client_info_service.rs index edd32a9bd..93821ad5b 100644 --- a/clients/openframe-client/src/services/openframe_client_info_service.rs +++ b/clients/openframe-client/src/services/openframe_client_info_service.rs @@ -65,6 +65,20 @@ impl OpenFrameClientInfoService { Ok(()) } + pub async fn reconcile_version(&self, running_version: &str) -> Result<()> { + let mut info = self.get().await?; + if info.current_version != running_version { + info!( + "Reconciling current_version '{}' -> running binary version '{}'", + info.current_version, running_version + ); + info.current_version = running_version.to_string(); + info.last_updated = Some(chrono::Utc::now().to_rfc3339()); + self.save(&info).await?; + } + Ok(()) + } + pub async fn set_update_status( &self, status: crate::models::openframe_client_info::ClientUpdateStatus, diff --git a/clients/openframe-client/src/services/openframe_client_update_service.rs b/clients/openframe-client/src/services/openframe_client_update_service.rs index 740bd3fad..1bb4ad66f 100644 --- a/clients/openframe-client/src/services/openframe_client_update_service.rs +++ b/clients/openframe-client/src/services/openframe_client_update_service.rs @@ -1,9 +1,11 @@ +use crate::config::update_config::ALLOW_DOWNGRADE; use crate::models::openframe_client_info::ClientUpdateStatus; use crate::models::openframe_client_update_message::OpenFrameClientUpdateMessage; use crate::models::update_state::{UpdatePhase, UpdateState}; use crate::platform::updater_launcher::{self, UpdaterParams}; use crate::service::FULL_SERVICE_NAME; use crate::services::github_download_service::GithubDownloadService; +use crate::services::last_known_good_service::LastKnownGoodService; use crate::services::openframe_client_info_service::OpenFrameClientInfoService; use crate::services::tool_run_manager::ToolRunManager; use crate::services::update_state_service::UpdateStateService; @@ -20,6 +22,7 @@ pub struct OpenFrameClientUpdateService { client_info_service: OpenFrameClientInfoService, github_download_service: GithubDownloadService, update_state_service: UpdateStateService, + last_known_good_service: LastKnownGoodService, tool_run_manager: ToolRunManager, /// Mutex to prevent concurrent updates (race condition protection) update_in_progress: Arc>, @@ -30,12 +33,14 @@ impl OpenFrameClientUpdateService { client_info_service: OpenFrameClientInfoService, github_download_service: GithubDownloadService, update_state_service: UpdateStateService, + last_known_good_service: LastKnownGoodService, tool_run_manager: ToolRunManager, ) -> Self { Self { client_info_service, github_download_service, update_state_service, + last_known_good_service, tool_run_manager, update_in_progress: Arc::new(Mutex::new(false)), } @@ -45,6 +50,8 @@ impl OpenFrameClientUpdateService { let requested_version = message.version.trim(); info!("Received update request for version: {}", requested_version); + self.tool_run_manager.mark_client_update_pending().await; + if self.tool_run_manager.any_tool_op_in_progress().await { warn!("Tool operation in progress, deferring client update to version {} (will redeliver)", requested_version); return Err(anyhow!( @@ -90,9 +97,48 @@ impl OpenFrameClientUpdateService { return Err(anyhow!("Invalid version format: {}", requested_version)); } - // 3. Parse requested version with semver to ensure valid format - Self::parse_version(requested_version) + let requested_semver = Self::parse_version(requested_version) .with_context(|| format!("Failed to parse requested version: {}", requested_version))?; + let canonical_version = requested_semver.to_string(); + + if canonical_version == env!("OPENFRAME_VERSION") { + info!( + "Already running version {}, ignoring update request", + canonical_version + ); + self.tool_run_manager.clear_client_update_pending().await; + return Ok(()); + } + + if !ALLOW_DOWNGRADE { + let anchor = match self.last_known_good_service.load().await { + Ok(anchor) => anchor, + Err(e) => { + warn!( + "Failed to load last-known-good anchor, skipping downgrade guard: {:#}", + e + ); + None + } + }; + if let Some(anchor) = anchor { + match Self::parse_version(&anchor) { + Ok(anchor_semver) if requested_semver < anchor_semver => { + warn!( + "refusing downgrade to {} — anchored at {}", + requested_version, anchor + ); + self.tool_run_manager.clear_client_update_pending().await; + return Ok(()); + } + Ok(_) => {} + Err(e) => warn!( + "Failed to parse last-known-good anchor '{}': {:#}", + anchor, e + ), + } + } + } // 4. Log current version for informational purposes let client_info = self @@ -114,7 +160,7 @@ impl OpenFrameClientUpdateService { } // 5. Create update state for tracking - let mut update_state = UpdateState::new(requested_version.to_string()); + let mut update_state = UpdateState::new(canonical_version.clone()); self.update_state_service .save(&update_state) .await @@ -124,7 +170,7 @@ impl OpenFrameClientUpdateService { self.client_info_service .set_update_status( ClientUpdateStatus::Updating, - Some(requested_version.to_string()), + Some(canonical_version.clone()), ) .await .context("Failed to set update status")?; @@ -141,10 +187,7 @@ impl OpenFrameClientUpdateService { // Set status to Failed if let Err(status_err) = self .client_info_service - .set_update_status( - ClientUpdateStatus::Failed, - Some(requested_version.to_string()), - ) + .set_update_status(ClientUpdateStatus::Failed, Some(canonical_version.clone())) .await { error!("Failed to set update status to Failed: {:#}", status_err); @@ -196,6 +239,8 @@ impl OpenFrameClientUpdateService { binary_bytes.len() ); + self.tool_run_manager.mark_client_update_pending().await; + // 3. Extract binary update_state.set_phase(UpdatePhase::Extracting); self.update_state_service.save(update_state).await?; @@ -232,6 +277,16 @@ impl OpenFrameClientUpdateService { target_exe: current_exe, service_name: FULL_SERVICE_NAME.to_string(), update_state_path: self.update_state_service.get_state_file_path(), + target_version: update_state.target_version.clone(), + boot_marker_path: self + .last_known_good_service + .boot_marker_path() + .to_path_buf(), + lkg_path: self.last_known_good_service.reserve_path().to_path_buf(), + transcript_path: self + .last_known_good_service + .new_transcript_path(&update_state.target_version), + rollback_only: false, }; if self.tool_run_manager.any_tool_op_in_progress().await { diff --git a/clients/openframe-client/src/services/tool_run_manager.rs b/clients/openframe-client/src/services/tool_run_manager.rs index c5289acf7..4c0e70ed7 100644 --- a/clients/openframe-client/src/services/tool_run_manager.rs +++ b/clients/openframe-client/src/services/tool_run_manager.rs @@ -391,6 +391,25 @@ pub(crate) fn launch_process_in_target_session( } } +#[derive(Clone, Default)] +pub(crate) struct ClientUpdatePendingFlag { + since: Arc>>, +} + +impl ClientUpdatePendingFlag { + pub(crate) async fn mark(&self) { + *self.since.write().await = Some(std::time::Instant::now()); + } + + pub(crate) async fn is_pending(&self, ttl: Duration) -> bool { + matches!(*self.since.read().await, Some(since) if since.elapsed() < ttl) + } + + pub(crate) async fn clear(&self) { + *self.since.write().await = None; + } +} + #[derive(Clone)] pub struct ToolRunManager { installed_tools_service: InstalledToolsService, @@ -400,6 +419,7 @@ pub struct ToolRunManager { updating_tools: Arc>>, tool_locks: Arc>>>>, shutting_down: Arc, + client_update_pending: ClientUpdatePendingFlag, } impl ToolRunManager { @@ -416,6 +436,7 @@ impl ToolRunManager { updating_tools: Arc::new(RwLock::new(HashMap::new())), tool_locks: Arc::new(RwLock::new(HashMap::new())), shutting_down: Arc::new(AtomicBool::new(false)), + client_update_pending: ClientUpdatePendingFlag::default(), } } @@ -433,6 +454,24 @@ impl ToolRunManager { info!("Tool run manager: shutdown signalled, no new launches will occur"); } + pub async fn mark_client_update_pending(&self) { + self.client_update_pending.mark().await; + info!("Client update pending: new tool operations will be parked"); + } + + pub async fn is_client_update_pending(&self) -> bool { + self.client_update_pending + .is_pending(Duration::from_secs( + crate::config::update_config::CLIENT_UPDATE_PENDING_TTL_SECS, + )) + .await + } + + pub async fn clear_client_update_pending(&self) { + self.client_update_pending.clear().await; + info!("Client update no longer pending: parked tool operations released"); + } + pub async fn mark_updating(&self, tool_id: &str) { let mut map = self.updating_tools.write().await; let count = map.entry(tool_id.to_string()).or_insert(0); @@ -853,3 +892,58 @@ impl ToolRunManager { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::ClientUpdatePendingFlag; + use std::time::Duration; + + const LONG_TTL: Duration = Duration::from_secs(3600); + + #[tokio::test] + async fn not_pending_before_first_mark() { + let flag = ClientUpdatePendingFlag::default(); + assert!(!flag.is_pending(LONG_TTL).await); + } + + #[tokio::test] + async fn pending_after_mark_within_ttl() { + let flag = ClientUpdatePendingFlag::default(); + flag.mark().await; + assert!(flag.is_pending(LONG_TTL).await); + } + + #[tokio::test] + async fn expired_when_ttl_elapsed() { + let flag = ClientUpdatePendingFlag::default(); + flag.mark().await; + assert!(!flag.is_pending(Duration::ZERO).await); + } + + #[tokio::test] + async fn remark_refreshes_the_ttl() { + let flag = ClientUpdatePendingFlag::default(); + flag.mark().await; + tokio::time::sleep(Duration::from_millis(30)).await; + assert!(!flag.is_pending(Duration::from_millis(10)).await); + flag.mark().await; + assert!(flag.is_pending(Duration::from_millis(10)).await); + } + + #[tokio::test] + async fn clones_share_state() { + let flag = ClientUpdatePendingFlag::default(); + let clone = flag.clone(); + clone.mark().await; + assert!(flag.is_pending(LONG_TTL).await); + } + + #[tokio::test] + async fn clear_releases_the_flag() { + let flag = ClientUpdatePendingFlag::default(); + flag.mark().await; + assert!(flag.is_pending(LONG_TTL).await); + flag.clear().await; + assert!(!flag.is_pending(LONG_TTL).await); + } +} diff --git a/clients/openframe-client/src/services/update_cleanup_service.rs b/clients/openframe-client/src/services/update_cleanup_service.rs index a949203d5..424d61db3 100644 --- a/clients/openframe-client/src/services/update_cleanup_service.rs +++ b/clients/openframe-client/src/services/update_cleanup_service.rs @@ -23,9 +23,9 @@ impl UpdateCleanupService { Err(e) => warn!("Failed to cleanup old backups: {:#}", e), } - match self.cleanup_all_old_logs().await { + match self.cleanup_temp_update_leftovers().await { Ok(count) => cleaned += count, - Err(e) => warn!("Failed to cleanup old logs: {:#}", e), + Err(e) => warn!("Failed to cleanup temp update leftovers: {:#}", e), } if cleaned > 0 { @@ -68,22 +68,46 @@ impl UpdateCleanupService { Ok(cleaned) } - async fn cleanup_all_old_logs(&self) -> Result { + async fn cleanup_temp_update_leftovers(&self) -> Result { let temp_dir = std::env::temp_dir(); let mut cleaned = 0; if let Ok(entries) = fs::read_dir(&temp_dir) { for entry in entries.filter_map(|e| e.ok()) { let name = entry.file_name().to_string_lossy().to_string(); - if name.starts_with("openframe-update-") && name.ends_with(".log") { - match fs::remove_file(entry.path()) { - Ok(_) => { - info!("Removed old log: {}", entry.path().display()); - cleaned += 1; - } - Err(e) => { - warn!("Failed to remove log {}: {}", entry.path().display(), e); - } + let is_update_artifact = name.starts_with("openframe-update-") + || (name.starts_with("openframe-updater-") + && (name.ends_with(".ps1") || name.ends_with(".sh"))); + if !is_update_artifact { + continue; + } + + let too_fresh = entry + .metadata() + .ok() + .and_then(|m| m.modified().ok()) + .and_then(|t| t.elapsed().ok()) + .map(|age| { + age.as_secs() < crate::config::update_config::TEMP_LEFTOVER_MIN_AGE_SECS + }) + .unwrap_or(true); + if too_fresh { + continue; + } + + let path = entry.path(); + let result = if path.is_dir() { + fs::remove_dir_all(&path) + } else { + fs::remove_file(&path) + }; + match result { + Ok(_) => { + info!("Removed update leftover: {}", path.display()); + cleaned += 1; + } + Err(e) => { + warn!("Failed to remove update leftover {}: {}", path.display(), e); } } } diff --git a/clients/openframe-client/src/services/update_handler_service.rs b/clients/openframe-client/src/services/update_handler_service.rs index dff80020a..3ee9acfa8 100644 --- a/clients/openframe-client/src/services/update_handler_service.rs +++ b/clients/openframe-client/src/services/update_handler_service.rs @@ -1,17 +1,22 @@ +use crate::config::update_config::{CRASH_LOOP_MAX_BOOT_ATTEMPTS, UPDATER_TRANSCRIPTS_KEPT}; use crate::models::openframe_client_info::ClientUpdateStatus; use crate::models::update_state::{UpdatePhase, UpdateState}; +use crate::platform::updater_launcher::{self, UpdaterParams}; +use crate::service::FULL_SERVICE_NAME; use crate::services::agent_configuration_service::AgentConfigurationService; use crate::services::installed_agent_message_publisher::InstalledAgentMessagePublisher; +use crate::services::last_known_good_service::LastKnownGoodService; use crate::services::openframe_client_info_service::OpenFrameClientInfoService; use crate::services::update_cleanup_service::UpdateCleanupService; use crate::services::update_state_service::UpdateStateService; -use anyhow::{Context, Result}; +use anyhow::Result; use tracing::{info, warn}; #[derive(Clone)] pub struct UpdateHandlerService { state_service: UpdateStateService, client_info_service: OpenFrameClientInfoService, cleanup_service: UpdateCleanupService, + last_known_good_service: LastKnownGoodService, installed_agent_publisher: InstalledAgentMessagePublisher, config_service: AgentConfigurationService, } @@ -21,6 +26,7 @@ impl UpdateHandlerService { state_service: UpdateStateService, client_info_service: OpenFrameClientInfoService, cleanup_service: UpdateCleanupService, + last_known_good_service: LastKnownGoodService, installed_agent_publisher: InstalledAgentMessagePublisher, config_service: AgentConfigurationService, ) -> Self { @@ -28,11 +34,88 @@ impl UpdateHandlerService { state_service, client_info_service, cleanup_service, + last_known_good_service, installed_agent_publisher, config_service, } } + pub async fn record_boot_attempt(&self) -> Result<()> { + let mut update_state = match self.state_service.load().await? { + Some(state) => state, + None => return Ok(()), + }; + + if !matches!( + update_state.phase, + UpdatePhase::UpdaterLaunched | UpdatePhase::Verifying + ) { + return Ok(()); + } + + if update_state.target_version == env!("OPENFRAME_VERSION") { + info!( + "Update state targets the running version {} — skipping crash-loop accounting", + update_state.target_version + ); + return Ok(()); + } + + if update_state.boot_attempts >= CRASH_LOOP_MAX_BOOT_ATTEMPTS { + warn!( + "Update to {} still unresolved after {} boots — treating as failed (crash-loop guard)", + update_state.target_version, update_state.boot_attempts + ); + let target_version = update_state.target_version.clone(); + self.handle_failure(update_state).await?; + self.launch_reserve_rollback(&target_version).await; + return Ok(()); + } + + update_state.boot_attempts += 1; + self.state_service.save(&update_state).await?; + info!( + "Update to {} unresolved, boot attempt {}/{}", + update_state.target_version, update_state.boot_attempts, CRASH_LOOP_MAX_BOOT_ATTEMPTS + ); + Ok(()) + } + + async fn launch_reserve_rollback(&self, target_version: &str) { + if !(cfg!(windows) || cfg!(target_os = "macos")) { + info!("Reserve rollback not implemented on this platform yet"); + return; + } + + let current_exe = match std::env::current_exe() { + Ok(exe) => exe, + Err(e) => { + warn!("Cannot resolve current exe for reserve rollback: {:#}", e); + return; + } + }; + + let params = UpdaterParams { + binary_path: std::path::PathBuf::new(), + target_exe: current_exe, + service_name: FULL_SERVICE_NAME.to_string(), + update_state_path: self.state_service.get_state_file_path(), + target_version: target_version.to_string(), + boot_marker_path: self + .last_known_good_service + .boot_marker_path() + .to_path_buf(), + lkg_path: self.last_known_good_service.reserve_path().to_path_buf(), + transcript_path: self.last_known_good_service.new_transcript_path("rollback"), + rollback_only: true, + }; + + match updater_launcher::launch_updater(params).await { + Ok(_) => info!("Reserve rollback launched (crash-loop guard), service will restart"), + Err(e) => warn!("Failed to launch reserve rollback: {:#}", e), + } + } + pub async fn handle_pending_update(&self) -> Result<()> { let update_state = match self.state_service.load().await? { Some(state) => state, @@ -40,43 +123,70 @@ impl UpdateHandlerService { }; info!( - "Found update state: version={}, phase={:?}", - update_state.target_version, update_state.phase + "Found update state: version={}, phase={:?}, boot_attempts={}", + update_state.target_version, update_state.phase, update_state.boot_attempts ); - let update_succeeded = if update_state.phase == UpdatePhase::Completed { - true - } else { - let client_info = self.client_info_service.get().await?; - client_info.current_version == update_state.target_version - }; + let running_version = env!("OPENFRAME_VERSION"); + if running_version == update_state.target_version { + return self.handle_success(update_state).await; + } - if update_succeeded { - self.handle_success(update_state).await - } else { - self.handle_failure(update_state).await + match update_state.phase { + UpdatePhase::UpdaterLaunched | UpdatePhase::Verifying => { + info!( + "Update to {} not verified yet (running {}) — keeping state for next boot", + update_state.target_version, running_version + ); + Ok(()) + } + _ => self.handle_failure(update_state).await, } } async fn handle_success(&self, state: UpdateState) -> Result<()> { - info!("Update to {} succeeded", state.target_version); + info!( + "Update to {} succeeded (running binary matches target)", + state.target_version + ); + + match self + .last_known_good_service + .promote(&state.target_version) + .await + { + Ok(_) => info!("Last-known-good anchor raised to {}", state.target_version), + Err(e) => warn!( + "Failed to raise last-known-good anchor to {} (keeping previous anchor): {:#}", + state.target_version, e + ), + } - self.client_info_service + if let Err(e) = self + .client_info_service .update_version(state.target_version.clone()) .await - .context("Failed to update client version")?; + { + warn!("Failed to update client version bookkeeping: {:#}", e); + } - self.client_info_service + if let Err(e) = self + .client_info_service .set_update_status( ClientUpdateStatus::Updated, Some(state.target_version.clone()), ) .await - .context("Failed to set update status")?; + { + warn!("Failed to set update status to Updated: {:#}", e); + } self.send_nats_notification(&state.target_version).await; self.cleanup_service.cleanup_all().await; + self.last_known_good_service + .prune_transcripts(UPDATER_TRANSCRIPTS_KEPT) + .await; self.state_service.clear().await?; info!("Update completed, notified backend, cleaned up"); @@ -85,19 +195,25 @@ impl UpdateHandlerService { async fn handle_failure(&self, state: UpdateState) -> Result<()> { info!( - "Update to {} failed (PowerShell rollback done)", - state.target_version + "Update to {} failed (phase: {:?}; binary restore is owned by the updater script)", + state.target_version, state.phase ); - self.client_info_service + if let Err(e) = self + .client_info_service .set_update_status( ClientUpdateStatus::Failed, Some(state.target_version.clone()), ) .await - .context("Failed to set update status")?; + { + warn!("Failed to set update status to Failed: {:#}", e); + } self.cleanup_service.cleanup_all().await; + self.last_known_good_service + .prune_transcripts(UPDATER_TRANSCRIPTS_KEPT) + .await; self.state_service.clear().await?; info!("Update marked as failed, NATS will retry"); diff --git a/clients/openframe-client/src/services/update_state_service.rs b/clients/openframe-client/src/services/update_state_service.rs index 0f0d2061d..92281c663 100644 --- a/clients/openframe-client/src/services/update_state_service.rs +++ b/clients/openframe-client/src/services/update_state_service.rs @@ -58,9 +58,12 @@ impl UpdateStateService { let json_content = serde_json::to_string_pretty(state) .context("Failed to serialize update state to JSON")?; - fs::write(&self.state_file_path, json_content).with_context(|| { + let temp_path = self.state_file_path.with_extension("json.tmp"); + fs::write(&temp_path, json_content) + .with_context(|| format!("Failed to write temp update state file: {:?}", temp_path))?; + fs::rename(&temp_path, &self.state_file_path).with_context(|| { format!( - "Failed to write update state file: {:?}", + "Failed to move update state file into place: {:?}", self.state_file_path ) })?; From 059c161e1e94875fad15430a589f5c4b21619e45 Mon Sep 17 00:00:00 2001 From: Danylo Date: Tue, 21 Jul 2026 23:40:06 +0300 Subject: [PATCH 14/19] Remove tacticalrmm-agent-version feature flag (#2173) --- clients/openframe-client/Cargo.toml | 2 -- clients/openframe-client/build.rs | 2 -- clients/openframe-client/src/models/tool_version_overrides.rs | 3 --- 3 files changed, 7 deletions(-) diff --git a/clients/openframe-client/Cargo.toml b/clients/openframe-client/Cargo.toml index 085d0f1ea..390136f19 100644 --- a/clients/openframe-client/Cargo.toml +++ b/clients/openframe-client/Cargo.toml @@ -59,7 +59,6 @@ bin = [] openframe-chat-version = [] meshcentral-agent-version = [] fleetmdm-agent-version = [] -tacticalrmm-agent-version = [] # Asset-level version overrides. Keyed by Asset.id inside a ToolInstallationMessage. # These binaries ship as assets under another tool and can be pinned independently. @@ -70,7 +69,6 @@ all-tool-versions = [ "openframe-chat-version", "meshcentral-agent-version", "fleetmdm-agent-version", - "tacticalrmm-agent-version", "osquery-version", ] diff --git a/clients/openframe-client/build.rs b/clients/openframe-client/build.rs index 9871008fa..8f6e057d1 100644 --- a/clients/openframe-client/build.rs +++ b/clients/openframe-client/build.rs @@ -7,8 +7,6 @@ fn main() { forward_required("MESHCENTRAL_AGENT_VERSION"); #[cfg(feature = "fleetmdm-agent-version")] forward_required("FLEETMDM_AGENT_VERSION"); - #[cfg(feature = "tacticalrmm-agent-version")] - forward_required("TACTICALRMM_AGENT_VERSION"); #[cfg(feature = "osquery-version")] forward_required("OSQUERY_VERSION"); } diff --git a/clients/openframe-client/src/models/tool_version_overrides.rs b/clients/openframe-client/src/models/tool_version_overrides.rs index 5158c775d..bcd5f8e21 100644 --- a/clients/openframe-client/src/models/tool_version_overrides.rs +++ b/clients/openframe-client/src/models/tool_version_overrides.rs @@ -10,9 +10,6 @@ pub fn lookup(tool_key: &str) -> Option<&'static str> { #[cfg(feature = "fleetmdm-agent-version")] "fleetmdm-server" => Some(env!("FLEETMDM_AGENT_VERSION")), - #[cfg(feature = "tacticalrmm-agent-version")] - "tactical-rmm" => Some(env!("TACTICALRMM_AGENT_VERSION")), - #[cfg(feature = "osquery-version")] "osqueryd" => Some(env!("OSQUERY_VERSION")), From dc55a832c0f85ac2a3eac70e066de0e78d5b26a3 Mon Sep 17 00:00:00 2001 From: Danylo Date: Wed, 22 Jul 2026 01:05:58 +0300 Subject: [PATCH 15/19] fix(client): ascii-only updater script, write with utf-8 bom --- .../src/platform/update_scripts/mod.rs | 28 +++++++++++++++++++ .../src/platform/update_scripts/windows.rs | 4 +-- .../src/platform/updater_launcher/windows.rs | 4 ++- 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/clients/openframe-client/src/platform/update_scripts/mod.rs b/clients/openframe-client/src/platform/update_scripts/mod.rs index 0e3742cdd..132c4bbc0 100644 --- a/clients/openframe-client/src/platform/update_scripts/mod.rs +++ b/clients/openframe-client/src/platform/update_scripts/mod.rs @@ -9,3 +9,31 @@ pub use windows::UPDATE_SCRIPT_WINDOWS; #[cfg(target_os = "macos")] pub use macos::{UPDATER_PLIST_TEMPLATE, UPDATE_SCRIPT_MACOS}; + +#[cfg(test)] +mod tests { + // Windows PowerShell 5.1 reads BOM-less script files as ANSI: a multi-byte + // UTF-8 character can decode into a smart quote (e.g. 0x94 from an em-dash) + // that terminates a string early and structurally breaks the script. + #[cfg(target_os = "windows")] + #[test] + fn windows_update_script_is_ascii() { + assert!( + super::windows::UPDATE_SCRIPT_WINDOWS.is_ascii(), + "UPDATE_SCRIPT_WINDOWS must stay pure ASCII" + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn macos_update_script_is_ascii() { + assert!( + super::macos::UPDATE_SCRIPT_MACOS.is_ascii(), + "UPDATE_SCRIPT_MACOS must stay pure ASCII" + ); + assert!( + super::macos::UPDATER_PLIST_TEMPLATE.is_ascii(), + "UPDATER_PLIST_TEMPLATE must stay pure ASCII" + ); + } +} diff --git a/clients/openframe-client/src/platform/update_scripts/windows.rs b/clients/openframe-client/src/platform/update_scripts/windows.rs index 0cd00aea7..f70afc47e 100644 --- a/clients/openframe-client/src/platform/update_scripts/windows.rs +++ b/clients/openframe-client/src/platform/update_scripts/windows.rs @@ -157,7 +157,7 @@ try { break } if ($markerVersion) { - Write-Output "Boot marker reports '$markerVersion', expected '$TargetVersion' — wrong binary booted" + Write-Output "Boot marker reports '$markerVersion', expected '$TargetVersion' - wrong binary booted" break } } @@ -188,7 +188,7 @@ catch { Write-Output "Updater failed: $_" if (Test-AgentUninstalled) { - Write-Output "Update state file is gone (agent uninstalled mid-update) — standing down without touching the service" + Write-Output "Update state file is gone (agent uninstalled mid-update) - standing down without touching the service" if ($TempExtract -and (Test-Path $TempExtract)) { Remove-Item -Path $TempExtract -Recurse -Force -ErrorAction SilentlyContinue } diff --git a/clients/openframe-client/src/platform/updater_launcher/windows.rs b/clients/openframe-client/src/platform/updater_launcher/windows.rs index fba8120c7..c361e87fa 100644 --- a/clients/openframe-client/src/platform/updater_launcher/windows.rs +++ b/clients/openframe-client/src/platform/updater_launcher/windows.rs @@ -18,7 +18,9 @@ pub async fn launch_updater(params: UpdaterParams) -> Result<()> { let script_path = std::env::temp_dir().join(format!("openframe-updater-{}.ps1", Uuid::new_v4())); - tokio::fs::write(&script_path, UPDATE_SCRIPT_WINDOWS) + // UTF-8 BOM: without it Windows PowerShell 5.1 reads the file as ANSI, and + // any multi-byte character can decode into a smart quote that breaks parsing. + tokio::fs::write(&script_path, format!("\u{FEFF}{}", UPDATE_SCRIPT_WINDOWS)) .await .context("Failed to write PowerShell script")?; From 7fcbd72105bae0438a180588b2c323fbdcafe80f Mon Sep 17 00:00:00 2001 From: Ivan Date: Wed, 22 Jul 2026 00:10:50 +0300 Subject: [PATCH 16/19] Remove Redundant Folder (#2174) --- .../infrastructure/meshcentral/mac.sh | 747 ---------- .../infrastructure/meshcentral/win.ps1 | 614 -------- .../infrastructure/tactical-rmm/mac_arm64.sh | 537 ------- .../infrastructure/tactical-rmm/win_amd64.ps1 | 1323 ----------------- 4 files changed, 3221 deletions(-) delete mode 100755 clients/openframe-client/infrastructure/meshcentral/mac.sh delete mode 100644 clients/openframe-client/infrastructure/meshcentral/win.ps1 delete mode 100755 clients/openframe-client/infrastructure/tactical-rmm/mac_arm64.sh delete mode 100644 clients/openframe-client/infrastructure/tactical-rmm/win_amd64.ps1 diff --git a/clients/openframe-client/infrastructure/meshcentral/mac.sh b/clients/openframe-client/infrastructure/meshcentral/mac.sh deleted file mode 100755 index f8b4a8946..000000000 --- a/clients/openframe-client/infrastructure/meshcentral/mac.sh +++ /dev/null @@ -1,747 +0,0 @@ -#!/bin/bash - -# MeshCentral Agent Installer for *nix systems with customizable parameters and detailed output - -# Color and Emoji definitions -GREEN="\033[1;32m" -RED="\033[1;31m" -YELLOW="\033[1;33m" -BLUE="\033[1;34m" -RESET="\033[0m" -CHECK="✅" -CROSS="❌" -INFO="ℹ️" -WARN="⚠️" - -# Default parameters -MESH_SERVER="" -TEMP_DIR="/tmp/mesh_install" -BACKUP_DIR="/tmp/mesh_backup" -NODE_ID="" -UNINSTALL=false -FORCE_NEW_CERT=false - -# Identity file preservation settings -IDENTITY_FILES=( - "mesh.db" # Main database file - "meshagent.msh" # Configuration file - "meshagent.db" # Agent database - "settings.json" # Agent settings - "state.json" # Agent state - "nodeinfo.json" # Node information - "identitydata.json" # Identity data -) - -IDENTITY_DIRS=( - "data" # Data directory - "db" # Database directory - "config" # Configuration directory -) - -# OS Detection -detect_os() { - if [ -f /etc/os-release ]; then - . /etc/os-release - OS_NAME=$ID - elif [ -f /etc/lsb-release ]; then - . /etc/lsb-release - OS_NAME=$DISTRIB_ID - elif [ "$(uname)" = "Darwin" ]; then - OS_NAME="macos" - else - OS_NAME="unknown" - fi - OS_NAME=$(echo "$OS_NAME" | tr '[:upper:]' '[:lower:]') -} - -# Architecture Detection -detect_arch() { - local arch=$(uname -m) - case $arch in - x86_64) - if [ -n "$(grep -E 'vmx|svm' /proc/cpuinfo 2>/dev/null)" ]; then - ARCH="x64" - else - ARCH="x86" - fi - ;; - aarch64 | arm64) - ARCH="arm64" - ;; - armv7* | armv8*) - ARCH="arm" - ;; - *) - echo -e "${RED}${CROSS} Unsupported architecture: $arch${RESET}" - exit 1 - ;; - esac -} - -# Get agent ID based on OS and architecture -get_agent_id() { - case $OS_NAME in - "macos") - case $ARCH in - "arm64") AGENT_ID="10005" ;; # Apple Silicon - "x64") AGENT_ID="4" ;; # Intel Mac - *) - echo -e "${RED}${CROSS} Unsupported macOS architecture${RESET}" - exit 1 - ;; - esac - ;; - "ubuntu" | "debian" | "linuxmint") - case $ARCH in - "x64") AGENT_ID="6" ;; - "arm64") AGENT_ID="10003" ;; - "arm") AGENT_ID="10004" ;; - *) - echo -e "${RED}${CROSS} Unsupported Linux architecture${RESET}" - exit 1 - ;; - esac - ;; - *) - echo -e "${RED}${CROSS} Unsupported operating system: $OS_NAME${RESET}" - exit 1 - ;; - esac -} - -# Debug print function -debug_print() { - echo -e "${YELLOW}${INFO} DEBUG: $1${RESET}" -} - -# Function for retries -retry() { - local retries=$1 - shift - local count=0 - debug_print "Executing command: $*" - until "$@"; do - exit_code=$? - wait_time=$((2 ** $count)) - count=$((count + 1)) - if [ $count -lt $retries ]; then - echo -e "${YELLOW}${WARN} Command failed. Retrying in $wait_time seconds...${RESET}" - sleep $wait_time - else - echo -e "${RED}${CROSS} Command failed after $retries attempts.${RESET}" - return $exit_code - fi - done - return 0 -} - -# Stop MeshAgent processes -stop_mesh_agent() { - debug_print "Stopping any running MeshAgent processes" - local pid - if pids=$(pgrep -f "meshagent"); then - for pid in $pids; do - debug_print "Stopping MeshAgent process with PID: $pid" - kill -15 "$pid" 2>/dev/null || true - done - sleep 2 # Give processes time to stop - else - debug_print "No running MeshAgent processes found" - fi -} - -# Backup identity files -backup_identity_files() { - local source_dir="$1" - local backup_dir="$2" - - if [ ! -d "$source_dir" ]; then - debug_print "No existing installation found to backup at: $source_dir" - return 1 - fi - - # Create backup directory - mkdir -p "$backup_dir" - debug_print "Created backup directory: $backup_dir" - - # Check if any identity files exist - local has_identity_files=false - - # Backup individual files - for file in "${IDENTITY_FILES[@]}"; do - local source_path="$source_dir/$file" - if [ -f "$source_path" ]; then - has_identity_files=true - local dest_path="$backup_dir/$file" - debug_print "Backing up identity file: $file" - cp -f "$source_path" "$dest_path" 2>/dev/null || true - fi - done - - # Backup directories - for dir in "${IDENTITY_DIRS[@]}"; do - local source_subdir="$source_dir/$dir" - if [ -d "$source_subdir" ]; then - has_identity_files=true - local dest_subdir="$backup_dir/$dir" - debug_print "Backing up identity directory: $dir" - mkdir -p "$dest_subdir" 2>/dev/null || true - cp -rf "$source_subdir"/* "$dest_subdir" 2>/dev/null || true - fi - done - - if [ "$has_identity_files" = true ]; then - echo -e "${GREEN}${CHECK} Successfully backed up identity files.${RESET}" - return 0 - else - echo -e "${YELLOW}${INFO} No identity files found to backup.${RESET}" - return 1 - fi -} - -# Restore identity files -restore_identity_files() { - local backup_dir="$1" - local target_dir="$2" - - if [ ! -d "$backup_dir" ]; then - debug_print "No backup directory found to restore from: $backup_dir" - return 1 - fi - - # Create target directory if it doesn't exist - mkdir -p "$target_dir" - - # Restore individual files - for file in "${IDENTITY_FILES[@]}"; do - local source_path="$backup_dir/$file" - if [ -f "$source_path" ]; then - local dest_path="$target_dir/$file" - debug_print "Restoring identity file: $file" - cp -f "$source_path" "$dest_path" 2>/dev/null || true - # Fix permissions - chmod 644 "$dest_path" 2>/dev/null || true - fi - done - - # Restore directories - for dir in "${IDENTITY_DIRS[@]}"; do - local source_subdir="$backup_dir/$dir" - if [ -d "$source_subdir" ]; then - local dest_subdir="$target_dir/$dir" - debug_print "Restoring identity directory: $dir" - mkdir -p "$dest_subdir" 2>/dev/null || true - cp -rf "$source_subdir"/* "$dest_subdir" 2>/dev/null || true - fi - done - - echo -e "${GREEN}${CHECK} Identity files restored.${RESET}" - return 0 -} - -# Selective cleanup function (preserves identity files) -selective_cleanup() { - local dir="$1" - - # Validate input - if [ -z "$dir" ]; then - echo -e "${RED}${CROSS} Error: Directory path is empty${RESET}" - return 1 - fi - - # Ensure we're working with absolute paths - if [[ "$dir" != /* ]]; then - echo -e "${RED}${CROSS} Error: Directory path must be absolute: $dir${RESET}" - return 1 - fi - - # IMPORTANT: Check that this is NOT a system directory - case "$dir" in - "/"|"/usr"|"/usr/bin"|"/bin"|"/sbin"|"/usr/sbin"|"/etc"|"/var"|"/opt"|"/lib"|"/lib64"|"/boot"|"/dev"|"/proc"|"/sys"|"/root"|"/home") - echo -e "${RED}${CROSS} Error: Refusing to clean system directory: $dir${RESET}" - return 1 - ;; - esac - - # Additional safety check for system directories - if [[ "$dir" =~ ^/(usr|bin|sbin|etc|var|opt|lib|lib64|boot|dev|proc|sys|root|home)(/|$) ]]; then - echo -e "${RED}${CROSS} Error: Refusing to clean directory that might be system-related: $dir${RESET}" - return 1 - fi - - debug_print "Performing selective cleanup of directory: $dir" - - # Create a temporary directory to store files to preserve - local temp_preserve_dir="/tmp/mesh_preserve_temp" - mkdir -p "$temp_preserve_dir" - - # Backup identity files to temporary location - for file in "${IDENTITY_FILES[@]}"; do - if [ -f "$dir/$file" ]; then - debug_print "Preserving file during cleanup: $file" - cp -f "$dir/$file" "$temp_preserve_dir/$file" 2>/dev/null || true - fi - done - - # Backup identity directories to temporary location - for subdir in "${IDENTITY_DIRS[@]}"; do - if [ -d "$dir/$subdir" ]; then - debug_print "Preserving directory during cleanup: $subdir" - mkdir -p "$temp_preserve_dir/$subdir" 2>/dev/null || true - cp -rf "$dir/$subdir"/* "$temp_preserve_dir/$subdir" 2>/dev/null || true - fi - done - - # Remove all files in the directory - debug_print "Removing files from directory: $dir" - rm -rf "$dir"/* 2>/dev/null || true - - # Restore the preserved files - for file in "${IDENTITY_FILES[@]}"; do - if [ -f "$temp_preserve_dir/$file" ]; then - debug_print "Restoring preserved file: $file" - cp -f "$temp_preserve_dir/$file" "$dir/$file" 2>/dev/null || true - fi - done - - # Restore the preserved directories - for subdir in "${IDENTITY_DIRS[@]}"; do - if [ -d "$temp_preserve_dir/$subdir" ]; then - debug_print "Restoring preserved directory: $subdir" - mkdir -p "$dir/$subdir" 2>/dev/null || true - cp -rf "$temp_preserve_dir/$subdir"/* "$dir/$subdir" 2>/dev/null || true - fi - done - - # Clean up the temporary preserve directory - rm -rf "$temp_preserve_dir" - - debug_print "Selective cleanup completed for: $dir" -} - -# Complete cleanup function (removes everything) -cleanup() { - local dir="$1" - debug_print "Cleaning up directory: $dir" - retry 3 sudo rm -rf "$dir" -} - -# Uninstall function -uninstall_mesh_agent() { - echo -e "${YELLOW}${INFO} Uninstalling MeshAgent...${RESET}" - - # Stop any running MeshAgent processes - stop_mesh_agent - - # Determine installation directories based on OS - local install_locations=() - - if [ "$OS_NAME" = "macos" ]; then - install_locations=( - "/usr/local/bin/meshagent" - "/usr/local/bin/meshagent.msh" - "/Library/MeshAgent" - "/usr/local/mesh_install" - "/var/mesh_install" - ) - - # Check for and remove Launch Agents/Daemons - if [ -f "/Library/LaunchDaemons/com.meshcentral.agent.plist" ]; then - debug_print "Removing launch daemon" - sudo launchctl unload "/Library/LaunchDaemons/com.meshcentral.agent.plist" 2>/dev/null || true - sudo rm -f "/Library/LaunchDaemons/com.meshcentral.agent.plist" 2>/dev/null || true - fi - - # Remove preference files - sudo rm -rf "/Library/Preferences/MeshAgent" 2>/dev/null || true - - else # Linux - install_locations=( - "/usr/local/bin/meshagent" - "/usr/local/bin/meshagent.msh" - "/opt/meshagent" - "/etc/meshagent" - ) - - # Remove systemd service if it exists - if [ -f "/etc/systemd/system/meshagent.service" ]; then - debug_print "Removing systemd service" - sudo systemctl stop meshagent 2>/dev/null || true - sudo systemctl disable meshagent 2>/dev/null || true - sudo rm -f "/etc/systemd/system/meshagent.service" 2>/dev/null || true - sudo systemctl daemon-reload 2>/dev/null || true - fi - fi - - # Remove all installation files - for location in "${install_locations[@]}"; do - if [ -e "$location" ]; then - debug_print "Removing: $location" - sudo rm -rf "$location" 2>/dev/null || true - fi - done - - # Clean up any temporary directories - cleanup "$TEMP_DIR" - cleanup "$BACKUP_DIR" - - echo -e "${GREEN}${CHECK} MeshAgent has been uninstalled.${RESET}" - exit 0 -} - -# Help function -show_help() { - echo -e "${BLUE}${INFO} MeshCentral Agent Installer for *nix darwin Systems${RESET}" - echo "" - echo "Usage: $0 [options]" - echo "" - echo "Options:" - echo " --server= (Required) URL of your MeshCentral server (without https://)" - echo " --nodeid= (Optional) NodeID to inject into the MSH file" - echo " --uninstall (Optional) Completely remove MeshAgent from this system" - echo " --force-new-cert (Optional) Force certificate reset to resolve server certificate mismatch issues" - echo " --help Display this help message" - echo "" - echo "Example:" - echo " $0 --server=mesh.yourdomain.com" - echo " $0 --server=mesh.yourdomain.com --nodeid=node//1E3vUyW4i1Je\$hiyT8ec87bEXPVj\$sEahRAFDtfNSKgS5XJQBotfsN9Y\$v0hw6xa" - echo " $0 --uninstall" - exit 0 -} - -# Parse arguments -for ARG in "$@"; do - case $ARG in - --server=*) MESH_SERVER="${ARG#*=}" ;; - --nodeid=*) NODE_ID="${ARG#*=}" ;; - --uninstall) UNINSTALL=true ;; - --force-new-cert) FORCE_NEW_CERT=true ;; - --help) show_help ;; - *) - echo -e "${RED}${CROSS} Unknown argument: $ARG${RESET}" - show_help - ;; - esac -done - -# Validate required parameters -if [ "$UNINSTALL" = false ] && [ -z "$MESH_SERVER" ]; then - echo -e "${RED}${CROSS} Error: Mesh server URL (--server) is required unless uninstalling.${RESET}" - show_help -fi - -# Ensure running as root -if [ "$EUID" -ne 0 ]; then - echo -e "${RED}${CROSS} Error: Please run this script with sudo or as root.${RESET}" - exit 1 -fi - -# Process uninstall request if specified -if [ "$UNINSTALL" = true ]; then - uninstall_mesh_agent -fi - -# Detect OS and architecture -detect_os -detect_arch -get_agent_id - -debug_print "Detected OS: $OS_NAME, Architecture: $ARCH, Agent ID: $AGENT_ID" - -# Check for existing installation and define install directory -if [ "$OS_NAME" = "macos" ]; then - INSTALL_DIR="/usr/local/bin" - DATA_DIR="/Library/MeshAgent" -else - INSTALL_DIR="/opt/meshagent" - DATA_DIR="/var/lib/meshagent" -fi - -# Stop any running instances first -stop_mesh_agent - -# Check for existing installation and backup identity files -HAS_EXISTING_INSTALLATION=false -HAS_IDENTITY_BACKUP=false - -if [ -f "$INSTALL_DIR/meshagent" ]; then - HAS_EXISTING_INSTALLATION=true - echo -e "${YELLOW}${INFO} Existing installation found. Preserving identity files...${RESET}" - - # If force certificate reset is specified, modify the identity files list - if [ "$FORCE_NEW_CERT" = true ]; then - echo -e "${YELLOW}${INFO} Certificate reset requested - will not preserve certificate data${RESET}" - debug_print "Certificate reset mode - limiting preserved files" - - # Modified list that excludes certificate-related files - IDENTITY_FILES=( - # Keep minimal identity info, but exclude certificate data - "nodeinfo.json" # Node information - ) - - IDENTITY_DIRS=() - fi - - if backup_identity_files "$INSTALL_DIR" "$BACKUP_DIR"; then - HAS_IDENTITY_BACKUP=true - if [ "$FORCE_NEW_CERT" = true ]; then - echo -e "${GREEN}${CHECK} Successfully backed up minimal identity files (certificate reset mode).${RESET}" - else - echo -e "${GREEN}${CHECK} Successfully backed up identity files.${RESET}" - fi - else - echo -e "${YELLOW}${INFO} No identity files found to backup.${RESET}" - fi - - # Also check data directory if it exists - if [ -d "$DATA_DIR" ] && [ "$HAS_IDENTITY_BACKUP" = false ]; then - if backup_identity_files "$DATA_DIR" "$BACKUP_DIR"; then - HAS_IDENTITY_BACKUP=true - if [ "$FORCE_NEW_CERT" = true ]; then - echo -e "${GREEN}${CHECK} Successfully backed up minimal identity files from data directory (certificate reset mode).${RESET}" - else - echo -e "${GREEN}${CHECK} Successfully backed up identity files from data directory.${RESET}" - fi - fi - fi -else - debug_print "No existing installation found. Will perform fresh install." -fi - -# Create directories -debug_print "Creating directories: $TEMP_DIR" -retry 3 sudo mkdir -p "$TEMP_DIR" - -# Display file paths for user clarity -echo -e "${BLUE}${INFO} File Destinations:${RESET}" -echo -e "${BLUE}${INFO} - Temporary directory: ${YELLOW}$TEMP_DIR${RESET}" -echo -e "${BLUE}${INFO} - Installation directory: ${YELLOW}$INSTALL_DIR${RESET}" -echo -e "${BLUE}${INFO} - Data directory: ${YELLOW}$DATA_DIR${RESET}" - -# Clean up temporary directory -cleanup "$TEMP_DIR" - -# Selectively clean installation directory if needed -if [ "$HAS_EXISTING_INSTALLATION" = true ]; then - debug_print "Performing selective cleanup of installation directory" - selective_cleanup "$INSTALL_DIR" - - # Also clean up data directory if it exists - if [ -d "$DATA_DIR" ]; then - selective_cleanup "$DATA_DIR" - fi -fi - -# Create directories again in case they were removed -mkdir -p "$TEMP_DIR" -mkdir -p "$INSTALL_DIR" -mkdir -p "$DATA_DIR" - -# Download MeshAgent binary -AGENT_URL="https://$MESH_SERVER/meshagents?id=$AGENT_ID" -AGENT_PATH="$TEMP_DIR/meshagent" - -debug_print "Downloading MeshAgent binary from $AGENT_URL" -echo -e "${BLUE}${INFO} - Agent binary location: ${YELLOW}$AGENT_PATH${RESET}" -retry 3 curl -k "$AGENT_URL" -o "$AGENT_PATH" - -if [ $? -ne 0 ]; then - echo -e "${RED}${CROSS} Error: Unable to download MeshAgent binary. Check your server URL and network connection.${RESET}" - exit 1 -fi - -retry 3 sudo chmod +x "$AGENT_PATH" - -# Platform-specific quarantine handling -if [ "$OS_NAME" = "macos" ]; then - debug_print "Removing quarantine attribute from downloaded MeshAgent binary (macOS specific)" - # Suppress errors if attribute doesn't exist by using || true - sudo xattr -d com.apple.quarantine "$AGENT_PATH" 2>/dev/null || true - # Alternative approach - set empty attribute - sudo xattr -w com.apple.quarantine "" "$AGENT_PATH" 2>/dev/null || true - - # Extra security approval for macOS - debug_print "Approving binary for execution" - sudo spctl --add --label "MeshAgent" "$AGENT_PATH" 2>/dev/null || true - sudo spctl --enable --label "MeshAgent" 2>/dev/null || true -fi - -CONFIG_URL="https://$MESH_SERVER/openframe_public/meshagent.msh" -CONFIG_PATH="$TEMP_DIR/meshagent.msh" - -# Download MeshAgent configuration file -debug_print "Downloading MeshAgent configuration file" -echo -e "${BLUE}${INFO} - Config file location: ${YELLOW}$CONFIG_PATH${RESET}" -retry 3 curl -k "$CONFIG_URL" -o "$CONFIG_PATH" - -if [ $? -ne 0 ]; then - echo -e "${RED}${CROSS} Error: Unable to download MeshAgent configuration file. Check your server URL and network connection.${RESET}" - exit 1 -fi - -# Add NodeID to the MSH file if provided -if [ -n "$NODE_ID" ]; then - debug_print "Adding NodeID to the MSH file: $NODE_ID" - echo "NodeID=$NODE_ID" >> "$CONFIG_PATH" - echo -e "${BLUE}${INFO} - Added NodeID to configuration file${RESET}" -fi - -# Platform-specific quarantine handling for config file -if [ "$OS_NAME" = "macos" ]; then - debug_print "Removing quarantine attribute from configuration file (macOS specific)" - # Suppress errors if attribute doesn't exist by using || true - sudo xattr -d com.apple.quarantine "$CONFIG_PATH" 2>/dev/null || true - # Alternative approach - set empty attribute - sudo xattr -w com.apple.quarantine "" "$CONFIG_PATH" 2>/dev/null || true -fi - -echo -e "${GREEN}${CHECK} MeshAgent and configuration successfully Downloaded.${RESET}" - -# Request screen sharing permissions on macOS -request_screen_permissions() { - if [ "$OS_NAME" = "macos" ]; then - debug_print "Checking screen sharing permissions" - - # Check if screen recording permission is already granted - # Try to capture a screenshot as a test - TEST_SCREENSHOT="/tmp/meshcentral_test_screenshot.png" - if screencapture -x "$TEST_SCREENSHOT" 2>/dev/null; then - debug_print "Screen recording permission already granted" - SCREEN_RECORDING_GRANTED=true - rm -f "$TEST_SCREENSHOT" - else - debug_print "Screen recording permission not granted" - SCREEN_RECORDING_GRANTED=false - fi - - # Check if full disk access is already granted - # Try to access a protected directory - if ls /Library/Application\ Support/com.apple.TCC 2>/dev/null; then - debug_print "Full disk access permission already granted" - FULL_DISK_ACCESS_GRANTED=true - else - debug_print "Full disk access permission not granted" - FULL_DISK_ACCESS_GRANTED=false - fi - - # Request screen recording permission if not granted - if [ "$SCREEN_RECORDING_GRANTED" = false ]; then - debug_print "Requesting screen recording permission" - osascript < Screen Recording - do shell script "open 'x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture'" - delay 1 - # User instructions via dialog - display dialog "Please click the '+' button and add the MeshCentral agent to allow screen sharing." buttons {"OK"} default button "OK" with icon caution with title "Screen Sharing Permission Required" - end tell -EOD - fi - - # Request full disk access if not granted - if [ "$FULL_DISK_ACCESS_GRANTED" = false ]; then - debug_print "Requesting full disk access permission" - osascript < Full Disk Access - do shell script "open 'x-apple.systempreferences:com.apple.preference.security?Privacy_AllFiles'" - delay 1 - # User instructions via dialog - display dialog "Please also grant Full Disk Access to the MeshCentral agent for complete functionality." buttons {"OK"} default button "OK" with icon caution with title "Full Disk Access Required" - end tell -EOD - fi - - # If any permissions were requested, give user time to approve - if [ "$SCREEN_RECORDING_GRANTED" = false ] || [ "$FULL_DISK_ACCESS_GRANTED" = false ]; then - echo -e "${YELLOW}${INFO} Waiting for permissions approval...${RESET}" - sleep 5 - else - debug_print "All required permissions already granted" - fi - fi -} - -# Request necessary permissions -request_screen_permissions - -# Create log directory if it doesn't exist -LOG_DIR="$(dirname "$TEMP_DIR")/meshagent_logs" -debug_print "Creating log directory: $LOG_DIR" -echo -e "${BLUE}${INFO} - Log directory: ${YELLOW}$LOG_DIR${RESET}" -retry 3 sudo mkdir -p "$LOG_DIR" - -# Set log file path -LOG_FILE="$LOG_DIR/meshagent.log" -debug_print "Agent output will be logged to: $LOG_FILE" -echo -e "${BLUE}${INFO} - Log file: ${YELLOW}$LOG_FILE${RESET}" - -# Copy files to installation directory -debug_print "Copying files to installation directory" -sudo cp "$AGENT_PATH" "$INSTALL_DIR/meshagent" -sudo chmod +x "$INSTALL_DIR/meshagent" - -# Always override the MSH configuration file -sudo cp "$CONFIG_PATH" "$INSTALL_DIR/meshagent.msh" -debug_print "Copied new configuration file to installation directory" - -FINAL_AGENT_PATH="$INSTALL_DIR/meshagent" -FINAL_CONFIG_PATH="$INSTALL_DIR/meshagent.msh" -echo -e "${BLUE}${INFO} - Final agent location: ${YELLOW}$FINAL_AGENT_PATH${RESET}" -echo -e "${BLUE}${INFO} - Final config location: ${YELLOW}$FINAL_CONFIG_PATH${RESET}" - -# Restore identity files if we backed them up -if [ "$HAS_IDENTITY_BACKUP" = true ]; then - echo -e "${YELLOW}${INFO} Restoring identity files from backup...${RESET}" - restore_identity_files "$BACKUP_DIR" "$INSTALL_DIR" - - # Also restore to data directory if needed - if [ -d "$DATA_DIR" ]; then - restore_identity_files "$BACKUP_DIR" "$DATA_DIR" - fi -fi - -# Clean up temp files before starting agent -debug_print "Cleaning up temporary directory: $TEMP_DIR" -cleanup "$TEMP_DIR" - -# Clean up backup directory after successful restore -if [ "$HAS_IDENTITY_BACKUP" = true ]; then - debug_print "Cleaning up backup directory: $BACKUP_DIR" - cleanup "$BACKUP_DIR" -fi - -# Verify agent status -debug_print "Running MeshCentral agent" - -# Installation summary -echo -e "${GREEN}${CHECK} Installation Summary:${RESET}" -echo -e "${BLUE}${INFO} - Agent Location: ${YELLOW}$FINAL_AGENT_PATH${RESET}" -echo -e "${BLUE}${INFO} - Config Location: ${YELLOW}$FINAL_CONFIG_PATH${RESET}" -echo -e "${BLUE}${INFO} - Log Location: ${YELLOW}$LOG_FILE${RESET}" -if [ "$HAS_IDENTITY_BACKUP" = true ]; then - if [ "$FORCE_NEW_CERT" = true ]; then - echo -e "${BLUE}${INFO} - Certificate reset mode - minimal identity files were preserved${RESET}" - else - echo -e "${BLUE}${INFO} - Identity files were preserved from previous installation${RESET}" - fi -fi -if [ "$FORCE_NEW_CERT" = true ]; then - echo -e "${YELLOW}${INFO} Certificate reset was applied${RESET}" -fi -echo -e "${GREEN}${CHECK} Installation completed successfully.${RESET}" -echo -e "${YELLOW}${INFO} Starting MeshAgent in connect mode...${RESET}" -echo -e "${YELLOW}${INFO} Press Ctrl+C to exit (agent will continue running in background)${RESET}" - -# Run agent with full path to the installation location -echo -e "${BLUE}${INFO} - Executing agent from: ${YELLOW}$FINAL_AGENT_PATH${RESET}" -retry 5 sudo "$FINAL_AGENT_PATH" connect - -# Final debug print -debug_print "Execution process completed successfully" - -exit 0 diff --git a/clients/openframe-client/infrastructure/meshcentral/win.ps1 b/clients/openframe-client/infrastructure/meshcentral/win.ps1 deleted file mode 100644 index 43adf3566..000000000 --- a/clients/openframe-client/infrastructure/meshcentral/win.ps1 +++ /dev/null @@ -1,614 +0,0 @@ -[CmdletBinding()] -param( - [Parameter(Mandatory=$false)] - [string]$Server, - - [Parameter(Mandatory=$false)] - [string]$NodeId, - - [Parameter(Mandatory=$false)] - [switch]$Help, - - [Parameter(Mandatory=$false)] - [switch]$Uninstall, - - [Parameter(Mandatory=$false)] - [switch]$ForceNewCert -) - -# MeshCentral Agent Installer for Windows systems -# Requires -RunAsAdministrator - -# Color definitions for Windows console -$Colors = @{ - Green = '[92m' - Red = '[91m' - Yellow = '[93m' - Blue = '[94m' - Reset = '[0m' -} - -function Write-ColorMessage { - param( - [string]$Message, - [string]$Color, - [switch]$NoNewLine - ) - if ($NoNewLine) { - Write-Host "$($Colors[$Color])$Message$($Colors['Reset'])" -NoNewline - } else { - Write-Host "$($Colors[$Color])$Message$($Colors['Reset'])" - } -} - -function Write-VerboseMessage { - param( - [string]$Message - ) - Write-Verbose " → $Message" -} - -function Show-Help { - Write-ColorMessage "MeshCentral Agent Installer for Windows Systems" "Blue" - Write-Host "`nUsage: $($MyInvocation.MyCommand.Name) [options]`n" - Write-Host "Options:" - Write-Host " -Server (Required) URL of your MeshCentral server (without https://)" - Write-Host " -NodeId (Optional) NodeID to inject into the MSH file" - Write-Host " -Help Display this help message" - Write-Host " -Uninstall Completely remove MeshAgent from this system" - Write-Host " -ForceNewCert Force certificate reset to resolve server certificate mismatch issues" - Write-Host " -Verbose Show detailed output`n" - Write-Host "Example:" - Write-Host " $($MyInvocation.MyCommand.Name) -Server mesh.yourdomain.com [-Verbose]" - Write-Host " $($MyInvocation.MyCommand.Name) -Server mesh.yourdomain.com -NodeId 'node//1E3vUyW4i1Je`$hiyT8ec87bEXPVj`$sEahRAFDtfNSKgS5XJQBotfsN9Y`$v0hw6xa'" - exit 1 -} - -function Stop-MeshAgent { - Write-VerboseMessage "Stopping any running MeshAgent processes..." - Get-Process | Where-Object { $_.ProcessName -eq "meshagent" } | ForEach-Object { - Write-VerboseMessage "Stopping process: $($_.Id)" - Stop-Process -Id $_.Id -Force -ErrorAction SilentlyContinue - } - Start-Sleep -Seconds 2 # Give processes time to stop -} - -function Remove-Directory { - param( - [string]$Path, - [switch]$PreserveIdentityFiles - ) - if (Test-Path $Path) { - Write-VerboseMessage "Removing directory: $Path" - - if ($PreserveIdentityFiles -and $Path -eq $InstallDir) { - Write-VerboseMessage "Preserving identity files during cleanup" - try { - # Remove only specific files, not the entire directory - $filesToRemove = Get-ChildItem -Path $Path -File | Where-Object { $_.Name -notin $IdentityFilesToPreserve } - foreach ($file in $filesToRemove) { - try { - Remove-Item $file.FullName -Force -ErrorAction SilentlyContinue - Write-VerboseMessage "Removed: $($file.FullName)" - } - catch { - Write-VerboseMessage "Could not remove: $($file.FullName)" - } - } - - # Remove non-identity subdirectories - $dirsToRemove = Get-ChildItem -Path $Path -Directory | Where-Object { $_.Name -notin $IdentityDirsToPreserve } - foreach ($dir in $dirsToRemove) { - try { - Remove-Item $dir.FullName -Recurse -Force -ErrorAction SilentlyContinue - Write-VerboseMessage "Removed directory: $($dir.FullName)" - } - catch { - Write-VerboseMessage "Could not remove directory: $($dir.FullName)" - } - } - } - catch { - Write-VerboseMessage "Error during selective removal: $($_.Exception.Message)" - } - } - else { - # Remove the entire directory - try { - Remove-Item -Path $Path -Recurse -Force -ErrorAction Stop - } - catch { - Write-VerboseMessage "Failed to remove directory: $($_.Exception.Message)" - # Try to remove files individually - Get-ChildItem -Path $Path -Recurse | ForEach-Object { - try { - Remove-Item $_.FullName -Force -ErrorAction SilentlyContinue - } - catch { - Write-VerboseMessage "Could not remove: $($_.FullName)" - } - } - } - } - } -} - -function Test-Administrator { - $currentUser = New-Object Security.Principal.WindowsPrincipal([Security.Principal.WindowsIdentity]::GetCurrent()) - return $currentUser.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator) -} - -function Get-AgentArchitecture { - if ([Environment]::Is64BitOperatingSystem) { - return @{ - Arch = "x64" - AgentId = "3" - } - } else { - return @{ - Arch = "x86" - AgentId = "1" - } - } -} - -function Test-ServerConnection { - param( - [string]$ServerUrl - ) - try { - Write-ColorMessage "Testing connection to $ServerUrl..." "Yellow" - $request = [System.Net.WebRequest]::Create("https://$ServerUrl") - $request.Method = "HEAD" - $request.Timeout = 5000 - $request.ServerCertificateValidationCallback = { $true } - - try { - Write-VerboseMessage "Sending HEAD request to verify server availability..." - $response = $request.GetResponse() - $response.Close() - Write-VerboseMessage "Server connection successful" - return $true - } - catch [System.Net.WebException] { - if ($_.Exception.Response -and $_.Exception.Response.StatusCode) { - # If we get any HTTP response, server is reachable - Write-VerboseMessage "Server responded with status code: $($_.Exception.Response.StatusCode)" - return $true - } - Write-ColorMessage "Server is not responding. Error: $($_.Exception.Message)" "Red" - return $false - } - } - catch { - Write-ColorMessage "Connection test failed: $($_.Exception.Message)" "Red" - return $false - } -} - -function Backup-IdentityFiles { - param( - [string]$SourceDir, - [string]$BackupDir - ) - - if (-not (Test-Path $SourceDir)) { - Write-VerboseMessage "No existing installation found to backup" - return $false - } - - # Create backup directory - if (-not (Test-Path $BackupDir)) { - New-Item -ItemType Directory -Path $BackupDir -Force | Out-Null - } - - # Check if any identity files exist - $hasIdentityFiles = $false - - # Look for database files (*.db) and mesh agent state files - foreach ($file in $IdentityFilesToPreserve) { - $sourcePath = Join-Path $SourceDir $file - if (Test-Path $sourcePath) { - $hasIdentityFiles = $true - $destPath = Join-Path $BackupDir $file - Write-VerboseMessage "Backing up identity file: $file" - Copy-Item -Path $sourcePath -Destination $destPath -Force -ErrorAction SilentlyContinue - } - } - - # Backup specific subdirectories that may contain identity information - foreach ($dir in $IdentityDirsToPreserve) { - $sourceSubDir = Join-Path $SourceDir $dir - if (Test-Path $sourceSubDir) { - $hasIdentityFiles = $true - $destSubDir = Join-Path $BackupDir $dir - Write-VerboseMessage "Backing up identity directory: $dir" - Copy-Item -Path $sourceSubDir -Destination $destSubDir -Recurse -Force -ErrorAction SilentlyContinue - } - } - - return $hasIdentityFiles -} - -function Restore-IdentityFiles { - param( - [string]$BackupDir, - [string]$TargetDir - ) - - if (-not (Test-Path $BackupDir)) { - Write-VerboseMessage "No backup directory found to restore from" - return $false - } - - # Ensure target directory exists - if (-not (Test-Path $TargetDir)) { - New-Item -ItemType Directory -Path $TargetDir -Force | Out-Null - } - - # Restore individual files - foreach ($file in $IdentityFilesToPreserve) { - $sourcePath = Join-Path $BackupDir $file - if (Test-Path $sourcePath) { - $destPath = Join-Path $TargetDir $file - Write-VerboseMessage "Restoring identity file: $file" - Copy-Item -Path $sourcePath -Destination $destPath -Force -ErrorAction SilentlyContinue - } - } - - # Restore directories - foreach ($dir in $IdentityDirsToPreserve) { - $sourceSubDir = Join-Path $BackupDir $dir - if (Test-Path $sourceSubDir) { - $destSubDir = Join-Path $TargetDir $dir - Write-VerboseMessage "Restoring identity directory: $dir" - Copy-Item -Path $sourceSubDir -Destination $destSubDir -Recurse -Force -ErrorAction SilentlyContinue - } - } - - return $true -} - -function Download-File { - param( - [string]$Url, - [string]$OutFile - ) - try { - Write-ColorMessage "Downloading from: $Url" "Yellow" - Write-VerboseMessage "Destination: $OutFile" - - # Configure SSL/TLS - [System.Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 -bor [Net.SecurityProtocolType]::Tls11 -bor [Net.SecurityProtocolType]::Tls - [System.Net.ServicePointManager]::ServerCertificateValidationCallback = {$true} - - $webClient = New-Object System.Net.WebClient - $webClient.Headers.Add("User-Agent", "PowerShell MeshAgent Installer") - - try { - $webClient.DownloadFile($Url, $OutFile) - } - catch { - Write-VerboseMessage "First download attempt failed, retrying with different method..." - Invoke-WebRequest -Uri $Url -OutFile $OutFile -SkipCertificateCheck - } - - if (Test-Path $OutFile) { - $fileSize = (Get-Item $OutFile).Length - Write-VerboseMessage "Download completed. File size: $([Math]::Round($fileSize/1KB, 2)) KB" - return $true - } - return $false - } - catch { - Write-ColorMessage "Download failed: $($_.Exception.Message)" "Red" - Write-VerboseMessage "Full error: $($_.Exception)" - return $false - } -} - -function Uninstall-MeshAgent { - param( - [string]$InstallDir - ) - - Write-ColorMessage "`nUninstalling MeshCentral Agent" "Yellow" - - # Stop any running instances - Stop-MeshAgent - - # Try to run agent's uninstall method if available - $agentPath = Join-Path $InstallDir "meshagent.exe" - if (Test-Path $agentPath) { - Write-VerboseMessage "Running agent's uninstall command..." - try { - Start-Process -FilePath $agentPath -ArgumentList "uninstall" -Wait -NoNewWindow - Start-Sleep -Seconds 2 - } - catch { - Write-VerboseMessage "Error running uninstall command: $($_.Exception.Message)" - } - } - - # Ensure process is stopped - Stop-MeshAgent - - # Remove entire directory without preserving any files - Write-VerboseMessage "Removing installation directory..." - Remove-Directory -Path $InstallDir - - # Clean up registry entries - $registryPaths = @( - "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\MeshAgent", - "HKLM:\SOFTWARE\WOW6432Node\Microsoft\Windows\CurrentVersion\Uninstall\MeshAgent" - ) - - foreach ($path in $registryPaths) { - if (Test-Path $path) { - Write-VerboseMessage "Removing registry key: $path" - Remove-Item -Path $path -Force -ErrorAction SilentlyContinue - } - } - - # Remove scheduled tasks if any - $taskName = "MeshAgent" - $taskExists = Get-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue - if ($taskExists) { - Write-VerboseMessage "Removing scheduled task: $taskName" - Unregister-ScheduledTask -TaskName $taskName -Confirm:$false -ErrorAction SilentlyContinue - } - - Write-ColorMessage "MeshCentral Agent has been uninstalled." "Green" - exit 0 -} - -# Show help if requested or if no parameters provided -if ($Help -or [string]::IsNullOrEmpty($Server)) { - Show-Help -} - -# Check for Administrator privileges -if (-not (Test-Administrator)) { - Write-ColorMessage "Error: Please run this script as Administrator." "Red" - exit 1 -} - -# Set up paths -$TempDir = Join-Path $env:TEMP "mesh_install" -$BackupDir = Join-Path $env:TEMP "mesh_backup" -$LogDir = Join-Path $env:ProgramData "MeshAgent\Logs" -$InstallDir = Join-Path $env:ProgramFiles "MeshAgent" -$DataDir = Join-Path $env:ProgramData "MeshAgent" - -# Define identity files and directories to preserve during installation -$IdentityFilesToPreserve = @( - "mesh.db", # Main database file - "meshagent.msh", # Configuration file - "meshagent.db", # Agent database - "settings.json", # Agent settings - "state.json", # Agent state - "nodeinfo.json", # Node information - "identitydata.json" # Identity data -) - -$IdentityDirsToPreserve = @( - "data", # Data directory - "db", # Database directory - "config" # Configuration directory -) - -# Process uninstall request if specified -if ($Uninstall) { - Uninstall-MeshAgent -InstallDir $InstallDir - exit 0 -} - -try { - Write-ColorMessage "`nMeshCentral Agent Installation Started" "Green" - Write-ColorMessage "======================================" "Green" - - # Stop any running instances first - Stop-MeshAgent - - # Check for existing installation and backup identity files - $hasExistingInstallation = Test-Path $InstallDir - $hasIdentityBackup = $false - - if ($hasExistingInstallation) { - Write-ColorMessage "Existing installation found. Preserving identity files..." "Yellow" - - # If ForceNewCert is specified, modify the identity files list to exclude certificate-related files - if ($ForceNewCert) { - Write-ColorMessage " ● Certificate reset requested - will not preserve certificate data" "Yellow" - - # Modified list that excludes certificate-related files - $IdentityFilesToPreserve = @( - # Keep only non-certificate related files - "nodeinfo.json" # Node information - ) - - $IdentityDirsToPreserve = @( - # No directories to preserve when forcing cert reset - ) - } - - $hasIdentityBackup = Backup-IdentityFiles -SourceDir $InstallDir -BackupDir $BackupDir - - if ($hasIdentityBackup) { - if ($ForceNewCert) { - Write-ColorMessage " ● Successfully backed up minimal identity files (certificate reset mode)" "Green" - } else { - Write-ColorMessage " ● Successfully backed up identity files" "Green" - } - } else { - Write-ColorMessage " ● No identity files found to backup" "Yellow" - } - } else { - Write-VerboseMessage "No existing installation found. Will perform fresh install." - } - - Write-VerboseMessage "Temporary directory: $TempDir" - Write-VerboseMessage "Backup directory: $BackupDir" - Write-VerboseMessage "Log directory: $LogDir" - Write-VerboseMessage "Installation directory: $InstallDir" - Write-VerboseMessage "Data directory: $DataDir" - - # Display file destinations for user clarity - Write-ColorMessage "File Destinations:" "Blue" - Write-ColorMessage " ● Temporary directory: $TempDir" "Yellow" - Write-ColorMessage " ● Log directory: $LogDir" "Yellow" - Write-ColorMessage " ● Installation directory: $InstallDir" "Yellow" - - # Clean up existing temporary directory - Remove-Directory $TempDir - - # Selectively clean installation directory, preserving identity files - if ($hasExistingInstallation) { - Write-VerboseMessage "Selectively cleaning installation directory while preserving identity files..." - Remove-Directory -Path $InstallDir -PreserveIdentityFiles - } - - # Create temporary directory - Write-VerboseMessage "Creating temporary directory..." - New-Item -ItemType Directory -Path $TempDir -Force | Out-Null - - # Detect architecture and set agent ID - $archInfo = Get-AgentArchitecture - Write-ColorMessage "System Information:" "Yellow" - Write-VerboseMessage "Architecture: $($archInfo.Arch)" - Write-VerboseMessage "Agent ID: $($archInfo.AgentId)" - Write-VerboseMessage "Windows Version: $([System.Environment]::OSVersion.Version)" - - # Test server connection first - if (-not (Test-ServerConnection -ServerUrl $Server)) { - throw "Unable to connect to MeshCentral server at https://$Server" - } - - # Configure SSL/TLS - Write-VerboseMessage "Configuring SSL/TLS settings..." - [System.Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 -bor [Net.SecurityProtocolType]::Tls11 -bor [Net.SecurityProtocolType]::Tls - [System.Net.ServicePointManager]::ServerCertificateValidationCallback = { $true } - - # Disable progress bar for faster downloads - $ProgressPreference = 'SilentlyContinue' - - # Download files - Write-ColorMessage "Downloading MeshAgent Files:" "Yellow" - - # Download agent - $agentUrl = "https://$Server/meshagents?id=$($archInfo.AgentId)" - $agentPath = Join-Path $TempDir "meshagent.exe" - Write-ColorMessage " ● Agent binary location: $agentPath" "Yellow" - if (-not (Download-File -Url $agentUrl -OutFile $agentPath)) { - throw "Failed to download MeshAgent binary" - } - - # Download config - $configUrl = "https://$Server/openframe_public/meshagent.msh" - $configPath = Join-Path $TempDir "meshagent.msh" - Write-ColorMessage " ● Config file location: $configPath" "Yellow" - if (-not (Download-File -Url $configUrl -OutFile $configPath)) { - throw "Failed to download MeshAgent configuration" - } - - # Add NodeID to the MSH file if provided - if (-not [string]::IsNullOrEmpty($NodeId)) { - Write-VerboseMessage "Adding NodeID to the MSH file: $NodeId" - Add-Content -Path $configPath -Value "NodeID=$NodeId" - Write-ColorMessage " ● Added NodeID to configuration file" "Yellow" - } - - # Verify downloads - Write-ColorMessage "Verifying downloaded files:" "Yellow" - if (-not (Test-Path $agentPath)) { - throw "MeshAgent binary was not downloaded successfully." - } - Write-VerboseMessage "Agent binary verified: $agentPath" - - if (-not (Test-Path $configPath)) { - throw "MeshAgent configuration was not downloaded successfully." - } - Write-VerboseMessage "Configuration file verified: $configPath" - - Write-ColorMessage "All files downloaded successfully." "Green" - - # Create directories - Write-ColorMessage "Setting up directories:" "Yellow" - - # Create log directory - if (-not (Test-Path $LogDir)) { - Write-VerboseMessage "Creating log directory: $LogDir" - New-Item -ItemType Directory -Path $LogDir -Force | Out-Null - } - - # Create install directory - if (-not (Test-Path $InstallDir)) { - Write-VerboseMessage "Creating installation directory: $InstallDir" - New-Item -ItemType Directory -Path $InstallDir -Force | Out-Null - } - - # Create data directory if needed - if (-not (Test-Path $DataDir)) { - Write-VerboseMessage "Creating data directory: $DataDir" - New-Item -ItemType Directory -Path $DataDir -Force | Out-Null - } - - # Copy files to install directory - Write-VerboseMessage "Copying files to installation directory..." - Copy-Item -Path $agentPath -Destination $InstallDir -Force - - # Always override the MSH configuration file - Copy-Item -Path $configPath -Destination $InstallDir -Force - Write-VerboseMessage "Copied new configuration file to installation directory" - - $finalAgentPath = Join-Path $InstallDir "meshagent.exe" - $finalConfigPath = Join-Path $InstallDir "meshagent.msh" - Write-ColorMessage " ● Final agent location: $finalAgentPath" "Yellow" - Write-ColorMessage " ● Final config location: $finalConfigPath" "Yellow" - - # Restore identity files if we backed them up - if ($hasIdentityBackup) { - Write-ColorMessage "Restoring identity files from backup..." "Yellow" - Restore-IdentityFiles -BackupDir $BackupDir -TargetDir $InstallDir - } - - # Clean up temp files before starting agent - Write-VerboseMessage "Cleaning up temporary directory: $TempDir" - Remove-Item -Path $TempDir -Recurse -Force -ErrorAction SilentlyContinue - - # Clean up backup directory after successful restore - if ($hasIdentityBackup) { - Write-VerboseMessage "Cleaning up backup directory: $BackupDir" - Remove-Item -Path $BackupDir -Recurse -Force -ErrorAction SilentlyContinue - } - - # Run agent - Write-ColorMessage "Starting MeshAgent:" "Yellow" - Write-VerboseMessage "Executing: $finalAgentPath connect" - Write-ColorMessage " ● Executing agent from: $finalAgentPath" "Yellow" - - Write-ColorMessage "`nInstallation Summary:" "Green" - Write-ColorMessage " ● Agent Location: $finalAgentPath" "Blue" - Write-ColorMessage " ● Config Location: $finalConfigPath" "Blue" - Write-ColorMessage " ● Log Location: $LogDir" "Blue" - if ($hasIdentityBackup) { - Write-ColorMessage " ● Identity files were preserved from previous installation" "Blue" - } - Write-ColorMessage "Installation completed successfully." "Green" - Write-ColorMessage "`nStarting MeshAgent in connect mode..." "Yellow" - Write-ColorMessage "Press Ctrl+C to exit (agent will continue running in background)" "Yellow" - - # Start the agent in the foreground - try { - & $finalAgentPath connect - } - catch { - Write-ColorMessage "Agent started in background mode" "Green" - } -} -catch { - Write-ColorMessage "`nInstallation Failed:" "Red" - Write-ColorMessage "Error: $($_.Exception.Message)" "Red" - Write-ColorMessage "Stack Trace: $($_.Exception.StackTrace)" "Red" - exit 1 -} \ No newline at end of file diff --git a/clients/openframe-client/infrastructure/tactical-rmm/mac_arm64.sh b/clients/openframe-client/infrastructure/tactical-rmm/mac_arm64.sh deleted file mode 100755 index 4051c6740..000000000 --- a/clients/openframe-client/infrastructure/tactical-rmm/mac_arm64.sh +++ /dev/null @@ -1,537 +0,0 @@ -#!/usr/bin/env bash -# -# mac_arm64.sh -# -# Purpose: -# - Install dependencies (Xcode CLT, Homebrew, Git, Go) on Apple Silicon macOS -# - Accept script args or prompt for org name, email, RMM URL, agent key, code-sign identity, log path, build folder -# - Clone, patch, compile rmmagent for macOS ARM64, optionally sign -# - Prompt to run agent or skip -# - **After install, automatically patch the LaunchDaemons plists** so that the -# Tactical Agent uses the custom log path (if provided). -# -# Usage Examples: -# 1) Interactive mode: -# ./mac_arm64.sh -# 2) Provide some or all args: -# ./mac_arm64.sh --org-name "OpenFrame" --rmm-url "http://localhost:8000" ... -# 3) Non-interactive (all args): -# ./mac_arm64.sh --org-name "MyOrg" ... --skip-run -# -# Requirements: -# - Apple Silicon macOS -# - Possibly root/sudo acceptance for installing Xcode Tools, Homebrew, Git, Go -# - Code-signing optional (needs Developer ID certificate) -# - -set -e - -############################ -# Default / Config -############################ - -RMMAGENT_REPO="https://github.com/amidaware/rmmagent.git" -RMMAGENT_BRANCH="master" -OUTPUT_BINARY="rmmagent-mac-arm64" - -# We'll store user-provided or prompted values in these variables: -ORG_NAME="" -CONTACT_EMAIL="" -RMM_SERVER_URL="" -AGENT_AUTH_KEY="" -CODESIGN_IDENTITY="" -AGENT_LOG_PATH="" -BUILD_FOLDER="rmmagent" # default -SKIP_RUN="false" -CLIENT_ID="" -SITE_ID="" -AGENT_TYPE="workstation" # default -NATS_PORT="" # NATS port (required) - -############################ -# Parse Script Arguments -############################ - -while [[ $# -gt 0 ]]; do - case "$1" in - --org-name) - ORG_NAME="$2" - shift 2 - ;; - --email) - CONTACT_EMAIL="$2" - shift 2 - ;; - --rmm-url) - RMM_SERVER_URL="$2" - shift 2 - ;; - --auth-key) - AGENT_AUTH_KEY="$2" - shift 2 - ;; - --client-id) - CLIENT_ID="$2" - shift 2 - ;; - --site-id) - SITE_ID="$2" - shift 2 - ;; - --agent-type) - AGENT_TYPE="$2" - shift 2 - ;; - --codesign-identity) - CODESIGN_IDENTITY="$2" - shift 2 - ;; - --log-path) - AGENT_LOG_PATH="$2" - shift 2 - ;; - --build-folder) - BUILD_FOLDER="$2" - shift 2 - ;; - --nats-port) - NATS_PORT="$2" - shift 2 - ;; - --skip-run) - SKIP_RUN="true" - shift - ;; - -h|--help) - echo "Usage: $0 [options]" - echo "Options:" - echo " --org-name Organization name placeholder" - echo " --email Contact email placeholder" - echo " --rmm-url RMM server URL" - echo " --auth-key Agent auth key" - echo " --client-id Client ID" - echo " --site-id Site ID" - echo " --agent-type Agent type (server/workstation) [default: server]" - echo " --codesign-identity Apple Developer ID for signing" - echo " --log-path Agent log file path" - echo " --build-folder Where to clone and compile (default: rmmagent)" - echo " --nats-port NATS WebSocket port (required)" - echo " --skip-run Skip final 'run agent' step" - echo "" - echo "Any missing fields are prompted interactively." - exit 0 - ;; - *) - echo "Unknown argument: $1" - exit 1 - ;; - esac -done - -############################ -# Install Dependencies -############################ - -function install_command_line_tools() { - echo "Checking Xcode Command Line Tools..." - if xcode-select -p &>/dev/null; then - echo "Xcode Command Line Tools appear installed." - else - echo "Installing Xcode Command Line Tools..." - xcode-select --install || true - echo "Please accept the GUI prompt if shown. Then re-run if needed." - sleep 2 - fi -} - -function install_homebrew_if_needed() { - echo "Checking Homebrew..." - if ! command -v brew &>/dev/null; then - echo "Installing Homebrew..." - /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - if [ -d "/opt/homebrew/bin" ]; then - export PATH="/opt/homebrew/bin:$PATH" - fi - else - echo "Homebrew found." - fi -} - -function install_git_if_needed() { - echo "Checking Git..." - if ! command -v git &>/dev/null; then - echo "Installing Git via Homebrew..." - brew install git - else - echo "Git found." - fi -} - -function install_go_if_needed() { - echo "Checking Go..." - if ! command -v go &>/dev/null; then - echo "Installing Go via Homebrew..." - brew install go - else - echo "Go found." - fi -} - -############################ -# Patching NATS WebSocket URL to use ws:// for local development -############################ - -function patch_nats_websocket_url() { - echo "Patching agent.go to use ws:// for NATS WebSocket..." - - # Print current working directory and list contents for debugging - echo "Current working directory: $(pwd)" - - # Find the agent.go file - use correct path - local agent_go_file="agent/agent.go" - - echo "Checking for agent.go at path: $agent_go_file" - if [ ! -f "$agent_go_file" ]; then - echo "ERROR: Cannot find $agent_go_file. Skipping NATS WebSocket URL patch." - # Try to find agent.go using find command - echo "Attempting to locate agent.go using find command:" - find . -name "agent.go" | grep -v test - return 1 - fi - - # Create a backup - cp "$agent_go_file" "$agent_go_file.bak" - - # Replace the wss:// with ws:// in the NATS WebSocket URL construction and use configured port - # This modifies the line: natsServer = fmt.Sprintf("wss://%s:%s", ac.APIURL, natsProxyPort) - sed -i '' "s/natsServer = fmt.Sprintf(\"wss:\/\/%s:%s\", ac.APIURL, natsProxyPort)/natsServer = fmt.Sprintf(\"ws:\/\/%s:$NATS_PORT\/natsws\", ac.APIURL)/g" "$agent_go_file" - - # Also modify the URL construction when NatsStandardPort is set to use configured port - sed -i '' "s/natsServer = fmt.Sprintf(\"nats:\/\/%s:%s\", ac.APIURL, ac.NatsStandardPort)/natsServer = fmt.Sprintf(\"ws:\/\/%s:$NATS_PORT\/natsws\", ac.APIURL)/g" "$agent_go_file" - - echo "NATS WebSocket URL patch applied to $agent_go_file with port $NATS_PORT" - - # Show the diff to verify changes - echo "Showing diff of changes:" - diff "$agent_go_file.bak" "$agent_go_file" || true -} - -############################ -# Aggressive Uninstallation -############################ - -function aggressive_uninstall() { - echo "" - echo "=== Performing Aggressive Uninstallation ===" - echo "This will remove all components of the Tactical RMM agent..." - - # 1. Stop and unload all services - echo "Stopping and unloading services..." - sudo launchctl unload /Library/LaunchDaemons/tacticalagent.plist 2>/dev/null || true - - # 2. Remove LaunchDaemons - echo "Removing LaunchDaemons..." - sudo rm -f /Library/LaunchDaemons/tacticalagent.plist - sudo rm -f /Library/LaunchDaemons/tacticalagent.plist.bak - - # 3. Remove Tactical Agent files and directories - echo "Removing Tactical Agent files..." - sudo rm -rf /opt/tacticalagent/ - - # 4. Clean up any logs - echo "Cleaning up logs..." - sudo rm -f /var/log/tacticalagent.log - - # 5. Additional cleanup for any other remnants - echo "Performing additional cleanup..." - # Search for and remove any other files containing 'tactical' in common locations - sudo find /opt -name "*tactical*" -exec rm -rf {} \; 2>/dev/null || true - - echo "Aggressive uninstallation completed. System is ready for fresh installation." - echo "" -} - -############################ -# Prompting for missing inputs -############################ - -function prompt_if_empty() { - local varname="$1" - local prompt_msg="$2" - local default_val="$3" - - local curr_val="${!varname}" - - if [ -z "$curr_val" ]; then - if [ -n "$default_val" ]; then - read -rp "$prompt_msg [$default_val]: " user_inp - user_inp="${user_inp:-$default_val}" - else - read -rp "$prompt_msg: " user_inp - fi - eval "$varname=\"\$user_inp\"" - fi -} - -############################ -# Cloning/Patching/Building -############################ - -function handle_existing_folder() { - # If BUILD_FOLDER already exists, check if it's a Git repo - # If yes, do a fetch/pull - # If no, prompt to remove or rename - if [ -d "$BUILD_FOLDER" ]; then - echo "Folder '$BUILD_FOLDER' already exists." - cd "$BUILD_FOLDER" - if [ -d ".git" ]; then - echo "It appears to be a valid Git repository. Pulling latest changes..." - git fetch --all - git checkout "$RMMAGENT_BRANCH" - git pull - else - echo "But it isn't a Git repo (no .git folder)." - echo "We can either remove it or rename it so we can clone fresh." - read -rp "Remove folder? (y/N): " REMOVE_CHOICE - if [[ "$REMOVE_CHOICE" =~ ^[Yy] ]]; then - cd .. - rm -rf "$BUILD_FOLDER" - echo "Removed folder. Now cloning fresh..." - git clone --branch "$RMMAGENT_BRANCH" "$RMMAGENT_REPO" "$BUILD_FOLDER" - cd "$BUILD_FOLDER" - else - echo "Aborting script. Please specify a different --build-folder or remove the folder manually." - exit 1 - fi - fi - else - echo "Cloning $RMMAGENT_REPO into '$BUILD_FOLDER'..." - git clone --branch "$RMMAGENT_BRANCH" "$RMMAGENT_REPO" "$BUILD_FOLDER" - cd "$BUILD_FOLDER" - fi -} - -function patch_placeholders() { - echo "" - echo "Patching code for org/email placeholders (if present)." - if grep -q 'DefaultOrgName' *.go 2>/dev/null; then - sed -i.bak "s|DefaultOrgName = \".*\"|DefaultOrgName = \"$ORG_NAME\"|" *.go - fi - if grep -q 'DefaultEmail' *.go 2>/dev/null; then - sed -i.bak "s|DefaultEmail = \".*\"|DefaultEmail = \"$CONTACT_EMAIL\"|" *.go - fi -} - -function compile_rmmagent() { - echo "" - echo "Compiling rmmagent for macOS ARM64..." - env CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 \ - go build -ldflags "-s -w" -o "$OUTPUT_BINARY" - - echo "Compilation done. Output: $(pwd)/$OUTPUT_BINARY" - file "$OUTPUT_BINARY" -} - -function sign_binary_if_requested() { - if [ -n "$CODESIGN_IDENTITY" ]; then - echo "" - echo "Signing with identity: $CODESIGN_IDENTITY" - xattr -d com.apple.quarantine ./"$OUTPUT_BINARY" 2>/dev/null || true - codesign --deep --force --options runtime \ - --sign "$CODESIGN_IDENTITY" \ - ./"$OUTPUT_BINARY" - echo "Code signing done. Checking signature..." - codesign -dv --verbose=4 ./"$OUTPUT_BINARY" || true - else - echo "No code-sign identity provided. Skipping signing." - fi -} - -############################ -# Patching the plists to include -log -############################ - -function patch_agent_plists_with_log() { - echo "" - echo "Attempting to patch LaunchDaemons for detailed logging configurations" - echo "This requires sudo privileges." - - local TACTICAL_PLIST="/Library/LaunchDaemons/tacticalagent.plist" - - # Default to a standard log path if none provided - if [ -z "$AGENT_LOG_PATH" ]; then - AGENT_LOG_PATH="/var/log/tacticalagent.log" - echo "No custom log path specified, using default: $AGENT_LOG_PATH" - fi - - # TacticalAgent plist modifications - if [ -f "$TACTICAL_PLIST" ]; then - echo "Backing up and patching tacticalagent.plist with enhanced logging..." - # Create backup - sudo cp "$TACTICAL_PLIST" "${TACTICAL_PLIST}.bak" - - # Check if ProgramArguments already has debug logging entries - if sudo /usr/libexec/PlistBuddy -c "Print :ProgramArguments" "$TACTICAL_PLIST" | grep -q -- "-log"; then - echo "Logging parameters already exist, updating values..." - # Update existing values - sudo /usr/libexec/PlistBuddy -c "Delete :ProgramArguments:3" "$TACTICAL_PLIST" 2>/dev/null || true - sudo /usr/libexec/PlistBuddy -c "Delete :ProgramArguments:3" "$TACTICAL_PLIST" 2>/dev/null || true - sudo /usr/libexec/PlistBuddy -c "Delete :ProgramArguments:3" "$TACTICAL_PLIST" 2>/dev/null || true - sudo /usr/libexec/PlistBuddy -c "Delete :ProgramArguments:3" "$TACTICAL_PLIST" 2>/dev/null || true - fi - - # Add logging parameters - sudo /usr/libexec/PlistBuddy -c "Add :ProgramArguments:3 string '-log'" "$TACTICAL_PLIST" 2>/dev/null || true - sudo /usr/libexec/PlistBuddy -c "Add :ProgramArguments:4 string 'DEBUG'" "$TACTICAL_PLIST" 2>/dev/null || true - sudo /usr/libexec/PlistBuddy -c "Add :ProgramArguments:5 string '-logto'" "$TACTICAL_PLIST" 2>/dev/null || true - sudo /usr/libexec/PlistBuddy -c "Add :ProgramArguments:6 string '$AGENT_LOG_PATH'" "$TACTICAL_PLIST" 2>/dev/null || true - - echo "Reloading LaunchDaemon for tacticalagent..." - sudo launchctl unload "$TACTICAL_PLIST" 2>/dev/null || true - sudo launchctl load "$TACTICAL_PLIST" 2>/dev/null || true - echo "TacticalAgent logging configured to use: $AGENT_LOG_PATH" - else - echo "Warning: $TACTICAL_PLIST not found. TacticalAgent may not be installed yet." - fi -} - -############################ -# Prompt to run -############################ - -function prompt_run_agent() { - echo "" - echo "=== Build Complete ===" - echo "You can run the agent with your RMM server & auth key. For example:" - echo " ./$OUTPUT_BINARY -m install \\" - echo " -api \"$RMM_SERVER_URL\" \\" - echo " -auth \"$AGENT_AUTH_KEY\" \\" - echo " -client-id -site-id -agent-type \\" - echo " -log \"DEBUG\" -logto \"$AGENT_LOG_PATH\"" - echo "" - - if [ "$SKIP_RUN" == "true" ]; then - echo "Skipping final run (--skip-run)." - return - fi - - # If all required parameters are provided, run automatically - if [ -n "$RMM_SERVER_URL" ] && [ -n "$AGENT_AUTH_KEY" ] && [ -n "$CLIENT_ID" ] && [ -n "$SITE_ID" ]; then - echo "All required parameters provided, proceeding with installation..." - RUN_NOW="y" - else - read -rp "Do you want to run the agent install command now? (y/N): " RUN_NOW - fi - - if [[ "$RUN_NOW" =~ ^[Yy] ]]; then - # Only prompt for values if they weren't provided as arguments - if [ -z "$CLIENT_ID" ]; then - read -rp "Enter client-id: " CLIENT_ID - fi - if [ -z "$SITE_ID" ]; then - read -rp "Enter site-id: " SITE_ID - fi - if [ -z "$AGENT_TYPE" ]; then - read -rp "Agent type (server/workstation) [server]: " AGENT_TYPE - AGENT_TYPE=${AGENT_TYPE:-server} - fi - - # If no log path was specified, create a default one with timestamp - if [ -z "$AGENT_LOG_PATH" ]; then - AGENT_LOG_PATH="/var/log/tacticalagent.log" - echo "Using default log path: $AGENT_LOG_PATH" - fi - - local CMD="sudo ./$OUTPUT_BINARY -m install -api \"$RMM_SERVER_URL\" -auth \"$AGENT_AUTH_KEY\" -client-id \"$CLIENT_ID\" -site-id \"$SITE_ID\" -agent-type \"$AGENT_TYPE\" -log \"DEBUG\" -logto \"$AGENT_LOG_PATH\" -nomesh" - - echo "Running: $CMD" - eval "$CMD" - - echo "" - echo "Agent started with maximum verbosity! Logs will be written to: $AGENT_LOG_PATH" - echo "To monitor the log in real-time, run: sudo tail -f $AGENT_LOG_PATH" - - # After successful install, patch plists with the custom log path - patch_agent_plists_with_log - - echo "" - echo "You can monitor the agent logs with this command:" - echo " sudo tail -f $AGENT_LOG_PATH # For tactical agent" - fi - - echo "" - echo "=== All Done! ===" - echo "Your agent is at: $(pwd)/$OUTPUT_BINARY" - echo "Consider notarizing if distributing externally." -} - -############################ -# Main Script Flow -############################ - -# 1) Install dependencies -echo "Checking and installing dependencies if needed..." -install_command_line_tools -install_homebrew_if_needed -install_git_if_needed -install_go_if_needed - -# Perform aggressive uninstallation before proceeding -aggressive_uninstall - -# 2) Prompt for missing fields -echo "" -echo "=== Checking user inputs ===" - -function prompt_all_inputs() { - prompt_if_empty "RMM_SERVER_URL" "RMM Server URL (e.g. https://rmm.myorg.com)" - prompt_if_empty "AGENT_AUTH_KEY" "Agent Auth Key (string from your RMM)" - prompt_if_empty "CLIENT_ID" "Client ID" - prompt_if_empty "SITE_ID" "Site ID" - prompt_if_empty "AGENT_TYPE" "Agent type (server/workstation) [server]" "server" - prompt_if_empty "NATS_PORT" "NATS WebSocket port (required)" - # Only prompt for log path if explicitly requested - if [ -n "$AGENT_LOG_PATH" ]; then - prompt_if_empty "AGENT_LOG_PATH" "Agent log path" - fi - # Only prompt for codesign if explicitly requested - if [ -n "$CODESIGN_IDENTITY" ]; then - prompt_if_empty "CODESIGN_IDENTITY" "Code-sign Identity (Developer ID ...)" - fi - prompt_if_empty "BUILD_FOLDER" "Destination build folder" "rmmagent" -} - -prompt_all_inputs - -# Only show final values and proceed prompt if we're missing required parameters -if [ -z "$RMM_SERVER_URL" ] || [ -z "$AGENT_AUTH_KEY" ] || [ -z "$CLIENT_ID" ] || [ -z "$SITE_ID" ] || [ -z "$NATS_PORT" ]; then - echo "" - echo "== Final values ==" - # Only display values that are actually set - [ -n "$RMM_SERVER_URL" ] && echo " RMM URL : $RMM_SERVER_URL" - [ -n "$AGENT_AUTH_KEY" ] && echo " Auth Key : $AGENT_AUTH_KEY" - [ -n "$CLIENT_ID" ] && echo " Client ID : $CLIENT_ID" - [ -n "$SITE_ID" ] && echo " Site ID : $SITE_ID" - [ -n "$AGENT_TYPE" ] && echo " Agent Type : $AGENT_TYPE" - [ -n "$NATS_PORT" ] && echo " NATS Port : $NATS_PORT" - [ -n "$AGENT_LOG_PATH" ] && echo " Log Path : $AGENT_LOG_PATH" - [ -n "$CODESIGN_IDENTITY" ] && echo " CodeSign ID : $CODESIGN_IDENTITY" - [ -n "$BUILD_FOLDER" ] && echo " Build Folder : $BUILD_FOLDER" - [ -n "$SKIP_RUN" ] && echo " skip-run : $SKIP_RUN" - echo "" - - # Only show the proceed prompt if we're not in skip-run mode - if [ "$SKIP_RUN" != "true" ]; then - read -rp "Press Enter to proceed, or Ctrl+C to cancel..." - fi -fi - -# 3) Clone & patch & build -handle_existing_folder -patch_nats_websocket_url -patch_placeholders -compile_rmmagent -sign_binary_if_requested - -# 4) Prompt to run (and patch plists if installed) -prompt_run_agent \ No newline at end of file diff --git a/clients/openframe-client/infrastructure/tactical-rmm/win_amd64.ps1 b/clients/openframe-client/infrastructure/tactical-rmm/win_amd64.ps1 deleted file mode 100644 index 724b8e316..000000000 --- a/clients/openframe-client/infrastructure/tactical-rmm/win_amd64.ps1 +++ /dev/null @@ -1,1323 +0,0 @@ -# -# windows_amd64.ps1 -# -# Purpose: -# - Install Tactical RMM agent on Windows AMD64 -# - Uses native AMD64 binary -# - Simple flow: check if installed, uninstall if yes, install from binary -# - Automatically configures the agent to use ws:// protocol instead of wss:// for WebSockets -# -# Usage Examples: -# 1) Interactive mode: -# .\windows_amd64.ps1 -# 2) Provide all args: -# .\windows_amd64.ps1 -RmmHost "rmm.example.com" -RmmPort 8000 -Secure -AuthKey "your-key" -ClientId "1" -SiteId "1" -AgentType "server" -# -# Requirements: -# - Windows AMD64 -# - PowerShell 5.1 or higher -# - Administrator privileges for installing dependencies and services -# - -# Windows AMD64 Tactical RMM Agent Installer -# Requires -RunAsAdministrator - -[CmdletBinding()] -param( - [Parameter(Mandatory=$false)] - [string]$OrgName = "", - - [Parameter(Mandatory=$false)] - [string]$ContactEmail = "", - - [Parameter(Mandatory=$false)] - [string]$RmmServerUrl = "", - - [Parameter(Mandatory=$false)] - [string]$AuthKey = "", - - [Parameter(Mandatory=$false)] - [string]$ClientId = "", - - [Parameter(Mandatory=$false)] - [string]$SiteId = "", - - [Parameter(Mandatory=$false)] - [string]$AgentType = "workstation", - - [Parameter(Mandatory=$false)] - [string]$BuildFolder = "rmmagent", - - [Parameter(Mandatory=$false)] - [string]$NatsPort = "", - - [Parameter(Mandatory=$false)] - [switch]$SkipRun, - - [Parameter(Mandatory=$false)] - [switch]$Help -) - -function Write-ColorMessage { - param( - [string]$Message, - [string]$Color, - [switch]$NoNewLine - ) - switch ($Color) { - "Green" { $colorParam = "Green" } - "Red" { $colorParam = "Red" } - "Yellow" { $colorParam = "Yellow" } - "Blue" { $colorParam = "Blue" } - default { $colorParam = "White" } - } - if ($NoNewLine) { - Write-Host $Message -ForegroundColor $colorParam -NoNewline - } else { - Write-Host $Message -ForegroundColor $colorParam - } -} - -# Ensure script is running with administrator privileges -if (-not ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) { - Write-Host "This script requires administrator privileges. Please restart as administrator." -ForegroundColor Red - exit 1 -} - -############################ -# Functions -############################ - -function Get-SystemArchitecture { - Write-Host "Detecting system architecture..." -ForegroundColor Yellow - $architecture = (Get-WmiObject Win32_OperatingSystem).OSArchitecture - Write-Host "System architecture: $architecture" -ForegroundColor Green - - # Handle different architecture string formats - switch ($architecture) { - "64-bit" { return "64-bit" } - "ARM64" { return "ARM64" } - "ARM 64-bit Processor" { return "ARM64" } - default { - Write-Host "Unsupported architecture: $architecture" -ForegroundColor Red - exit 1 - } - } -} - -function Install-Software { - param ( - [string]$SoftwareName, - [string]$CommandName, - [string]$VersionCommand, - [string]$InstallPath, - [string]$PathToAdd, - [string]$DownloadUrl, - [string]$InstallerArgs, - [string]$InstallerType - ) - - Write-Host "=============== Installing $SoftwareName ===============" -ForegroundColor Cyan - Write-Host "Download URL: $DownloadUrl" - Write-Host "Installer Type: $InstallerType" - Write-Host "Install Path: $InstallPath" - Write-Host "Installer Args: $InstallerArgs" - - # Check if already installed - Write-Host "Checking $SoftwareName installation..." - try { - $version = & $CommandName $VersionCommand - if ($LASTEXITCODE -eq 0) { - Write-Host "$SoftwareName is already installed: $version" -ForegroundColor Green - return $true - } - } catch { - Write-Host "$SoftwareName is not installed or not in PATH" -ForegroundColor Yellow - } - - # Download installer - $installerName = $SoftwareName.ToLower().Replace(" ", "") - $installer = Join-Path $env:TEMP "${installerName}_installer.$InstallerType" - $logFile = Join-Path $env:TEMP "${installerName}_install.log" - - Write-Host "Downloading installer to: $installer" - Write-Host "Installation log will be saved to: $logFile" - - try { - Invoke-WebRequest -Uri $DownloadUrl -OutFile $installer - if (-not (Test-Path $installer)) { - throw "Failed to download installer" - } - } catch { - Write-Host "Failed to download installer: $_" -ForegroundColor Red - return $false - } - - # Install software - Write-Host "Installing $SoftwareName..." - try { - if ($InstallerType -eq "msi") { - $msiArgs = @( - '/i', - "`"$installer`"", - '/quiet', - '/l*v', - "`"$logFile`"" - ) - $process = Start-Process msiexec.exe -ArgumentList $msiArgs -Wait -PassThru -Verb RunAs - if ($process.ExitCode -ne 0) { - Write-Host "Installation failed with exit code: $($process.ExitCode)" -ForegroundColor Red - Write-Host "Installation log from ${logFile}:" -ForegroundColor Yellow - if (Test-Path $logFile) { - Get-Content $logFile -Tail 10 - } - return $false - } - } else { - $process = Start-Process $installer -ArgumentList $InstallerArgs -Wait -PassThru -Verb RunAs - if ($process.ExitCode -ne 0) { - Write-Host "Installation failed with exit code: $($process.ExitCode)" -ForegroundColor Red - return $false - } - } - - # Update PATH if needed - if ($PathToAdd) { - $currentPath = [System.Environment]::GetEnvironmentVariable("Path", "Machine") - if ($currentPath -notlike "*$PathToAdd*") { - Write-Host "Adding $PathToAdd to system PATH..." - [System.Environment]::SetEnvironmentVariable("Path", $currentPath + ";$PathToAdd", "Machine") - } - } - - # Update current session's PATH - $env:Path = [System.Environment]::GetEnvironmentVariable("Path", "Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path", "User") - - # Verify installation - Write-Host "Verifying installation..." - Start-Sleep -Seconds 2 # Give the system time to update PATH - try { - $version = & $CommandName $VersionCommand - if ($LASTEXITCODE -eq 0) { - Write-Host "$SoftwareName installed successfully: $version" -ForegroundColor Green - return $true - } else { - Write-Host "Installation verification failed" -ForegroundColor Red - return $false - } - } catch { - Write-Host "Installation verification failed: $_" -ForegroundColor Red - return $false - } - } finally { - # Cleanup - if (Test-Path $installer) { - Remove-Item $installer -Force - } - } -} - -function Install-Go { - $architecture = Get-SystemArchitecture - - # Select appropriate Go installer based on architecture - $goUrl = switch ($architecture) { - "64-bit" { "https://golang.org/dl/go1.21.6.windows-amd64.msi" } - "ARM64" { "https://golang.org/dl/go1.21.6.windows-arm64.msi" } - default { - Write-Host "Unsupported architecture: $architecture" -ForegroundColor Red - exit 1 - } - } - - Install-Software -SoftwareName "Go" ` - -CommandName "go" ` - -VersionCommand "version" ` - -InstallPath "C:\Go" ` - -PathToAdd "C:\Go\bin" ` - -DownloadUrl $goUrl ` - -InstallerArgs "/quiet /norestart ADDLOCAL=ALL ALLUSERS=1" ` - -InstallerType "msi" -} - -function Install-Git { - $architecture = Get-SystemArchitecture - - # Select appropriate Git installer based on architecture - $gitUrl = switch ($architecture) { - "64-bit" { "https://github.com/git-for-windows/git/releases/download/v2.43.0.windows.1/Git-2.43.0-64-bit.exe" } - "ARM64" { "https://github.com/git-for-windows/git/releases/download/v2.43.0.windows.1/Git-2.43.0-arm64.exe" } - default { - Write-Host "Unsupported architecture: $architecture" -ForegroundColor Red - exit 1 - } - } - - Install-Software -SoftwareName "Git" ` - -CommandName "git" ` - -VersionCommand "--version" ` - -InstallPath "C:\Program Files\Git" ` - -PathToAdd "" ` - -DownloadUrl $gitUrl ` - -InstallerArgs "/VERYSILENT /NORESTART /SUPPRESSMSGBOXES" ` - -InstallerType "exe" -} - -function Clone-Repository { - param( - [string]$RepoUrl, - [string]$Branch, - [string]$Folder - ) - - if (Test-Path $Folder) { - Write-Host "Folder '$Folder' already exists. Updating..." -ForegroundColor Yellow - Set-Location $Folder - git fetch --all - git checkout $Branch - git pull - } else { - Write-Host "Cloning repository..." -ForegroundColor Yellow - git clone --branch $Branch $RepoUrl $Folder - Set-Location $Folder - } -} - -function Patch-NatsWebsocketUrl { - param( - [string]$RmmUrl, - [string]$NatsPort - ) - - Write-Host "Patching NATS WebSocket URL..." -ForegroundColor Yellow - $agentGoFile = "agent/agent.go" - - if (Test-Path $agentGoFile) { - $content = Get-Content $agentGoFile -Raw - - # Extract host from RMM URL - $uri = [System.Uri]$RmmUrl - $rmmHost = $uri.Host - - # Set the NATS server URL using the RMM server host and provided NATS port - $natsUrl = "ws://${rmmHost}:${NatsPort}/natsws" - - # Pattern 1: WebSocket secure pattern - $wsPattern = 'natsServer = fmt.Sprintf\("wss://%s:%s", ac.APIURL, natsProxyPort\)' - $wsMatches = [regex]::Matches($content, $wsPattern) - if ($wsMatches.Count -gt 0) { - Write-Host "`nFound WebSocket secure pattern in ${agentGoFile}:" -ForegroundColor Yellow - foreach ($match in $wsMatches) { - $lineNumber = [regex]::Matches($content.Substring(0, $match.Index), "`n").Count + 1 - Write-Host "Line $lineNumber - $($match.Value)" -ForegroundColor Yellow - } - } - - # Pattern 2: Standard NATS pattern - $natsPattern = 'natsServer = fmt.Sprintf\("nats://%s:%s", ac.APIURL, ac.NatsStandardPort\)' - $natsMatches = [regex]::Matches($content, $natsPattern) - if ($natsMatches.Count -gt 0) { - Write-Host "`nFound standard NATS pattern in ${agentGoFile}:" -ForegroundColor Yellow - foreach ($match in $natsMatches) { - $lineNumber = [regex]::Matches($content.Substring(0, $match.Index), "`n").Count + 1 - Write-Host "Line $lineNumber - $($match.Value)" -ForegroundColor Yellow - } - } - - # Perform the replacements - $content = $content -replace $wsPattern, "natsServer = `"${natsUrl}`"" - $content = $content -replace $natsPattern, "natsServer = `"${natsUrl}`"" - - Write-Host "`nReplacing with:" -ForegroundColor Yellow - Write-Host "natsServer = `"${natsUrl}`"" -ForegroundColor Green - - Set-Content $agentGoFile $content - - Write-Host "`nNATS WebSocket URL patched successfully with: $natsUrl" -ForegroundColor Green - } else { - Write-Host "Warning: agent.go file not found." -ForegroundColor Red - } -} - -function Compile-Agent { - Write-ColorMessage "Compiling agent..." "Yellow" - $env:GOOS = "windows" - $env:GOARCH = "amd64" - go build -ldflags "-s -w" -o "rmmagent.exe" - - if (Test-Path "rmmagent.exe") { - Write-ColorMessage "Agent compiled successfully." "Green" - } else { - Write-ColorMessage "Error: Agent compilation failed." "Red" - exit 1 - } -} - -function Install-Agent { - param( - [string]$RmmUrl, - [string]$AuthKey, - [string]$ClientId, - [string]$SiteId, - [string]$AgentType - ) - - Write-ColorMessage "Installing agent..." "Yellow" - - # Validate required parameters - if ([string]::IsNullOrEmpty($AuthKey)) { - Write-ColorMessage "Error: AuthKey is required" "Red" - exit 1 - } - - # Get the full path to the agent executable from ProgramData - $agentPath = Join-Path $env:ProgramData "TacticalRMM\rmmagent.exe" - - if (-not (Test-Path $agentPath)) { - Write-ColorMessage "Error: Agent executable not found at: $agentPath" "Red" - exit 1 - } - - Write-ColorMessage "Found agent executable at: $agentPath" "Green" - - # Create Program Files directory if it doesn't exist - $programFilesDir = Join-Path $env:ProgramFiles "TacticalAgent" - if (-not (Test-Path $programFilesDir)) { - Write-ColorMessage "Creating Program Files directory: $programFilesDir" "Yellow" - Write-ColorMessage "Executing command: New-Item -ItemType Directory -Path '$programFilesDir' -Force" "Blue" - New-Item -ItemType Directory -Path $programFilesDir -Force | Out-Null - } - - # Copy binary to Program Files with new name - $targetPath = Join-Path $programFilesDir "tacticalrmm.exe" - Write-ColorMessage "Copying binary to: $targetPath" "Yellow" - Write-ColorMessage "Executing command: Copy-Item -Path '$agentPath' -Destination '$targetPath' -Force" "Blue" - Copy-Item -Path $agentPath -Destination $targetPath -Force - - if (-not (Test-Path $targetPath)) { - Write-ColorMessage "Error: Failed to copy binary to Program Files" "Red" - exit 1 - } - Write-ColorMessage "Successfully copied binary to Program Files" "Green" - - # Build the arguments array to match mac_arm64.sh exactly - $args = @( - "-m", "install", - "-api", "`"$RmmUrl`"", - "-auth", "`"$AuthKey`"", - "-client-id", "`"$ClientId`"", - "-site-id", "`"$SiteId`"", - "-agent-type", "`"$AgentType`"", - "-log", "`"DEBUG`"", - "-nomesh", - "/VERYSILENT", - "/SUPPRESSMSGBOXES", - "-silent" - ) - - Write-ColorMessage "Running agent installation with parameters..." "Yellow" - Write-ColorMessage "RMM URL: $RmmUrl" "Yellow" - Write-Host "Client ID: $ClientId" -ForegroundColor Yellow - Write-Host "Site ID: $SiteId" -ForegroundColor Yellow - Write-Host "Agent Type: $AgentType" -ForegroundColor Yellow - Write-Host "Agent Path: $targetPath" -ForegroundColor Yellow - - # Convert arguments array to a single string - $argsString = $args -join " " - Write-ColorMessage "Full command: $targetPath $argsString" "Blue" - - # Start the process with the full path and arguments string - Write-ColorMessage "Starting agent installation process..." "Yellow" - Write-ColorMessage "Executing command: Start-Process -FilePath '$targetPath' -ArgumentList '$argsString' -Wait -NoNewWindow -PassThru" "Blue" - $process = Start-Process -FilePath $targetPath -ArgumentList $argsString -Wait -NoNewWindow -PassThru - - Write-ColorMessage "Process exit code: $($process.ExitCode)" "Yellow" - - if ($process.ExitCode -ne 0) { - Write-ColorMessage "Error: Agent installation failed with exit code $($process.ExitCode)" "Red" - exit 1 - } - - # Wait for service to be created - Write-ColorMessage "Waiting for service to be created..." "Yellow" - $maxAttempts = 30 - $attempt = 0 - $serviceCreated = $false - - while (-not $serviceCreated -and $attempt -lt $maxAttempts) { - Write-ColorMessage "Executing command: Get-Service -Name 'tacticalrmm' -ErrorAction SilentlyContinue" "Blue" - $service = Get-Service -Name "tacticalrmm" -ErrorAction SilentlyContinue - if ($service) { - $serviceCreated = $true - Write-ColorMessage "Tactical RMM service was created successfully." "Green" - Write-ColorMessage "Service status: $($service.Status)" "Yellow" - } else { - $attempt++ - Start-Sleep -Seconds 1 - } - } - - if (-not $serviceCreated) { - Write-ColorMessage "Error: Service was not created after $maxAttempts seconds" "Red" - exit 1 - } - - # Try to start the service - try { - Write-ColorMessage "Attempting to start the service..." "Yellow" - - # Try to start the service using sc.exe - Write-ColorMessage "Starting service using sc.exe..." "Yellow" - Write-ColorMessage "Executing command: sc.exe start tacticalrmm" "Blue" - $scResult = & sc.exe start tacticalrmm - Write-ColorMessage "sc.exe result: $scResult" "Yellow" - - # Wait for service to start - $startAttempts = 0 - $maxStartAttempts = 30 - $serviceStarted = $false - - while (-not $serviceStarted -and $startAttempts -lt $maxStartAttempts) { - $service.Refresh() - if ($service.Status -eq "Running") { - $serviceStarted = $true - Write-ColorMessage "Service started successfully." "Green" - } else { - $startAttempts++ - Start-Sleep -Seconds 1 - } - } - - if (-not $serviceStarted) { - Write-ColorMessage "Error: Service failed to start after $maxStartAttempts seconds" "Red" - exit 1 - } - - } catch { - Write-ColorMessage "Error: Could not start service: $_" "Red" - exit 1 - } - - # Check if the agent executable was installed - $installedPath = Join-Path $env:ProgramFiles "TacticalAgent\tacticalrmm.exe" - if (Test-Path $installedPath) { - Write-ColorMessage "Agent was installed successfully at: $installedPath" "Green" - } else { - Write-ColorMessage "Error: Agent executable not found at expected location: $installedPath" "Red" - exit 1 - } - - Write-ColorMessage "Agent installation process completed successfully." "Green" -} - -function Get-ValueIfEmpty { - param ( - [string]$VarName, - [string]$PromptMsg, - [string]$DefaultVal = "", - [switch]$Silent = $false - ) - - # Extract the variable name without the script: prefix if present - $actualVarName = $VarName -replace "^script:", "" - - $currVal = Get-Variable -Name $actualVarName -ValueOnly -ErrorAction SilentlyContinue - - # If value is empty or null, use default or prompt for value - if ([string]::IsNullOrEmpty($currVal)) { - if ($Silent) { - # In silent mode, always use default value without prompting - if (-not [string]::IsNullOrEmpty($DefaultVal)) { - Set-Variable -Name $actualVarName -Value $DefaultVal -Scope Script - Write-Host "Using default value for ${actualVarName}: ${DefaultVal}" -ForegroundColor Yellow - } else { - Write-Host "ERROR: ${actualVarName} is required in non-interactive mode" -ForegroundColor Red - exit 1 - } - } else { - # In interactive mode, prompt for value - $promptDefault = if (-not [string]::IsNullOrEmpty($DefaultVal)) { " (default: $DefaultVal)" } else { "" } - $promptValue = Read-Host "$PromptMsg$promptDefault" - - # If user didn't provide a value, use default - if ([string]::IsNullOrEmpty($promptValue) -and -not [string]::IsNullOrEmpty($DefaultVal)) { - $promptValue = $DefaultVal - Write-Host "Using default value: ${DefaultVal}" -ForegroundColor Yellow - } - - # Update the variable with the new value - Set-Variable -Name $actualVarName -Value $promptValue -Scope Script - } - } else { - # Value already exists, display it - Write-Host "Using provided ${actualVarName}: '${currVal}' (type: $(${currVal}.GetType().Name))" -ForegroundColor Green - } -} - -function Check-TacticalInstalled { - Write-Host "=== STEP 1: Checking if Tactical RMM is already installed ===" -ForegroundColor Cyan - - # Check for Tactical RMM service - $tacticalService = Get-Service -Name "tacticalrmm" -ErrorAction SilentlyContinue - - # Check for Tactical RMM executable in Program Files - $programFilesPath = "${env:ProgramFiles}" - $programFilesX86Path = "${env:ProgramFiles(x86)}" - - $tacticalExePath = "$programFilesPath\TacticalAgent\tacticalrmm.exe" - $tacticalExeX86Path = "$programFilesX86Path\TacticalAgent\tacticalrmm.exe" - - $tacticalExeExists = Test-Path $tacticalExePath - $tacticalExeX86Exists = Test-Path $tacticalExeX86Path - - # Check for TacticalRMM registry key - $tacticalRmmKey = "HKLM:\SOFTWARE\TacticalRMM" - $registryExists = Test-Path $tacticalRmmKey - - if ($tacticalService -or $tacticalExeExists -or $tacticalExeX86Exists -or $registryExists) { - Write-Host "Tactical RMM is already installed." -ForegroundColor Yellow - - if ($tacticalService) { - Write-Host "Found Tactical RMM service." -ForegroundColor Yellow - } - - if ($tacticalExeExists) { - Write-Host "Found Tactical RMM executable at: $tacticalExePath" -ForegroundColor Yellow - } - - if ($tacticalExeX86Exists) { - Write-Host "Found Tactical RMM executable at: $tacticalExeX86Path" -ForegroundColor Yellow - } - - if ($registryExists) { - Write-Host "Found TacticalRMM registry key at: $tacticalRmmKey" -ForegroundColor Yellow - try { - $values = Get-ItemProperty -Path $tacticalRmmKey -ErrorAction SilentlyContinue - if ($values) { - Write-Host "Registry values found:" -ForegroundColor Yellow - $values.PSObject.Properties | Where-Object { $_.Name -notlike "PS*" } | ForEach-Object { - Write-Host " - $($_.Name): $($_.Value)" -ForegroundColor Yellow - } - } - } catch { - Write-Host "Could not read registry values: $_" -ForegroundColor Yellow - } - } - - return $true - } else { - Write-Host "Tactical RMM is not installed." -ForegroundColor Green - return $false - } -} - -function Uninstall-TacticalRMM { - Write-Host "=== STEP 2: Uninstalling existing Tactical RMM agent ===" -ForegroundColor Cyan - - # Try to stop the service first - try { - $service = Get-Service -Name "tacticalrmm" -ErrorAction SilentlyContinue - if ($service) { - Write-Host "Stopping Tactical RMM service..." -ForegroundColor Yellow - Stop-Service -Name "tacticalrmm" -Force -ErrorAction SilentlyContinue - Write-Host "Service stopped." -ForegroundColor Green - # Wait for service to fully stop - Start-Sleep -Seconds 5 - } - } catch { - Write-Host "Warning: Could not stop service: ${_}" -ForegroundColor Yellow - } - - # Kill any running tactical processes - Write-Host "Terminating any running Tactical RMM processes..." -ForegroundColor Yellow - Get-Process -Name "tacticalrmm" -ErrorAction SilentlyContinue | Stop-Process -Force - Get-Process -Name "meshagent" -ErrorAction SilentlyContinue | Stop-Process -Force - Start-Sleep -Seconds 3 - - # Check for uninstaller and agent executable in Program Files - $programFilesPath = "${env:ProgramFiles}" - $programFilesX86Path = "${env:ProgramFiles(x86)}" - - $uninstallerPath = "$programFilesPath\TacticalAgent\unins000.exe" - $uninstallerX86Path = "$programFilesX86Path\TacticalAgent\unins000.exe" - $agentPath = "$programFilesPath\TacticalAgent\tacticalrmm.exe" - $agentX86Path = "$programFilesX86Path\TacticalAgent\tacticalrmm.exe" - - # First try to run the agent's uninstall command if available - if (Test-Path $agentPath) { - Write-Host "Running agent uninstall command: & `"$agentPath`" -m uninstall -silent /VERYSILENT /SUPPRESSMSGBOXES" -ForegroundColor Yellow - Start-Process -FilePath $agentPath -ArgumentList "-m uninstall -silent /VERYSILENT /SUPPRESSMSGBOXES" -Wait -NoNewWindow - Write-Host "Agent uninstall command completed." -ForegroundColor Green - Start-Sleep -Seconds 10 - } elseif (Test-Path $agentX86Path) { - Write-Host "Running agent uninstall command: & `"$agentX86Path`" -m uninstall -silent /VERYSILENT /SUPPRESSMSGBOXES" -ForegroundColor Yellow - Start-Process -FilePath $agentX86Path -ArgumentList "-m uninstall -silent /VERYSILENT /SUPPRESSMSGBOXES" -Wait -NoNewWindow - Write-Host "Agent uninstall command completed." -ForegroundColor Green - Start-Sleep -Seconds 10 - } - - # Then run the uninstaller if available - if (Test-Path $uninstallerPath) { - Write-Host "Running uninstaller: $uninstallerPath /VERYSILENT /SUPPRESSMSGBOXES" -ForegroundColor Yellow - Start-Process -FilePath $uninstallerPath -ArgumentList "/VERYSILENT /SUPPRESSMSGBOXES" -Wait -NoNewWindow - Write-Host "Uninstaller completed." -ForegroundColor Green - Start-Sleep -Seconds 10 - } elseif (Test-Path $uninstallerX86Path) { - Write-Host "Running uninstaller: $uninstallerX86Path /VERYSILENT /SUPPRESSMSGBOXES" -ForegroundColor Yellow - Start-Process -FilePath $uninstallerX86Path -ArgumentList "/VERYSILENT /SUPPRESSMSGBOXES" -Wait -NoNewWindow - Write-Host "Uninstaller completed." -ForegroundColor Green - Start-Sleep -Seconds 10 - } - - # Finally, attempt manual cleanup - Write-Host "Performing final cleanup..." -ForegroundColor Yellow - - # Try to remove service - try { - $service = Get-Service -Name "tacticalrmm" -ErrorAction SilentlyContinue - if ($service) { - Write-Host "Stopping and removing Tactical RMM service..." -ForegroundColor Yellow - # First stop the service - Stop-Service -Name "tacticalrmm" -Force -ErrorAction SilentlyContinue - Start-Sleep -Seconds 2 - - # Then remove it using sc.exe - $scResult = & sc.exe delete "tacticalrmm" - Write-Host "sc.exe delete result: $scResult" -ForegroundColor Yellow - - # Verify service is removed - $service = Get-Service -Name "tacticalrmm" -ErrorAction SilentlyContinue - if ($service) { - Write-Host "Warning: Service still exists after removal attempt. Trying alternative method..." -ForegroundColor Yellow - # Try alternative method using WMI - $wmiService = Get-WmiObject -Class Win32_Service -Filter "Name='tacticalrmm'" -ErrorAction SilentlyContinue - if ($wmiService) { - $wmiService.Delete() - Write-Host "Service removed using WMI method." -ForegroundColor Green - } - } else { - Write-Host "Service removed successfully." -ForegroundColor Green - } - Start-Sleep -Seconds 5 - } - } catch { - Write-Host ("Warning: Could not remove service {0}: {1}" -f "tacticalrmm", $_.Exception.Message) -ForegroundColor Yellow - } - - # Try to remove directories - try { - if (Test-Path "$programFilesPath\TacticalAgent") { - Write-Host "Removing $programFilesPath\TacticalAgent directory..." -ForegroundColor Yellow - Remove-Item -Path "$programFilesPath\TacticalAgent" -Recurse -Force -ErrorAction SilentlyContinue - } - - if (Test-Path "$programFilesX86Path\TacticalAgent") { - Write-Host "Removing $programFilesX86Path\TacticalAgent directory..." -ForegroundColor Yellow - Remove-Item -Path "$programFilesX86Path\TacticalAgent" -Recurse -Force -ErrorAction SilentlyContinue - } - - if (Test-Path "$programFilesPath\Mesh Agent") { - Write-Host "Removing $programFilesPath\Mesh Agent directory..." -ForegroundColor Yellow - Remove-Item -Path "$programFilesPath\Mesh Agent" -Recurse -Force -ErrorAction SilentlyContinue - } - - if (Test-Path "$programFilesX86Path\Mesh Agent") { - Write-Host "Removing $programFilesX86Path\Mesh Agent directory..." -ForegroundColor Yellow - Remove-Item -Path "$programFilesX86Path\Mesh Agent" -Recurse -Force -ErrorAction SilentlyContinue - } - } catch { - Write-Host ("Warning: Could not remove directories {0}: {1}" -f $_, $_.Exception.Message) -ForegroundColor Yellow - } - - # Verify uninstallation - Write-Host "Verifying uninstallation..." -ForegroundColor Yellow - $maxAttempts = 3 - $attempt = 1 - $uninstallComplete = $false - - while (-not $uninstallComplete -and $attempt -le $maxAttempts) { - $service = Get-Service -Name "tacticalrmm" -ErrorAction SilentlyContinue - $programFilesExists = Test-Path "$programFilesPath\TacticalAgent" - $programFilesX86Exists = Test-Path "$programFilesX86Path\TacticalAgent" - $meshExists = (Test-Path "$programFilesPath\Mesh Agent") -or (Test-Path "$programFilesX86Path\Mesh Agent") - $processes = Get-Process -Name "tacticalrmm" -ErrorAction SilentlyContinue - $meshProcesses = Get-Process -Name "meshagent" -ErrorAction SilentlyContinue - - if (-not $service -and -not $programFilesExists -and -not $programFilesX86Exists -and -not $meshExists -and -not $processes -and -not $meshProcesses) { - $uninstallComplete = $true - Write-Host "Uninstallation verified successfully." -ForegroundColor Green - } else { - Write-Host "Uninstallation verification attempt $attempt of $maxAttempts..." -ForegroundColor Yellow - Start-Sleep -Seconds 5 - $attempt++ - } - } - - if (-not $uninstallComplete) { - Write-Host "Warning: Could not verify complete uninstallation. Some components may still be present." -ForegroundColor Yellow - } - - # Final wait after uninstallation - Write-Host "Waiting for system to stabilize after uninstallation..." -ForegroundColor Yellow - Start-Sleep -Seconds 10 -} - -function Remove-TacticalRMMCompletely { - Write-Host "=== Performing aggressive cleanup of Tactical RMM components ===" -ForegroundColor Cyan - - # First try to run uninstall from installation directory - $installDir = "${env:ProgramFiles}\TacticalAgent" - $agentExe = Join-Path $installDir "tacticalrmm.exe" - if (Test-Path $agentExe) { - Write-Host "Found agent executable at: $agentExe" -ForegroundColor Yellow - Write-Host "Running uninstall command..." -ForegroundColor Yellow - Write-Host "Executing command: Start-Process -FilePath '$agentExe' -ArgumentList '-m uninstall -silent /VERYSILENT /SUPPRESSMSGBOXES' -Wait -NoNewWindow" -ForegroundColor Blue - try { - Start-Process -FilePath $agentExe -ArgumentList "-m uninstall -silent /VERYSILENT /SUPPRESSMSGBOXES" -Wait -NoNewWindow - Write-Host "Uninstall command completed." -ForegroundColor Green - Start-Sleep -Seconds 10 # Wait for uninstall to complete - } catch { - Write-Host ("Warning: Could not run uninstall command: {0}" -f $_.Exception.Message) -ForegroundColor Yellow - } - } - - # Then remove the TacticalRMM registry key if it exists - $tacticalRmmKey = "HKLM:\SOFTWARE\TacticalRMM" - if (Test-Path $tacticalRmmKey) { - Write-Host "Found TacticalRMM registry key: $tacticalRmmKey" -ForegroundColor Yellow - try { - # Get all values in the main key - Write-Host "Executing command: Get-ItemProperty -Path '$tacticalRmmKey' -ErrorAction SilentlyContinue" -ForegroundColor Blue - $values = Get-ItemProperty -Path $tacticalRmmKey -ErrorAction SilentlyContinue - if ($values) { - Write-Host "Found values in main key to remove:" -ForegroundColor Yellow - $values.PSObject.Properties | Where-Object { $_.Name -notlike "PS*" } | ForEach-Object { - Write-Host " - $($_.Name): $($_.Value)" -ForegroundColor Yellow - } - } - - # Get all subkeys - Write-Host "Executing command: Get-ChildItem -Path '$tacticalRmmKey' -Recurse -ErrorAction SilentlyContinue" -ForegroundColor Blue - $subkeys = Get-ChildItem -Path $tacticalRmmKey -Recurse -ErrorAction SilentlyContinue - if ($subkeys) { - Write-Host "Found subkeys to remove:" -ForegroundColor Yellow - foreach ($subkey in $subkeys) { - Write-Host " - $($subkey.PSPath)" -ForegroundColor Yellow - # Remove all values in subkey first - Write-Host "Executing command: Get-ItemProperty -Path '$($subkey.PSPath)' -ErrorAction SilentlyContinue" -ForegroundColor Blue - $subValues = Get-ItemProperty -Path $subkey.PSPath -ErrorAction SilentlyContinue - if ($subValues) { - $subValues.PSObject.Properties | Where-Object { $_.Name -notlike "PS*" } | ForEach-Object { - Write-Host " - Value: $($_.Name): $($_.Value)" -ForegroundColor Yellow - Write-Host "Executing command: Remove-ItemProperty -Path '$($subkey.PSPath)' -Name '$($_.Name)' -Force -ErrorAction SilentlyContinue" -ForegroundColor Blue - Remove-ItemProperty -Path $subkey.PSPath -Name $_.Name -Force -ErrorAction SilentlyContinue - } - } - # Remove the subkey - Write-Host "Executing command: Remove-Item -Path '$($subkey.PSPath)' -Recurse -Force -ErrorAction SilentlyContinue" -ForegroundColor Blue - Remove-Item -Path $subkey.PSPath -Recurse -Force -ErrorAction SilentlyContinue - } - } - - # Remove all values in main key - $values.PSObject.Properties | Where-Object { $_.Name -notlike "PS*" } | ForEach-Object { - Write-Host "Executing command: Remove-ItemProperty -Path '$tacticalRmmKey' -Name '$($_.Name)' -Force -ErrorAction SilentlyContinue" -ForegroundColor Blue - Remove-ItemProperty -Path $tacticalRmmKey -Name $_.Name -Force -ErrorAction SilentlyContinue - } - - # Remove the main key itself - Write-Host "Executing command: Remove-Item -Path '$tacticalRmmKey' -Recurse -Force -ErrorAction SilentlyContinue" -ForegroundColor Blue - Remove-Item -Path $tacticalRmmKey -Recurse -Force -ErrorAction SilentlyContinue - - # Verify removal - if (Test-Path $tacticalRmmKey) { - Write-Host "Warning: Registry key still exists after removal attempt. Trying alternative method..." -ForegroundColor Yellow - # Try using reg.exe as alternative - $regKeyPath = "HKLM\SOFTWARE\TacticalRMM" - Write-Host "Executing command: reg.exe delete '$regKeyPath' /f" -ForegroundColor Blue - & reg.exe delete $regKeyPath /f - Start-Sleep -Seconds 2 - - if (Test-Path $tacticalRmmKey) { - Write-Host "Error: Could not remove registry key completely." -ForegroundColor Red - } else { - Write-Host "Successfully removed TacticalRMM registry key using alternative method." -ForegroundColor Green - } - } else { - Write-Host "Successfully removed TacticalRMM registry key." -ForegroundColor Green - } - - Start-Sleep -Seconds 5 # Wait for registry changes to take effect - } catch { - Write-Host ("Warning: Could not remove TacticalRMM registry key: {0}" -f $_.Exception.Message) -ForegroundColor Yellow - Write-Host "Attempting alternative removal method..." -ForegroundColor Yellow - try { - # Try using reg.exe as fallback - $regKeyPath = "HKLM\SOFTWARE\TacticalRMM" - Write-Host "Executing command: reg.exe delete '$regKeyPath' /f" -ForegroundColor Blue - & reg.exe delete $regKeyPath /f - Start-Sleep -Seconds 2 - - if (-not (Test-Path $tacticalRmmKey)) { - Write-Host "Successfully removed TacticalRMM registry key using alternative method." -ForegroundColor Green - } - } catch { - Write-Host ("Error: Alternative removal method failed: {0}" -f $_.Exception.Message) -ForegroundColor Red - } - } - } - - # Then try to run the uninstaller if it exists - $uninstallerPath = "${env:ProgramFiles}\TacticalAgent\unins000.exe" - if (Test-Path $uninstallerPath) { - Write-Host "Running Tactical RMM uninstaller..." -ForegroundColor Yellow - Write-Host "Executing command: Start-Process -FilePath '$uninstallerPath' -ArgumentList '/VERYSILENT /SUPPRESSMSGBOXES' -Wait -NoNewWindow" -ForegroundColor Blue - try { - Start-Process -FilePath $uninstallerPath -ArgumentList "/VERYSILENT /SUPPRESSMSGBOXES" -Wait -NoNewWindow - Write-Host "Uninstaller completed." -ForegroundColor Green - Start-Sleep -Seconds 10 # Wait for uninstaller to complete - } catch { - Write-Host ("Warning: Could not run uninstaller: {0}" -f $_.Exception.Message) -ForegroundColor Yellow - } - } - - # Stop and remove all related services - $services = @( - "tacticalrmm", - "tacticalagent", - "tacticalrpc", - "checkrunner", - "Mesh Agent" - ) - - foreach ($service in $services) { - try { - Write-Host "Executing command: Get-Service -Name '$service' -ErrorAction SilentlyContinue" -ForegroundColor Blue - $svc = Get-Service -Name $service -ErrorAction SilentlyContinue - if ($svc) { - Write-Host "Stopping and removing service: $service" -ForegroundColor Yellow - Write-Host "Executing command: Stop-Service -Name '$service' -Force -ErrorAction SilentlyContinue" -ForegroundColor Blue - Stop-Service -Name $service -Force -ErrorAction SilentlyContinue - Start-Sleep -Seconds 2 - Write-Host "Executing command: sc.exe delete '$service'" -ForegroundColor Blue - & sc.exe delete $service - Write-Host "Service $service removed." -ForegroundColor Green - } - } catch { - Write-Host ("Warning: Could not remove service {0}: {1}" -f $service, $_.Exception.Message) -ForegroundColor Yellow - } - } - - # Kill any running processes - $processes = @( - "tacticalrmm", - "tacticalagent", - "meshagent" - ) - - foreach ($proc in $processes) { - try { - Write-Host "Executing command: Get-Process -Name '$proc' -ErrorAction SilentlyContinue | Stop-Process -Force" -ForegroundColor Blue - Get-Process -Name $proc -ErrorAction SilentlyContinue | Stop-Process -Force - Write-Host "Terminated process: $proc" -ForegroundColor Yellow - } catch { - Write-Host ("Warning: Could not terminate process {0}: {1}" -f $proc, $_.Exception.Message) -ForegroundColor Yellow - } - } - - # Remove files and directories - $paths = @( - "${env:ProgramFiles}\TacticalAgent", - "${env:ProgramFiles(x86)}\TacticalAgent", - "${env:ProgramFiles}\Mesh Agent", - "${env:ProgramFiles(x86)}\Mesh Agent", - "${env:ProgramData}\TacticalRMM", - "${env:ProgramData}\Microsoft\Windows\Start Menu\Programs\Tactical RMM Agent" - ) - - foreach ($path in $paths) { - if (Test-Path $path) { - Write-Host "Removing directory: $path" -ForegroundColor Yellow - Write-Host "Executing command: Remove-Item -Path '$path' -Recurse -Force -ErrorAction SilentlyContinue" -ForegroundColor Blue - Remove-Item -Path $path -Recurse -Force -ErrorAction SilentlyContinue - } - } - - # Clean registry entries - Write-Host "Cleaning registry entries..." -ForegroundColor Yellow - - # Then remove other registry entries - $registryPaths = @( - "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Installer\UserData\*", - "HKLM:\SYSTEM\CurrentControlSet\Services\tacticalrmm", - "HKLM:\SYSTEM\CurrentControlSet\Services\tacticalagent", - "HKLM:\SYSTEM\CurrentControlSet\Services\tacticalrpc", - "HKLM:\SYSTEM\CurrentControlSet\Services\checkrunner", - "HKLM:\SYSTEM\CurrentControlSet\Services\Mesh Agent" - ) - - Write-Host "Found registry keys to remove:" -ForegroundColor Yellow - foreach ($regPath in $registryPaths) { - if (Test-Path $regPath) { - Write-Host " - $regPath" -ForegroundColor Yellow - } - } - - foreach ($regPath in $registryPaths) { - try { - if (Test-Path $regPath) { - Write-Host "Removing registry key: $regPath" -ForegroundColor Yellow - Write-Host "Executing command: Remove-Item -Path '$regPath' -Recurse -Force -ErrorAction SilentlyContinue" -ForegroundColor Blue - Remove-Item -Path $regPath -Recurse -Force -ErrorAction SilentlyContinue - } - } catch { - Write-Host ("Warning: Could not remove registry key {0}: {1}" -f $regPath, $_.Exception.Message) -ForegroundColor Yellow - } - } - - # Search for and remove any remaining Tactical RMM related registry entries - Write-Host "Searching for remaining Tactical RMM registry entries..." -ForegroundColor Yellow - $searchTerms = @("TacticalRMM", "TacticalAgent", "tacticalrmm", "tacticalagent") - - # Define specific registry locations to search - $registryLocations = @( - "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Installer\UserData", - "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Installer\InProgress", - "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Installer\UpgradeCodes", - "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Installer\Components", - "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Installer\Patches", - "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Installer\Products", - "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Installer\Features", - "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Installer\SourceList", - "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Installer\Subscriptions", - "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Installer\Transforms", - "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Installer\UserData\S-1-5-18\Products", - "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Installer\UserData\S-1-5-18\Components", - "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Installer\UserData\S-1-5-18\Patches", - "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Installer\UserData\S-1-5-18\Features", - "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Installer\UserData\S-1-5-18\Transforms" - ) - - Write-Host "Searching in registry locations:" -ForegroundColor Yellow - foreach ($location in $registryLocations) { - Write-Host " - $location" -ForegroundColor Yellow - } - - foreach ($location in $registryLocations) { - try { - if (Test-Path $location) { - foreach ($term in $searchTerms) { - Write-Host "Executing command: Get-ChildItem -Path '$location' -ErrorAction SilentlyContinue | Where-Object { `$_.PSPath -like '*$term*' }" -ForegroundColor Blue - $keys = Get-ChildItem -Path $location -ErrorAction SilentlyContinue | - Where-Object { $_.PSPath -like "*$term*" } - - if ($keys) { - Write-Host "Found matching keys in $location for term '$term':" -ForegroundColor Yellow - foreach ($key in $keys) { - Write-Host " - $($key.PSPath)" -ForegroundColor Yellow - } - } - - foreach ($key in $keys) { - Write-Host "Removing registry key: $($key.PSPath)" -ForegroundColor Yellow - Write-Host "Executing command: Remove-Item -Path '$($key.PSPath)' -Recurse -Force -ErrorAction SilentlyContinue" -ForegroundColor Blue - Remove-Item -Path $key.PSPath -Recurse -Force -ErrorAction SilentlyContinue - } - } - } - } catch { - Write-Host ("Warning: Could not search/remove registry entries in {0}: {1}" -f $location, $_.Exception.Message) -ForegroundColor Yellow - } - } - - # Final verification - Write-Host "Performing final verification..." -ForegroundColor Yellow - $remaining = @() - - # Check services - foreach ($service in $services) { - Write-Host "Executing command: Get-Service -Name '$service' -ErrorAction SilentlyContinue" -ForegroundColor Blue - if (Get-Service -Name $service -ErrorAction SilentlyContinue) { - $remaining += "Service: $service" - } - } - - # Check processes - foreach ($proc in $processes) { - Write-Host "Executing command: Get-Process -Name '$proc' -ErrorAction SilentlyContinue" -ForegroundColor Blue - if (Get-Process -Name $proc -ErrorAction SilentlyContinue) { - $remaining += "Process: $proc" - } - } - - # Check directories - foreach ($path in $paths) { - Write-Host "Executing command: Test-Path '$path'" -ForegroundColor Blue - if (Test-Path $path) { - $remaining += "Directory: $path" - } - } - - # Check for TacticalRMM registry key - Write-Host "Executing command: Test-Path '$tacticalRmmKey'" -ForegroundColor Blue - if (Test-Path $tacticalRmmKey) { - $remaining += "Registry Key: $tacticalRmmKey" - } - - if ($remaining.Count -gt 0) { - Write-Host "Warning: The following components could not be removed:" -ForegroundColor Yellow - $remaining | ForEach-Object { Write-Host " - $_" -ForegroundColor Yellow } - } else { - Write-Host "All Tactical RMM components have been removed successfully." -ForegroundColor Green - } - - # Final wait - Write-Host "Waiting for system to stabilize..." -ForegroundColor Yellow - Start-Sleep -Seconds 3 -} - -function Show-Help { - [CmdletBinding()] - param() - - Write-Host "=========================================================" -ForegroundColor Cyan - Write-Host "Windows AMD64 Tactical RMM Agent Installer" -ForegroundColor Cyan - Write-Host "=========================================================" -ForegroundColor Cyan - Write-Host "" - Write-Host "This script installs the Tactical RMM agent on Windows AMD64 systems." - Write-Host "It uses the native AMD64 binary." - Write-Host "" - Write-Host "Usage:" -ForegroundColor Yellow - Write-Host " .\windows_amd64.ps1 -Help" - Write-Host " .\windows_amd64.ps1 [parameters]" - Write-Host "" - Write-Host "Parameters:" -ForegroundColor Yellow - Write-Host " -RmmHost Hostname or IP of the RMM server" - Write-Host " -NatsPort NATS WebSocket port (required)" - Write-Host " -Secure Use HTTPS/WSS for secure connection" - Write-Host " -AuthKey Authentication key for the RMM server" - Write-Host " -ClientId Client ID" - Write-Host " -SiteId Site ID" - Write-Host " -AgentType Agent type (workstation/server)" - Write-Host " -Help Display this help message" - Write-Host "" - Write-Host "Examples:" -ForegroundColor Yellow - Write-Host " # Show help:" - Write-Host " .\windows_amd64.ps1 -Help" - Write-Host "" - Write-Host " # Non-interactive mode with all parameters:" - Write-Host " .\windows_amd64.ps1 -RmmHost 'rmm.example.com' -NatsPort 8000 -AuthKey 'your-auth-key' -ClientId 1 -SiteId 1 -AgentType 'server'" - Write-Host "" - Write-Host "Note: This script requires administrator privileges." -ForegroundColor Red - Write-Host "=========================================================" -ForegroundColor Cyan - exit 0 -} - -function Remove-ExistingBuildFolder { - param( - [string]$Folder - ) - - if (Test-Path $Folder) { - Write-ColorMessage "Removing existing build folder: $Folder" "Yellow" - Remove-Item -Path $Folder -Recurse -Force - Write-ColorMessage "Build folder removed successfully." "Green" - } -} - -function Patch-AgentCode { - Write-ColorMessage "Patching agent code..." "Yellow" - - # Get all .go files in the current directory - $goFiles = Get-ChildItem -Path . -Filter "*.go" -File - - foreach ($file in $goFiles) { - Write-ColorMessage "Checking file: $($file.Name)" "Yellow" - - # Read the file content - $content = Get-Content $file.FullName -Raw - - # Check and replace DefaultOrgName - if ($content -match 'DefaultOrgName = ".*"') { - Write-ColorMessage "Found DefaultOrgName in $($file.Name)" "Yellow" - $content = $content -replace 'DefaultOrgName = ".*"', "DefaultOrgName = `"$OrgName`"" - Write-ColorMessage "Replaced DefaultOrgName with: $OrgName" "Green" - } - - # Check and replace DefaultEmail - if ($content -match 'DefaultEmail = ".*"') { - Write-ColorMessage "Found DefaultEmail in $($file.Name)" "Yellow" - $content = $content -replace 'DefaultEmail = ".*"', "DefaultEmail = `"$ContactEmail`"" - Write-ColorMessage "Replaced DefaultEmail with: $ContactEmail" "Green" - } - - # If this is main.go, patch the log level - if ($file.Name -eq "main.go") { - Write-ColorMessage "Found main.go, checking for log level configuration..." "Yellow" - - # Pattern to match the entire setupLogging function - $setupLoggingPattern = 'func setupLogging\(level, to \*string\) \{[\s\S]*?ll, err := logrus\.ParseLevel\(\*level\)[\s\S]*?if err != nil \{[\s\S]*?ll = logrus\.InfoLevel[\s\S]*?\}[\s\S]*?log\.SetLevel\(ll\)' - - $setupLoggingMatches = [regex]::Matches($content, $setupLoggingPattern) - if ($setupLoggingMatches.Count -gt 0) { - Write-Host "`nFound setupLogging function in main.go:" -ForegroundColor Yellow - foreach ($match in $setupLoggingMatches) { - $lineNumber = [regex]::Matches($content.Substring(0, $match.Index), "`n").Count + 1 - Write-Host "Starting at Line $lineNumber" -ForegroundColor Yellow - Write-Host $match.Value -ForegroundColor Yellow - } - - # New setupLogging implementation that directly sets debug level - $newSetupLogging = @' -func setupLogging(level, to *string) { - // Always set debug level - log.SetLevel(logrus.DebugLevel) -'@ - - Write-Host "`nReplacing with:" -ForegroundColor Yellow - Write-Host $newSetupLogging -ForegroundColor Green - - $content = $content -replace $setupLoggingPattern, $newSetupLogging - Write-Host "`nSetupLogging function updated to always use DEBUG level" -ForegroundColor Green - } - - # Keep the rest of the setupLogging function (output configuration) unchanged - Write-ColorMessage "Log level patching completed" "Green" - } - - # Write the modified content back to the file - Set-Content -Path $file.FullName -Value $content - Write-ColorMessage "Updated $($file.Name)" "Green" - } -} - -############################ -# Default / Config -############################ - -$OUTPUT_BINARY = "rmmagent-windows-amd64.exe" -$AMD64_BINARY = "tacticalagent-v2.9.0-windows-amd64.exe" -$AMD64_BINARY_PATH = Join-Path (Split-Path -Parent $PSCommandPath) "binaries\$AMD64_BINARY" - -# We'll store user-provided or prompted values in these variables: -$script:RmmHost = if ([string]::IsNullOrEmpty($RmmHost) -or $RmmHost -eq $true -or $RmmHost -eq "True") { "" } else { $RmmHost } -$script:RmmPort = if ($RmmPort -eq 0) { 8000 } else { $RmmPort } -$script:Secure = $Secure -$script:AuthKey = if ([string]::IsNullOrEmpty($AuthKey) -or $AuthKey -eq $true -or $AuthKey -eq "True") { "" } else { $AuthKey } - -# Initialize parameters with defaults if not provided -$script:ClientId = $ClientId -$script:SiteId = $SiteId -[string]$script:AgentType = if ([string]::IsNullOrEmpty($AgentType) -or $AgentType -eq $true -or $AgentType -eq "True") { "" } else { "$AgentType" } - -# Show help if requested -if ($Help) { - Show-Help -} - -# Main script flow -try { - Write-ColorMessage "`nTactical RMM Agent Installation Started" "Green" - Write-ColorMessage "======================================" "Green" - - # Store the original directory - $originalDir = Get-Location - - # Check for existing installation and perform thorough cleanup if found - if (Check-TacticalInstalled) { - Write-ColorMessage "Existing installation found. Performing thorough cleanup..." "Yellow" - Remove-TacticalRMMCompletely - Start-Sleep -Seconds 10 # Wait for cleanup to complete - } - - # Install dependencies - Install-Go - Install-Git - - # Remove any existing build folder - Remove-ExistingBuildFolder -Folder $BuildFolder - - # Clone repository - Clone-Repository -RepoUrl "https://github.com/amidaware/rmmagent.git" -Branch "master" -Folder $BuildFolder - - # Validate NATS port is provided - if ([string]::IsNullOrEmpty($NatsPort)) { - Write-ColorMessage "Error: NATS port is required. Please provide -NatsPort parameter." "Red" - exit 1 - } - - # Patch NATS WebSocket URL with the RMM server URL and NATS port - Patch-NatsWebsocketUrl -RmmUrl $RmmServerUrl -NatsPort $NatsPort - - # Patch agent code (org name, email, and log level) - Patch-AgentCode - - # Compile agent - Compile-Agent - - # Create ProgramData directory if it doesn't exist - $programDataDir = Join-Path $env:ProgramData "TacticalRMM" - if (-not (Test-Path $programDataDir)) { - New-Item -ItemType Directory -Path $programDataDir -Force | Out-Null - Write-ColorMessage "Created ProgramData directory: $programDataDir" "Green" - } - - # Copy the compiled binary to ProgramData - $sourceBinary = Join-Path (Get-Location) "rmmagent.exe" - $targetBinary = Join-Path $programDataDir "rmmagent.exe" - if (Test-Path $sourceBinary) { - Copy-Item -Path $sourceBinary -Destination $targetBinary -Force - Write-ColorMessage "Copied binary to: $targetBinary" "Green" - } else { - Write-ColorMessage "Error: Compiled binary not found at: $sourceBinary" "Red" - exit 1 - } - - # Return to original directory - Set-Location $originalDir - - # Clean up the build folder - if (Test-Path $BuildFolder) { - Write-ColorMessage "Cleaning up build folder..." "Yellow" - Remove-Item -Path $BuildFolder -Recurse -Force - Write-ColorMessage "Build folder cleaned up successfully." "Green" - } - - # Install agent if not skipping - if (-not $SkipRun) { - # Validate required parameters - if ([string]::IsNullOrEmpty($AuthKey)) { - Write-ColorMessage "Error: AuthKey is required" "Red" - exit 1 - } - - Write-ColorMessage "Installing agent with parameters:" "Yellow" - Write-ColorMessage "RMM URL: $RmmServerUrl" "Yellow" - Write-ColorMessage "Auth Key: $AuthKey" "Yellow" - Write-ColorMessage "Client ID: $ClientId" "Yellow" - Write-ColorMessage "Site ID: $SiteId" "Yellow" - Write-ColorMessage "Agent Type: $AgentType" "Yellow" - - # Pass the parameters directly to Install-Agent - Install-Agent -RmmUrl $RmmServerUrl -AuthKey $AuthKey -ClientId $ClientId -SiteId $SiteId -AgentType $AgentType - } - - Write-ColorMessage "`nInstallation completed successfully!" "Green" - Write-ColorMessage "Agent binary location: $targetBinary" "Green" - - # Add log monitoring instructions - $agentLogPath = Join-Path $env:ProgramFiles "TacticalAgent\agent.log" - Write-ColorMessage "`nTo monitor the agent log in real-time, run one of these commands in PowerShell:" "Yellow" - Write-ColorMessage "Option 1 (PowerShell):" "Blue" - Write-ColorMessage " Get-Content -Path '$agentLogPath' -Wait" "White" - Write-ColorMessage "Option 2 (PowerShell, last 50 lines):" "Blue" - Write-ColorMessage " Get-Content -Path '$agentLogPath' -Tail 50 -Wait" "White" - Write-ColorMessage "Option 3 (Command Prompt):" "Blue" - Write-ColorMessage " type '$agentLogPath'" "White" -} -catch { - Write-ColorMessage "`nInstallation Failed:" "Red" - Write-ColorMessage "Error: $($_.Exception.Message)" "Red" - Write-ColorMessage "Stack Trace: $($_.Exception.StackTrace)" "Red" - exit 1 -} From de9455b90438f79a0c7bb794c4d74213497bd3c9 Mon Sep 17 00:00:00 2001 From: mikhailm-coder Date: Wed, 22 Jul 2026 14:29:43 +0200 Subject: [PATCH 17/19] chore: remove all remaining references to Tactical RMM (#2177) Co-authored-by: Claude Fable 5 Co-authored-by: Ivan --- clients/openframe-client/config/agent.toml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/clients/openframe-client/config/agent.toml b/clients/openframe-client/config/agent.toml index 88bc5b3b7..e1e50c113 100644 --- a/clients/openframe-client/config/agent.toml +++ b/clients/openframe-client/config/agent.toml @@ -65,10 +65,6 @@ meshcentral_enabled = false meshcentral_url = "" meshcentral_token = "" -tactical_rmm_enabled = false -tactical_rmm_url = "" -tactical_rmm_token = "" - fleet_mdm_enabled = false fleet_mdm_url = "" fleet_mdm_token = "" From 387f689f1eed3386ed2608e49c82e05bb4f1a70f Mon Sep 17 00:00:00 2001 From: mikhailm-coder Date: Wed, 22 Jul 2026 14:56:34 +0200 Subject: [PATCH 18/19] fix(client): clippy unnecessary_sort_by in ported LKG service sort_by(|a, b| b.0.cmp(&a.0)) -> sort_by_key(Reverse) per the repo -D warnings gate; originates in tenant #2169, same fix applies upstream. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011z7MGH2qYeKt2ZQV88EaD8 --- .../openframe-client/src/services/last_known_good_service.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/openframe-client/src/services/last_known_good_service.rs b/clients/openframe-client/src/services/last_known_good_service.rs index df4ca629a..c5b409a93 100644 --- a/clients/openframe-client/src/services/last_known_good_service.rs +++ b/clients/openframe-client/src/services/last_known_good_service.rs @@ -207,7 +207,7 @@ impl LastKnownGoodService { return; } - transcripts.sort_by(|a, b| b.0.cmp(&a.0)); // newest first + transcripts.sort_by_key(|t| std::cmp::Reverse(t.0)); // newest first for (_, path) in transcripts.into_iter().skip(keep) { match fs::remove_file(&path) { Ok(_) => info!("Removed old updater transcript: {}", path.display()), From 6ae9cdb78b57f5db14bb57bf6aa3f7ca23205084 Mon Sep 17 00:00:00 2001 From: mikhailm-coder Date: Wed, 22 Jul 2026 15:12:13 +0200 Subject: [PATCH 19/19] fix(client): drop redundant borrow in anyhow! arg (clippy 1.97 useless_borrows_in_formatting) CI runners moved to Rust 1.97 whose new lint fires on pre-existing code in registration_client.rs (untouched by this port; latent on main). Verified clippy-clean on 1.97 for both the host target and x86_64-pc-windows-gnu. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011z7MGH2qYeKt2ZQV88EaD8 --- clients/openframe-client/src/clients/registration_client.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/openframe-client/src/clients/registration_client.rs b/clients/openframe-client/src/clients/registration_client.rs index 0483b1b33..7a98a0e55 100644 --- a/clients/openframe-client/src/clients/registration_client.rs +++ b/clients/openframe-client/src/clients/registration_client.rs @@ -95,7 +95,7 @@ impl RegistrationClient { return Err(anyhow::anyhow!( "Failed to register agent with status {} and body {}", status, - &body + body ) .into()); }