From 39af3204ff942422e8a654746b7481481f5d4ee6 Mon Sep 17 00:00:00 2001 From: Andrei Hasna Date: Thu, 23 Jul 2026 15:48:07 +0300 Subject: [PATCH] fix(auth): auto-switch profile on genuine exhaustion (regression from #374's false-cascade fix) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #374 stopped a false weekly-limit cascade by making rolling account/rateLimits/updated snapshots display-only and gating all exhaustion bookkeeping + auto-switch to the account-verified AccountUsage read. That over- corrected: the rolling snapshot was also what drove auto-switch on GENUINE mid-turn exhaustion (it populated auth_profile_auto_switch_snapshots_by_limit_id and called maybe_auto_switch_auth_profile_for_rate_limit). After #374, when the current profile is really exhausted the switch map is empty (the ~60s authoritative heartbeat has not re-read yet and is then suppressed), and the usage-limit turn-error path only switched when a manual reset was active — so with auto-reset unavailable the failed turn just parked on a self-heal retry until the (days-away) reset instead of switching. Fix: drive auto-switch from the current profile's OWN authoritative usage-limit / 429 turn error (turn_runtime.rs on_rate_limit_error and the reset-unavailable fallback in usage_limit_reset/automatic.rs). This is authoritative for the current profile and immune to the #374 misattribution: a sibling agent's identity-less rolling snapshot never fails THIS profile's own turn. It prefers a cached account-verified exhausted window and otherwise synthesizes the trigger window from the operator's enabled auto-switch config (weekly preferred, else 5h), reusing the existing candidate selection that cycles to an Unknown-but- untried profile and bails (no loop) when every alternate is exhausted. Rolling snapshots remain strictly display-only and never seed the switch map, so #374's no-false-cascade invariant holds. Tests (tui/src/chatwidget/tests/status_and_layout.rs): - genuine_usage_limit_error_auto_switches_without_a_cached_snapshot (the regression) - rolling_snapshot_alone_never_switches_and_never_seeds_the_switch_map (#374 preserved) - genuine_usage_limit_error_does_not_loop_when_all_profiles_exhausted (graceful) --- codex-rs/tui/src/chatwidget/rate_limits.rs | 80 +++++++++++ .../src/chatwidget/tests/status_and_layout.rs | 133 ++++++++++++++++++ codex-rs/tui/src/chatwidget/turn_runtime.rs | 12 +- .../chatwidget/usage_limit_reset/automatic.rs | 5 +- .../tui/src/chatwidget/usage_self_heal.rs | 4 +- 5 files changed, 230 insertions(+), 4 deletions(-) diff --git a/codex-rs/tui/src/chatwidget/rate_limits.rs b/codex-rs/tui/src/chatwidget/rate_limits.rs index c5bcea52d7..049028bff8 100644 --- a/codex-rs/tui/src/chatwidget/rate_limits.rs +++ b/codex-rs/tui/src/chatwidget/rate_limits.rs @@ -11,8 +11,11 @@ use crate::chatwidget::usage_profile_broker::exhausted_auto_switch_window_for_sn use crate::chatwidget::usage_profile_broker::fallback_limit_label as broker_fallback_limit_label; use crate::chatwidget::usage_profile_broker::get_limits_duration as broker_get_limits_duration; use crate::chatwidget::usage_profile_broker::limit_label_for_window as broker_limit_label_for_window; +use crate::chatwidget::usage_self_heal::parse_usage_limit_reset_timestamp; use crate::chatwidget::user_messages::QueueInsertionPosition; +use crate::legacy_core::usage_profile_health::FIVE_HOUR_LIMIT_LABEL; use crate::legacy_core::usage_profile_health::UsageProfileCooldownKey; +use crate::legacy_core::usage_profile_health::WEEKLY_LIMIT_LABEL; use crate::legacy_core::usage_profile_health::cooldown_duration_for_reset; use codex_app_server_protocol::CodexErrorInfo as AppServerCodexErrorInfo; use codex_login::list_auth_profiles; @@ -406,6 +409,83 @@ impl ChatWidget { self.send_auth_profile_auto_switch(next_profile, trigger_key, window); } + /// Auto-switch away from the CURRENT profile in response to its own, authoritative + /// usage-limit / 429 turn failure. + /// + /// A usage-limit *turn error* is emitted by the app server rejecting *this* profile's + /// own turn, so — unlike a rolling `account/rateLimits/updated` snapshot, which #374 + /// made display-only because a sibling agent spawned on a *different* account can emit + /// an identity-less 100%-usage snapshot that would be misattributed to the current + /// profile — it authoritatively identifies the current profile as exhausted. Switching + /// on it therefore cannot revive the cross-agent false-cascade #374 fixed: a foreign + /// snapshot never produces a usage-limit turn error on this widget's own profile. + /// + /// Prefers an authoritative exhausted snapshot already cached for the current profile + /// (which carries the exact limiting window). When none has been observed yet — the + /// common case, because the limit is usually crossed mid-turn, before the ~60s + /// authoritative heartbeat re-reads it — falls back to a window synthesized from the + /// operator's enabled auto-switch config so the failed turn still moves to another + /// configured profile instead of stalling until the (possibly days-away) reset. + pub(in crate::chatwidget) fn try_auth_profile_switch_for_usage_limit( + &mut self, + is_usage_limit: bool, + error_message: Option<&str>, + ) -> bool { + // Cached, account-verified exhaustion (populated only by `AccountUsage` reads, + // never by a rolling notification) takes priority and carries the exact window. + if self.try_auth_profile_switch_after_reset_unavailable() { + return true; + } + // Only a hard usage-limit block should synthesize a switch without a corroborating + // cached snapshot; a transient overload/429 must not rotate profiles. + if !is_usage_limit || !self.is_session_configured() { + return false; + } + let Some(window) = self.synthetic_usage_limit_auto_switch_window(error_message) else { + return false; + }; + let Some((next_profile, trigger_key)) = + self.auth_profile_auto_switch_target("codex", &window) + else { + return false; + }; + self.send_auth_profile_auto_switch(next_profile, trigger_key, window); + true + } + + /// Trigger window used to auto-switch on a genuine usage-limit turn error when no + /// authoritative exhausted snapshot has been cached yet. + /// + /// Honors the operator's enabled windows: a hard usage-limit block ("try again + /// ") is the weekly cap in practice, so weekly is preferred when enabled and the + /// 5h window is used otherwise. Returns `None` when auto-switch is disabled or both + /// windows are opted out, so a disabled window is never switched on. The reset instant + /// is parsed from the error text when present, keeping the trigger key distinct per + /// exhaustion so repeated genuine failures can cycle across profiles. + fn synthetic_usage_limit_auto_switch_window( + &self, + error_message: Option<&str>, + ) -> Option { + let config = &self.config.auth_profile_auto_switch; + if !config.enabled { + return None; + } + let label = if config.on_weekly_limit { + WEEKLY_LIMIT_LABEL + } else if config.on_5h_limit { + FIVE_HOUR_LIMIT_LABEL + } else { + return None; + }; + let resets_at = error_message + .and_then(parse_usage_limit_reset_timestamp) + .map(|reset_at| reset_at.timestamp()); + Some(UsageProfileAutoSwitchWindow { + label: label.to_string(), + resets_at, + }) + } + pub(super) fn maybe_auto_switch_auth_profile_before_user_turn( &mut self, user_message: &UserMessage, diff --git a/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs b/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs index 7e2afea040..b4293e104d 100644 --- a/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs +++ b/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs @@ -1528,6 +1528,139 @@ async fn rolling_rate_limit_update_does_not_auto_switch_auth_profile() { } } +#[tokio::test] +async fn genuine_usage_limit_error_auto_switches_without_a_cached_snapshot() { + // Regression (from #374): when the current profile is genuinely exhausted mid-turn the + // authoritative account read has not observed it yet, so no exhausted snapshot is + // cached. #374 made the rolling per-turn snapshot display-only, which used to be what + // drove the switch here — leaving the failed turn to just stall until the (days-away) + // reset. The profile's OWN usage-limit turn error is authoritative for it, so it must + // still auto-switch to another configured profile. + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + save_test_auth_profile(&chat, "work"); + save_test_auth_profile(&chat, "personal"); + chat.config.selected_auth_profile = Some("work".to_string()); + chat.config.auth_profile_auto_switch.enabled = true; + chat.config.auth_profile_auto_switch.on_weekly_limit = true; + chat.config.auth_profile_auto_switch.profiles = + vec!["work".to_string(), "personal".to_string()]; + // No usage-limit reset flow available, so the switch must be driven directly by the + // authoritative turn failure rather than a reset attempt. + chat.config.usage_limit.auto_reset_enabled = false; + configure_test_session(&mut chat); + drain_insert_history(&mut rx); + + // Deliberately no `on_rate_limit_snapshot(...)`: the switch map is empty, mirroring a + // limit crossed mid-turn before the authoritative heartbeat could re-read it. + assert!( + chat.auth_profile_auto_switch_snapshots_by_limit_id + .is_empty() + ); + chat.on_rate_limit_error( + RateLimitErrorKind::UsageLimit, + "You've hit your usage limit. Visit https://chatgpt.com/codex/settings/usage to purchase more credits or try again at Jul 28th, 2026 4:53 PM." + .to_string(), + ); + + let switches = std::iter::from_fn(|| rx.try_recv().ok()) + .filter_map(|event| match event { + AppEvent::SwitchAuthProfile { + profile, reason, .. + } => Some((profile, reason)), + _ => None, + }) + .collect::>(); + assert_eq!( + switches, + vec![( + Some("personal".to_string()), + crate::app_event::AuthProfileSwitchReason::AutoRateLimit { + window: "weekly".to_string(), + }, + )], + "genuine usage-limit error must auto-switch to the next configured profile" + ); + // The failed turn is re-queued so it resumes on the freshly selected profile. + assert!(chat.pending_usage_self_heal_retry_id().is_none()); +} + +#[tokio::test] +async fn rolling_snapshot_alone_never_switches_and_never_seeds_the_switch_map() { + // Preserves #374: a rolling `account/rateLimits/updated` notification carries no + // account identity, so under multi-agent spawning it can originate from a sibling on a + // *different* exhausted account. It must never switch profiles, and — unlike the + // authoritative turn-error path added for genuine exhaustion — it must never seed the + // switch map that a later user turn consumes, otherwise the #374 false-cascade returns. + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + save_test_auth_profile(&chat, "work"); + save_test_auth_profile(&chat, "personal"); + chat.config.selected_auth_profile = Some("work".to_string()); + chat.config.auth_profile_auto_switch.enabled = true; + chat.config.auth_profile_auto_switch.profiles = + vec!["work".to_string(), "personal".to_string()]; + configure_test_session(&mut chat); + drain_insert_history(&mut rx); + + chat.on_rolling_rate_limit_snapshot(rate_limit_snapshot_for_window( + /*used_percent*/ 100, + /*window_duration_mins*/ 7 * 24 * 60, + /*resets_at*/ chrono::Utc::now().timestamp() + 7 * 24 * 60 * 60, + )); + + while let Ok(event) = rx.try_recv() { + assert!( + !matches!(event, AppEvent::SwitchAuthProfile { .. }), + "rolling snapshot must not auto-switch, got {event:?}" + ); + } + assert!( + chat.auth_profile_auto_switch_snapshots_by_limit_id + .is_empty(), + "rolling snapshot must not seed the auth-profile auto-switch map" + ); +} + +#[tokio::test] +async fn genuine_usage_limit_error_does_not_loop_when_all_profiles_exhausted() { + // Graceful degradation: when the only alternate profile is itself known-exhausted the + // switch must bail without emitting a switch, and repeating the failure must not loop. + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + save_test_auth_profile(&chat, "work"); + save_test_auth_profile(&chat, "personal"); + chat.config.selected_auth_profile = Some("work".to_string()); + chat.config.auth_profile_auto_switch.enabled = true; + chat.config.auth_profile_auto_switch.on_weekly_limit = true; + chat.config.auth_profile_auto_switch.profiles = + vec!["work".to_string(), "personal".to_string()]; + chat.config.usage_limit.auto_reset_enabled = false; + configure_test_session(&mut chat); + drain_insert_history(&mut rx); + + // The only alternate profile is already exhausted, so there is nowhere healthy to go. + chat.on_auth_profile_rate_limit_snapshots( + Some("personal".to_string()), + vec![rate_limit_snapshot_for_window( + /*used_percent*/ 100, + /*window_duration_mins*/ 7 * 24 * 60, + /*resets_at*/ chrono::Utc::now().timestamp() + 7 * 24 * 60 * 60, + )], + ); + drain_insert_history(&mut rx); + + for _ in 0..2 { + chat.on_rate_limit_error( + RateLimitErrorKind::UsageLimit, + "You've hit your usage limit. Try again at Jul 28th, 2026 4:53 PM.".to_string(), + ); + while let Ok(event) = rx.try_recv() { + assert!( + !matches!(event, AppEvent::SwitchAuthProfile { .. }), + "no profile has capacity, so no switch must be emitted, got {event:?}" + ); + } + } +} + #[tokio::test] async fn exhausted_limit_auto_switches_to_profile_with_most_fresh_usage() { let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; diff --git a/codex-rs/tui/src/chatwidget/turn_runtime.rs b/codex-rs/tui/src/chatwidget/turn_runtime.rs index b7ffbfd796..6189baa220 100644 --- a/codex-rs/tui/src/chatwidget/turn_runtime.rs +++ b/codex-rs/tui/src/chatwidget/turn_runtime.rs @@ -464,13 +464,21 @@ impl ChatWidget { ); let reset_recovery_was_opted_out = matches!(auto_reset_check, UsageLimitAutoResetCheckOutcome::OptedOut); + // When no usage-limit reset flow will own the failed turn (auto-reset disabled, + // opted out, non-canonical backend, or a manual reset already in progress), + // auto-switch to another configured profile driven by this profile's OWN + // authoritative usage-limit failure. This restores the switch that #374 removed + // when it made rolling snapshots display-only, without reviving the cross-agent + // false-cascade (a sibling's snapshot never fails *this* profile's turn). let profile_fallback_owns_failed_turn = should_retry && matches!( auto_reset_check, UsageLimitAutoResetCheckOutcome::Unavailable ) - && self.manual_usage_limit_reset_is_active() - && self.try_auth_profile_switch_after_reset_unavailable(); + && self.try_auth_profile_switch_for_usage_limit( + matches!(error_kind, RateLimitErrorKind::UsageLimit), + Some(message.as_str()), + ); if profile_fallback_owns_failed_turn { self.resume_after_usage_limit_reset(); } diff --git a/codex-rs/tui/src/chatwidget/usage_limit_reset/automatic.rs b/codex-rs/tui/src/chatwidget/usage_limit_reset/automatic.rs index bab58084b9..7d99c1e9f0 100644 --- a/codex-rs/tui/src/chatwidget/usage_limit_reset/automatic.rs +++ b/codex-rs/tui/src/chatwidget/usage_limit_reset/automatic.rs @@ -202,7 +202,10 @@ impl ChatWidget { } pub(super) fn fallback_auth_profile_switch_after_reset_unavailable(&mut self) { - if self.try_auth_profile_switch_after_reset_unavailable() { + // Reached only after a genuine usage-limit failure whose reset recovery is + // unavailable, so the current profile is authoritatively exhausted: switch even + // when the authoritative read has not yet cached the exhausted window. + if self.try_auth_profile_switch_for_usage_limit(/*is_usage_limit*/ true, None) { return; } let retry_delay = self.maybe_schedule_usage_self_heal_retry( diff --git a/codex-rs/tui/src/chatwidget/usage_self_heal.rs b/codex-rs/tui/src/chatwidget/usage_self_heal.rs index 9997fc080c..2af2e7043e 100644 --- a/codex-rs/tui/src/chatwidget/usage_self_heal.rs +++ b/codex-rs/tui/src/chatwidget/usage_self_heal.rs @@ -222,7 +222,9 @@ impl ChatWidget { } } -fn parse_usage_limit_reset_timestamp(message: &str) -> Option> { +pub(in crate::chatwidget) fn parse_usage_limit_reset_timestamp( + message: &str, +) -> Option> { let marker = "try again at "; let start = message.to_ascii_lowercase().find(marker)? + marker.len(); let candidate = message.get(start..)?.trim_start();