Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 80 additions & 0 deletions codex-rs/tui/src/chatwidget/rate_limits.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,11 @@ use crate::chatwidget::usage_profile_broker::exhausted_auto_switch_window_for_sn
use crate::chatwidget::usage_profile_broker::fallback_limit_label as broker_fallback_limit_label;
use crate::chatwidget::usage_profile_broker::get_limits_duration as broker_get_limits_duration;
use crate::chatwidget::usage_profile_broker::limit_label_for_window as broker_limit_label_for_window;
use crate::chatwidget::usage_self_heal::parse_usage_limit_reset_timestamp;
use crate::chatwidget::user_messages::QueueInsertionPosition;
use crate::legacy_core::usage_profile_health::FIVE_HOUR_LIMIT_LABEL;
use crate::legacy_core::usage_profile_health::UsageProfileCooldownKey;
use crate::legacy_core::usage_profile_health::WEEKLY_LIMIT_LABEL;
use crate::legacy_core::usage_profile_health::cooldown_duration_for_reset;
use codex_app_server_protocol::CodexErrorInfo as AppServerCodexErrorInfo;
use codex_login::list_auth_profiles;
Expand Down Expand Up @@ -406,6 +409,83 @@ impl ChatWidget {
self.send_auth_profile_auto_switch(next_profile, trigger_key, window);
}

/// Auto-switch away from the CURRENT profile in response to its own, authoritative
/// usage-limit / 429 turn failure.
///
/// A usage-limit *turn error* is emitted by the app server rejecting *this* profile's
/// own turn, so — unlike a rolling `account/rateLimits/updated` snapshot, which #374
/// made display-only because a sibling agent spawned on a *different* account can emit
/// an identity-less 100%-usage snapshot that would be misattributed to the current
/// profile — it authoritatively identifies the current profile as exhausted. Switching
/// on it therefore cannot revive the cross-agent false-cascade #374 fixed: a foreign
/// snapshot never produces a usage-limit turn error on this widget's own profile.
///
/// Prefers an authoritative exhausted snapshot already cached for the current profile
/// (which carries the exact limiting window). When none has been observed yet — the
/// common case, because the limit is usually crossed mid-turn, before the ~60s
/// authoritative heartbeat re-reads it — falls back to a window synthesized from the
/// operator's enabled auto-switch config so the failed turn still moves to another
/// configured profile instead of stalling until the (possibly days-away) reset.
pub(in crate::chatwidget) fn try_auth_profile_switch_for_usage_limit(
&mut self,
is_usage_limit: bool,
error_message: Option<&str>,
) -> bool {
// Cached, account-verified exhaustion (populated only by `AccountUsage` reads,
// never by a rolling notification) takes priority and carries the exact window.
if self.try_auth_profile_switch_after_reset_unavailable() {
return true;
}
// Only a hard usage-limit block should synthesize a switch without a corroborating
// cached snapshot; a transient overload/429 must not rotate profiles.
if !is_usage_limit || !self.is_session_configured() {
return false;
}
let Some(window) = self.synthetic_usage_limit_auto_switch_window(error_message) else {
return false;
};
let Some((next_profile, trigger_key)) =
self.auth_profile_auto_switch_target("codex", &window)
else {
return false;
};
self.send_auth_profile_auto_switch(next_profile, trigger_key, window);
true
}

/// Trigger window used to auto-switch on a genuine usage-limit turn error when no
/// authoritative exhausted snapshot has been cached yet.
///
/// Honors the operator's enabled windows: a hard usage-limit block ("try again
/// <date>") is the weekly cap in practice, so weekly is preferred when enabled and the
/// 5h window is used otherwise. Returns `None` when auto-switch is disabled or both
/// windows are opted out, so a disabled window is never switched on. The reset instant
/// is parsed from the error text when present, keeping the trigger key distinct per
/// exhaustion so repeated genuine failures can cycle across profiles.
fn synthetic_usage_limit_auto_switch_window(
&self,
error_message: Option<&str>,
) -> Option<UsageProfileAutoSwitchWindow> {
let config = &self.config.auth_profile_auto_switch;
if !config.enabled {
return None;
}
let label = if config.on_weekly_limit {
WEEKLY_LIMIT_LABEL
} else if config.on_5h_limit {
FIVE_HOUR_LIMIT_LABEL
} else {
return None;
};
let resets_at = error_message
.and_then(parse_usage_limit_reset_timestamp)
.map(|reset_at| reset_at.timestamp());
Some(UsageProfileAutoSwitchWindow {
label: label.to_string(),
resets_at,
})
}

pub(super) fn maybe_auto_switch_auth_profile_before_user_turn(
&mut self,
user_message: &UserMessage,
Expand Down
133 changes: 133 additions & 0 deletions codex-rs/tui/src/chatwidget/tests/status_and_layout.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1528,6 +1528,139 @@ async fn rolling_rate_limit_update_does_not_auto_switch_auth_profile() {
}
}

#[tokio::test]
async fn genuine_usage_limit_error_auto_switches_without_a_cached_snapshot() {
// Regression (from #374): when the current profile is genuinely exhausted mid-turn the
// authoritative account read has not observed it yet, so no exhausted snapshot is
// cached. #374 made the rolling per-turn snapshot display-only, which used to be what
// drove the switch here — leaving the failed turn to just stall until the (days-away)
// reset. The profile's OWN usage-limit turn error is authoritative for it, so it must
// still auto-switch to another configured profile.
let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await;
save_test_auth_profile(&chat, "work");
save_test_auth_profile(&chat, "personal");
chat.config.selected_auth_profile = Some("work".to_string());
chat.config.auth_profile_auto_switch.enabled = true;
chat.config.auth_profile_auto_switch.on_weekly_limit = true;
chat.config.auth_profile_auto_switch.profiles =
vec!["work".to_string(), "personal".to_string()];
// No usage-limit reset flow available, so the switch must be driven directly by the
// authoritative turn failure rather than a reset attempt.
chat.config.usage_limit.auto_reset_enabled = false;
configure_test_session(&mut chat);
drain_insert_history(&mut rx);

// Deliberately no `on_rate_limit_snapshot(...)`: the switch map is empty, mirroring a
// limit crossed mid-turn before the authoritative heartbeat could re-read it.
assert!(
chat.auth_profile_auto_switch_snapshots_by_limit_id
.is_empty()
);
chat.on_rate_limit_error(
RateLimitErrorKind::UsageLimit,
"You've hit your usage limit. Visit https://chatgpt.com/codex/settings/usage to purchase more credits or try again at Jul 28th, 2026 4:53 PM."
.to_string(),
);

let switches = std::iter::from_fn(|| rx.try_recv().ok())
.filter_map(|event| match event {
AppEvent::SwitchAuthProfile {
profile, reason, ..
} => Some((profile, reason)),
_ => None,
})
.collect::<Vec<_>>();
assert_eq!(
switches,
vec![(
Some("personal".to_string()),
crate::app_event::AuthProfileSwitchReason::AutoRateLimit {
window: "weekly".to_string(),
},
)],
"genuine usage-limit error must auto-switch to the next configured profile"
);
// The failed turn is re-queued so it resumes on the freshly selected profile.
assert!(chat.pending_usage_self_heal_retry_id().is_none());
}

#[tokio::test]
async fn rolling_snapshot_alone_never_switches_and_never_seeds_the_switch_map() {
// Preserves #374: a rolling `account/rateLimits/updated` notification carries no
// account identity, so under multi-agent spawning it can originate from a sibling on a
// *different* exhausted account. It must never switch profiles, and — unlike the
// authoritative turn-error path added for genuine exhaustion — it must never seed the
// switch map that a later user turn consumes, otherwise the #374 false-cascade returns.
let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await;
save_test_auth_profile(&chat, "work");
save_test_auth_profile(&chat, "personal");
chat.config.selected_auth_profile = Some("work".to_string());
chat.config.auth_profile_auto_switch.enabled = true;
chat.config.auth_profile_auto_switch.profiles =
vec!["work".to_string(), "personal".to_string()];
configure_test_session(&mut chat);
drain_insert_history(&mut rx);

chat.on_rolling_rate_limit_snapshot(rate_limit_snapshot_for_window(
/*used_percent*/ 100,
/*window_duration_mins*/ 7 * 24 * 60,
/*resets_at*/ chrono::Utc::now().timestamp() + 7 * 24 * 60 * 60,
));

while let Ok(event) = rx.try_recv() {
assert!(
!matches!(event, AppEvent::SwitchAuthProfile { .. }),
"rolling snapshot must not auto-switch, got {event:?}"
);
}
assert!(
chat.auth_profile_auto_switch_snapshots_by_limit_id
.is_empty(),
"rolling snapshot must not seed the auth-profile auto-switch map"
);
}

#[tokio::test]
async fn genuine_usage_limit_error_does_not_loop_when_all_profiles_exhausted() {
// Graceful degradation: when the only alternate profile is itself known-exhausted the
// switch must bail without emitting a switch, and repeating the failure must not loop.
let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await;
save_test_auth_profile(&chat, "work");
save_test_auth_profile(&chat, "personal");
chat.config.selected_auth_profile = Some("work".to_string());
chat.config.auth_profile_auto_switch.enabled = true;
chat.config.auth_profile_auto_switch.on_weekly_limit = true;
chat.config.auth_profile_auto_switch.profiles =
vec!["work".to_string(), "personal".to_string()];
chat.config.usage_limit.auto_reset_enabled = false;
configure_test_session(&mut chat);
drain_insert_history(&mut rx);

// The only alternate profile is already exhausted, so there is nowhere healthy to go.
chat.on_auth_profile_rate_limit_snapshots(
Some("personal".to_string()),
vec![rate_limit_snapshot_for_window(
/*used_percent*/ 100,
/*window_duration_mins*/ 7 * 24 * 60,
/*resets_at*/ chrono::Utc::now().timestamp() + 7 * 24 * 60 * 60,
)],
);
drain_insert_history(&mut rx);

for _ in 0..2 {
chat.on_rate_limit_error(
RateLimitErrorKind::UsageLimit,
"You've hit your usage limit. Try again at Jul 28th, 2026 4:53 PM.".to_string(),
);
while let Ok(event) = rx.try_recv() {
assert!(
!matches!(event, AppEvent::SwitchAuthProfile { .. }),
"no profile has capacity, so no switch must be emitted, got {event:?}"
);
}
}
}

#[tokio::test]
async fn exhausted_limit_auto_switches_to_profile_with_most_fresh_usage() {
let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await;
Expand Down
12 changes: 10 additions & 2 deletions codex-rs/tui/src/chatwidget/turn_runtime.rs
Original file line number Diff line number Diff line change
Expand Up @@ -464,13 +464,21 @@ impl ChatWidget {
);
let reset_recovery_was_opted_out =
matches!(auto_reset_check, UsageLimitAutoResetCheckOutcome::OptedOut);
// When no usage-limit reset flow will own the failed turn (auto-reset disabled,
// opted out, non-canonical backend, or a manual reset already in progress),
// auto-switch to another configured profile driven by this profile's OWN
// authoritative usage-limit failure. This restores the switch that #374 removed
// when it made rolling snapshots display-only, without reviving the cross-agent
// false-cascade (a sibling's snapshot never fails *this* profile's turn).
let profile_fallback_owns_failed_turn = should_retry
&& matches!(
auto_reset_check,
UsageLimitAutoResetCheckOutcome::Unavailable
)
&& self.manual_usage_limit_reset_is_active()
&& self.try_auth_profile_switch_after_reset_unavailable();
&& self.try_auth_profile_switch_for_usage_limit(
matches!(error_kind, RateLimitErrorKind::UsageLimit),
Some(message.as_str()),
);
if profile_fallback_owns_failed_turn {
self.resume_after_usage_limit_reset();
}
Expand Down
5 changes: 4 additions & 1 deletion codex-rs/tui/src/chatwidget/usage_limit_reset/automatic.rs
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,10 @@ impl ChatWidget {
}

pub(super) fn fallback_auth_profile_switch_after_reset_unavailable(&mut self) {
if self.try_auth_profile_switch_after_reset_unavailable() {
// Reached only after a genuine usage-limit failure whose reset recovery is
// unavailable, so the current profile is authoritatively exhausted: switch even
// when the authoritative read has not yet cached the exhausted window.
if self.try_auth_profile_switch_for_usage_limit(/*is_usage_limit*/ true, None) {
return;
}
let retry_delay = self.maybe_schedule_usage_self_heal_retry(
Expand Down
4 changes: 3 additions & 1 deletion codex-rs/tui/src/chatwidget/usage_self_heal.rs
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,9 @@ impl ChatWidget {
}
}

fn parse_usage_limit_reset_timestamp(message: &str) -> Option<DateTime<Utc>> {
pub(in crate::chatwidget) fn parse_usage_limit_reset_timestamp(
message: &str,
) -> Option<DateTime<Utc>> {
let marker = "try again at ";
let start = message.to_ascii_lowercase().find(marker)? + marker.len();
let candidate = message.get(start..)?.trim_start();
Expand Down
Loading