Skip to content

fix(oauth): harden xAI Retry-After and abort handling #4060

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
chilung-cgu wants to merge 2 commits into
base: dev
Choose a base branch
from
+413 −95
Draft

fix(oauth): harden xAI Retry-After and abort handling #4060

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7c8448b
fix(oauth): harden xAI Retry-After and abort handling
chilung-cgu Sep 8, 2026
1d19c1a
fix(oauth): keep malformed token bodies non-fatal
chilung-cgu Sep 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 2 additions & 88 deletions src/combos/failover.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { parseResetCooldownMs } from "../codex/routing";
import { classifyError, isCyberPolicyCode } from "../lib/errors";
import { parseRetryAfterMs as parseRetryAfterMsBounded } from "../lib/http-retry-after";
import type { OcxComboTarget } from "../types";
import { targetKey } from "./types";
import {
Expand Down Expand Up @@ -29,13 +30,6 @@ const QUOTA_LIMIT_CODES = new Set([
"insufficient_quota",
]);
const TRANSIENT_REQUEST_RATE_CODES = new Set(["1302", "1305"]);
const IMF_FIXDATE_RE = /^(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun), (\d{2}) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) (\d{4}) (\d{2}):(\d{2}):(\d{2}) GMT$/i;
const RFC850_DATE_RE = /^(?:Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday), (\d{2})-(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-(\d{2}) (\d{2}):(\d{2}):(\d{2}) GMT$/i;
const ASCTIME_DATE_RE = /^(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) ( \d|\d{2}) (\d{2}):(\d{2}):(\d{2}) (\d{4})$/i;
const HTTP_MONTH_INDEX: Record<string, number> = {
jan: 0, feb: 1, mar: 2, apr: 3, may: 4, jun: 5,
jul: 6, aug: 7, sep: 8, oct: 9, nov: 10, dec: 11,
};

/** Map<`${comboId}\0${provider/model}`, TargetCooldown> */
const targetCooldowns = new Map<string, TargetCooldown>();
Expand All @@ -49,92 +43,12 @@ function cooldownMapKey(
return `${comboId}\0${targetKey(target)}`;
}

function parseUtcDateParts(
year: number,
monthName: string,
day: number,
hour: number,
minute: number,
second: number,
): number | undefined {
const month = HTTP_MONTH_INDEX[monthName.toLowerCase()];
if (month === undefined) return undefined;
const timestamp = Date.UTC(year, month, day, hour, minute, second);
const parsed = new Date(timestamp);
return parsed.getUTCFullYear() === year
&& parsed.getUTCMonth() === month
&& parsed.getUTCDate() === day
&& parsed.getUTCHours() === hour
&& parsed.getUTCMinutes() === minute
&& parsed.getUTCSeconds() === second
? timestamp
: undefined;
}

function parseHttpDate(value: string, now: number): number | undefined {
const imf = IMF_FIXDATE_RE.exec(value);
if (imf) {
return parseUtcDateParts(
Number(imf[3]), imf[2]!, Number(imf[1]),
Number(imf[4]), Number(imf[5]), Number(imf[6]),
);
}
const rfc850 = RFC850_DATE_RE.exec(value);
if (rfc850) {
const current = new Date(now);
const currentYear = current.getUTCFullYear();
const month = HTTP_MONTH_INDEX[rfc850[2]!.toLowerCase()];
if (month === undefined) return undefined;
let year = Math.floor(currentYear / 100) * 100 + Number(rfc850[3]);
const yearDelta = year - currentYear;
const candidateTimeOfYear = Date.UTC(
2000, month, Number(rfc850[1]),
Number(rfc850[4]), Number(rfc850[5]), Number(rfc850[6]),
);
const currentTimeOfYear = Date.UTC(
2000, current.getUTCMonth(), current.getUTCDate(),
current.getUTCHours(), current.getUTCMinutes(), current.getUTCSeconds(),
current.getUTCMilliseconds(),
);
if (yearDelta < -50 || (yearDelta === -50 && candidateTimeOfYear < currentTimeOfYear)) {
year += 100;
} else if (yearDelta > 50 || (yearDelta === 50 && candidateTimeOfYear > currentTimeOfYear)) {
year -= 100;
}
return parseUtcDateParts(
year, rfc850[2]!, Number(rfc850[1]),
Number(rfc850[4]), Number(rfc850[5]), Number(rfc850[6]),
);
}
const asctime = ASCTIME_DATE_RE.exec(value);
if (!asctime) return undefined;
return parseUtcDateParts(
Number(asctime[6]), asctime[1]!, Number(asctime[2]),
Number(asctime[3]), Number(asctime[4]), Number(asctime[5]),
);
}

export function parseRetryAfterMs(
value: string | null | undefined,
now = Date.now(),
options?: { preserveImmediate?: boolean },
): number | undefined {
const text = value?.trim();
if (!text) return undefined;
if (/^\d+(?:\.\d+)?$/.test(text)) {
const seconds = Number(text);
if (
Number.isFinite(seconds)
&& (seconds > 0 || (options?.preserveImmediate && seconds === 0))
) {
return Math.min(Math.max(Math.ceil(seconds * 1000), 1), MAX_COOLDOWN_MS);
}
}
const timestamp = parseHttpDate(text, now);
if (timestamp === undefined) return undefined;
const delay = timestamp - now;
if (delay > 0) return Math.min(delay, MAX_COOLDOWN_MS);
return options?.preserveImmediate ? 1 : undefined;
return parseRetryAfterMsBounded(value, now, { ...options, maxMs: MAX_COOLDOWN_MS });
}

export function isComboTargetInCooldown(
Expand Down
96 changes: 96 additions & 0 deletions src/lib/http-retry-after.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
const IMF_FIXDATE_RE = /^(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun), (\d{2}) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) (\d{4}) (\d{2}):(\d{2}):(\d{2}) GMT$/i;
const RFC850_DATE_RE = /^(?:Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday), (\d{2})-(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-(\d{2}) (\d{2}):(\d{2}):(\d{2}) GMT$/i;
const ASCTIME_DATE_RE = /^(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) ( \d|\d{2}) (\d{2}):(\d{2}):(\d{2}) (\d{4})$/i;
const HTTP_MONTH_INDEX: Record<string, number> = {
jan: 0, feb: 1, mar: 2, apr: 3, may: 4, jun: 5,
jul: 6, aug: 7, sep: 8, oct: 9, nov: 10, dec: 11,
};

function parseUtcDateParts(
year: number,
monthName: string,
day: number,
hour: number,
minute: number,
second: number,
): number | undefined {
const month = HTTP_MONTH_INDEX[monthName.toLowerCase()];
if (month === undefined) return undefined;
const timestamp = Date.UTC(year, month, day, hour, minute, second);
const parsed = new Date(timestamp);
return parsed.getUTCFullYear() === year
&& parsed.getUTCMonth() === month
&& parsed.getUTCDate() === day
&& parsed.getUTCHours() === hour
&& parsed.getUTCMinutes() === minute
&& parsed.getUTCSeconds() === second
? timestamp
: undefined;
}

function parseHttpDate(value: string, now: number): number | undefined {
const imf = IMF_FIXDATE_RE.exec(value);
if (imf) {
return parseUtcDateParts(
Number(imf[3]), imf[2]!, Number(imf[1]),
Number(imf[4]), Number(imf[5]), Number(imf[6]),
);
}
const rfc850 = RFC850_DATE_RE.exec(value);
if (rfc850) {
const current = new Date(now);
const currentYear = current.getUTCFullYear();
const month = HTTP_MONTH_INDEX[rfc850[2]!.toLowerCase()];
if (month === undefined) return undefined;
let year = Math.floor(currentYear / 100) * 100 + Number(rfc850[3]);
const yearDelta = year - currentYear;
const candidateTimeOfYear = Date.UTC(
2000, month, Number(rfc850[1]),
Number(rfc850[4]), Number(rfc850[5]), Number(rfc850[6]),
);
const currentTimeOfYear = Date.UTC(
2000, current.getUTCMonth(), current.getUTCDate(),
current.getUTCHours(), current.getUTCMinutes(), current.getUTCSeconds(),
current.getUTCMilliseconds(),
);
if (yearDelta < -50 || (yearDelta === -50 && candidateTimeOfYear < currentTimeOfYear)) {
year += 100;
} else if (yearDelta > 50 || (yearDelta === 50 && candidateTimeOfYear > currentTimeOfYear)) {
year -= 100;
}
return parseUtcDateParts(
year, rfc850[2]!, Number(rfc850[1]),
Number(rfc850[4]), Number(rfc850[5]), Number(rfc850[6]),
);
}
const asctime = ASCTIME_DATE_RE.exec(value);
if (!asctime) return undefined;
return parseUtcDateParts(
Number(asctime[6]), asctime[1]!, Number(asctime[2]),
Number(asctime[3]), Number(asctime[4]), Number(asctime[5]),
);
}

export function parseRetryAfterMs(
value: string | null | undefined,
now = Date.now(),
options?: { preserveImmediate?: boolean; maxMs?: number },
): number | undefined {
const text = value?.trim();
if (!text) return undefined;
const maxMs = options?.maxMs ?? 10 * 60_000;
if (/^\d+(?:\.\d+)?$/.test(text)) {
const seconds = Number(text);
if (
Number.isFinite(seconds)
&& (seconds > 0 || (options?.preserveImmediate && seconds === 0))
) {
return Math.min(Math.max(Math.ceil(seconds * 1000), 1), maxMs);
}
}
const timestamp = parseHttpDate(text, now);
if (timestamp === undefined) return undefined;
const delay = timestamp - now;
if (delay > 0) return Math.min(delay, maxMs);
return options?.preserveImmediate ? 1 : undefined;
}
19 changes: 12 additions & 7 deletions src/oauth/xai.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import { OAuthCallbackFlow, type OAuthCallbackFlowOptions } from "./callback-server";
import { generatePKCE } from "./pkce";
import type { LocalTokenImportMode, OAuthController, OAuthCredentials } from "./types";
import { parseRetryAfterMs as parseStrictRetryAfterMs } from "../lib/http-retry-after";

const XAI_OAUTH_ISSUER = "https://auth.x.ai";
export const XAI_OAUTH_DISCOVERY_URL = `${XAI_OAUTH_ISSUER}/.well-known/openid-configuration`;
Expand Down Expand Up @@ -93,25 +94,29 @@ function getTokenIdentity(accessToken: string, idToken: string | undefined): { a
}

export class XaiTokenRequestError extends Error { constructor(public readonly status?:number,public readonly oauthError?:string,message="xAI token request failed",options?:{cause?:unknown}){super(message,options);this.name="XaiTokenRequestError";} }
export interface XaiTokenRetryDeps { sleep?:(ms:number)=>Promise<void>; random?:()=>number }
function isAbortError(error:unknown):boolean{return error instanceof DOMException&&error.name==="AbortError";}
function retryDelay(attempt:number,retryAfter:string|null,random:()=>number):number{const base=attempt===1?100:250,j=Math.round(base*(.75+random()*.5)),seconds=retryAfter!==null&&/^\d+$/.test(retryAfter)?Number(retryAfter):0;return Math.min(2000,Math.max(j,seconds*1000));}
async function readTokenError(response:Response):Promise<XaiTokenRequestError>{let oauthError:string|undefined,detail="";try{const body=await response.json() as {error?:unknown;error_description?:unknown};if(typeof body.error==="string")oauthError=body.error;if(typeof body.error_description==="string")detail=body.error_description;}catch{}const suffix=detail?`: ${detail}`:oauthError?`: ${oauthError}`:"";return new XaiTokenRequestError(response.status,oauthError,`xAI token request failed: ${response.status}${suffix}`);}
export interface XaiTokenRetryDeps { sleep?:(ms:number,signal?:AbortSignal)=>Promise<void>; random?:()=>number }
const MAX_RETRY_DELAY_MS = 60_000;
function parseRetryAfterMs(header:string|null):number|undefined{const trimmed=header?.trim();if(!trimmed)return undefined;if(/^(?:\d+(?:\.\d+)?|\.\d+)$/.test(trimmed)){const seconds=Number(trimmed);return Number.isFinite(seconds)&&seconds>0?Math.min(MAX_RETRY_DELAY_MS,Math.max(1,Math.ceil(seconds*1000))):undefined;}const parsed=parseStrictRetryAfterMs(trimmed,Date.now(),{maxMs:MAX_RETRY_DELAY_MS});return parsed===undefined?undefined:Math.min(MAX_RETRY_DELAY_MS,parsed);}
function retryDelay(attempt:number,retryAfter:string|null,random:()=>number):number{const base=attempt===1?100:250,j=Math.round(base*(.75+random()*.5)),localCap=Math.min(2000,j);const retryAfterMs=parseRetryAfterMs(retryAfter);return retryAfterMs===undefined?localCap:Math.min(MAX_RETRY_DELAY_MS,Math.max(localCap,retryAfterMs));}
function abortError(signal?:AbortSignal):unknown{return signal?.aborted?signal.reason:new DOMException("The operation was aborted.","AbortError");}
function sleep(ms:number,signal?:AbortSignal):Promise<void>{return new Promise((resolve,reject)=>{if(signal?.aborted){reject(abortError(signal));return;}let timer:ReturnType<typeof setTimeout>;const cleanup=()=>signal?.removeEventListener("abort",onAbort);const onAbort=()=>{clearTimeout(timer);cleanup();reject(abortError(signal));};timer=setTimeout(()=>{cleanup();resolve();},ms);signal?.addEventListener("abort",onAbort,{once:true});if(signal?.aborted)onAbort();});}
function waitForRetry(wait:(ms:number,signal?:AbortSignal)=>Promise<void>,ms:number,signal?:AbortSignal):Promise<void>{if(!signal)return wait(ms);return new Promise((resolve,reject)=>{let settled=false;const cleanup=()=>signal.removeEventListener("abort",onAbort);const finish=()=>{if(settled)return;settled=true;cleanup();resolve();};const fail=(error:unknown)=>{if(settled)return;settled=true;cleanup();reject(error);};const onAbort=()=>fail(abortError(signal));signal.addEventListener("abort",onAbort,{once:true});if(signal.aborted){onAbort();return;}let pending:Promise<void>;try{pending=wait(ms,signal);}catch(error){fail(error);return;}void pending.then(finish,fail);});}
async function readTokenError(response:Response):Promise<XaiTokenRequestError>{let oauthError:string|undefined,detail="";try{const body=await response.json() as {error?:unknown;error_description?:unknown};if(typeof body.error==="string")oauthError=body.error;if(typeof body.error_description==="string")detail=body.error_description;}catch(error){if(error instanceof SyntaxError)detail="";}const suffix=detail?`: ${detail}`:oauthError?`: ${oauthError}`:"";return new XaiTokenRequestError(response.status,oauthError,`xAI token request failed: ${response.status}${suffix}`);}
export async function postXaiToken(
tokenEndpoint: string,
body: Record<string, string>,
signal?: AbortSignal, deps:XaiTokenRetryDeps={},
): Promise<XaiTokenPayload> {
const sleep=deps.sleep??(ms=>Bun.sleep(ms)),random=deps.random??Math.random;let last:unknown;
for(let attempt=1;attempt<=3;attempt++){let response:Response;try{response=await fetch(tokenEndpoint, {
const wait=deps.sleep??sleep,random=deps.random??Math.random;let last:unknown;
for(let attempt=1;attempt<=3;attempt++){if(attempt>1&&signal?.aborted)throw abortError(signal);let response:Response;try{response=await fetch(tokenEndpoint, {
method: "POST",
headers: {
Accept: "application/json",
"Content-Type": "application/x-www-form-urlencoded",
},
body: new URLSearchParams(body).toString(),
signal: requestSignal(signal),
});}catch(error){if(isAbortError(error)&&signal?.aborted)throw error;last=error;if(attempt===3)throw new XaiTokenRequestError(undefined,undefined,"xAI token request failed: network error",{cause:error});await sleep(retryDelay(attempt,null,random));continue;}if(response.ok)return await response.json() as XaiTokenPayload;const error=await readTokenError(response);last=error;if(!(response.status===429||response.status>=500)||attempt===3)throw error;await sleep(retryDelay(attempt,response.headers.get("retry-after"),random));}throw last;
});}catch(error){if(signal?.aborted)throw abortError(signal);if(signal?.reason!==undefined&&error===signal.reason)throw error;last=error;if(attempt===3)throw new XaiTokenRequestError(undefined,undefined,"xAI token request failed: network error",{cause:error});await waitForRetry(wait,retryDelay(attempt,null,random),signal);continue;}if(response.ok){if(signal?.aborted)throw abortError(signal);return await response.json() as XaiTokenPayload;}const error=await readTokenError(response);if(signal?.aborted)throw abortError(signal);last=error;if(!(response.status===429||response.status>=500)||attempt===3)throw error;await waitForRetry(wait,retryDelay(attempt,response.headers.get("retry-after"),random),signal);}throw last;
}

function credentialsFromTokenPayload(payload: XaiTokenPayload, refreshFallback = ""): OAuthCredentials {
Expand Down
Loading
Loading