diff --git a/src/app/v1/_lib/proxy/errors.ts b/src/app/v1/_lib/proxy/errors.ts index 4581a78c8..b84f74a16 100644 --- a/src/app/v1/_lib/proxy/errors.ts +++ b/src/app/v1/_lib/proxy/errors.ts @@ -576,6 +576,24 @@ function extractErrorContentForDetection(error: Error): string { return error.message; } +const PROVIDER_LOCAL_MODEL_UNAVAILABLE_MARKER = + "not supported by any configured account in this group"; + +/** A model capability gap local to one upstream account pool, not to the request. */ +export function isProviderLocalModelUnavailableError(error: unknown): error is ProxyError { + if ( + !(error instanceof ProxyError) || + error.statusCode !== 404 || + error.upstreamError?.statusCodeInferred === true + ) { + return false; + } + + return [error.message, error.upstreamError?.body].some((content) => + content?.toLowerCase().includes(PROVIDER_LOCAL_MODEL_UNAVAILABLE_MARKER) + ); +} + /** * 错误规则检测结果缓存 * @@ -1001,6 +1019,13 @@ export async function categorizeErrorAsync(error: Error): Promise return ErrorCategory.SYSTEM_ERROR; } + // Some upstream account pools use model_not_found for a provider-local capability + // gap. The request may still succeed on another Provider, so classify this exact + // 404 before broad client-input rules match the generic model_not_found wording. + if (isProviderLocalModelUnavailableError(error)) { + return ErrorCategory.RESOURCE_NOT_FOUND; + } + // Some upstream relays report their own storage-capacity protection as HTTP 400. // Classify this known provider failure before broad client-error rules can match text // such as "invalid request" in the same response body. diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index e389be940..b0d60e50f 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -75,6 +75,7 @@ import { isClientAbortError, isEmptyResponseError, isHttp2Error, + isProviderLocalModelUnavailableError, isRetryableUpstreamStorageCapacityError, isSSLCertificateError, ProxyError, @@ -969,7 +970,10 @@ async function tryApplyReactiveRectifier(params: { retryAttemptNumber: number; retryState: ReactiveRectifierRetryState; }): Promise { - if (isRetryableUpstreamStorageCapacityError(params.error)) { + if ( + isProviderLocalModelUnavailableError(params.error) || + isRetryableUpstreamStorageCapacityError(params.error) + ) { return { matched: false }; } @@ -2054,10 +2058,12 @@ export class ProxyForwarder { break; // ⭐ 跳出内层循环，进入供应商切换逻辑 } - // ⭐ 5. 上游 404 错误处理（不计入熔断器，先重试当前供应商，重试耗尽后切换） + // 5. 上游 404 错误处理（不计入熔断器；Provider 局部模型缺口直接切换） if (errorCategory === ErrorCategory.RESOURCE_NOT_FOUND) { const proxyError = lastError as ProxyError; - const willRetry = attemptCount < maxAttemptsPerProvider; + const providerLocalModelUnavailable = isProviderLocalModelUnavailableError(proxyError); + const willRetry = + !providerLocalModelUnavailable && attemptCount < maxAttemptsPerProvider; logger.warn("ProxyForwarder: Upstream 404 error", { providerId: currentProvider.id, @@ -2068,6 +2074,7 @@ export class ProxyForwarder { attemptNumber: attemptCount, totalProvidersAttempted, willRetry, + providerLocalModelUnavailable, }); // 记录到决策链（标记为 resource_not_found，不计入熔断） diff --git a/tests/unit/proxy/provider-local-model-unavailable.test.ts b/tests/unit/proxy/provider-local-model-unavailable.test.ts new file mode 100644 index 000000000..2746b1a61 --- /dev/null +++ b/tests/unit/proxy/provider-local-model-unavailable.test.ts @@ -0,0 +1,135 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { + ErrorCategory, + ProxyError, + categorizeErrorAsync, + isProviderLocalModelUnavailableError, +} from "@/app/v1/_lib/proxy/errors"; +import type { ErrorDetectionResult } from "@/lib/error-rule-detector"; + +const mocks = vi.hoisted(() => ({ + detectAsync: vi.fn<(content: string) => Promise>(), +})); + +vi.mock("@/lib/error-rule-detector", () => ({ + errorRuleDetector: { detectAsync: mocks.detectAsync }, +})); + +describe("provider-local model availability errors", () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.detectAsync.mockResolvedValue({ matched: true }); + }); + + it("switches Provider when an account group does not support the requested model", async () => { + const body = JSON.stringify({ + error: { + message: 'Model "gpt-5.6-sol" is not supported by any configured account in this group', + type: "model_not_found", + }, + }); + const response = new Response(body, { + status: 404, + headers: { "content-type": "application/json" }, + }); + const error = await ProxyError.fromUpstreamResponse(response, { + id: 96, + name: "provider-a", + }); + + expect(isProviderLocalModelUnavailableError(error)).toBe(true); + expect(await categorizeErrorAsync(error)).toBe(ErrorCategory.RESOURCE_NOT_FOUND); + expect(mocks.detectAsync).not.toHaveBeenCalled(); + }); + + it("matches the provider-local marker case-insensitively in the extracted message", async () => { + const error = new ProxyError( + "MODEL IS NOT SUPPORTED BY ANY CONFIGURED ACCOUNT IN THIS GROUP", + 404, + { + body: "{}", + providerId: 96, + providerName: "provider-a", + } + ); + + expect(isProviderLocalModelUnavailableError(error)).toBe(true); + expect(await categorizeErrorAsync(error)).toBe(ErrorCategory.RESOURCE_NOT_FOUND); + expect(mocks.detectAsync).not.toHaveBeenCalled(); + }); + + it("does not use the raw upstream body for provider-local matching", async () => { + const error = new ProxyError("model_not_found", 404, { + body: "{}", + rawBody: "not supported by any configured account in this group", + providerId: 96, + providerName: "provider-a", + }); + + expect(isProviderLocalModelUnavailableError(error)).toBe(false); + expect(await categorizeErrorAsync(error)).toBe(ErrorCategory.NON_RETRYABLE_CLIENT_ERROR); + expect(mocks.detectAsync).toHaveBeenCalledOnce(); + }); + + it("keeps ordinary model_not_found responses non-retryable", async () => { + const error = new ProxyError("The requested model was not found", 404, { + body: JSON.stringify({ + error: { + code: "model_not_found", + message: "The requested model was not found", + }, + }), + providerId: 96, + providerName: "provider-a", + }); + + expect(isProviderLocalModelUnavailableError(error)).toBe(false); + expect(await categorizeErrorAsync(error)).toBe(ErrorCategory.NON_RETRYABLE_CLIENT_ERROR); + expect(mocks.detectAsync).toHaveBeenCalledOnce(); + }); + + it("does not match a near-miss account-group message", async () => { + const error = new ProxyError( + "Model is not supported by every configured account in this group", + 404, + { + body: '{"error":"model_not_found"}', + providerId: 96, + providerName: "provider-a", + } + ); + + expect(isProviderLocalModelUnavailableError(error)).toBe(false); + expect(await categorizeErrorAsync(error)).toBe(ErrorCategory.NON_RETRYABLE_CLIENT_ERROR); + expect(mocks.detectAsync).toHaveBeenCalledOnce(); + }); + + it("does not override the same wording on a non-404 client error", async () => { + const error = new ProxyError( + "Model is not supported by any configured account in this group", + 400, + { + body: "invalid request", + providerId: 96, + providerName: "provider-a", + } + ); + + expect(isProviderLocalModelUnavailableError(error)).toBe(false); + expect(await categorizeErrorAsync(error)).toBe(ErrorCategory.NON_RETRYABLE_CLIENT_ERROR); + expect(mocks.detectAsync).toHaveBeenCalledOnce(); + }); + + it("does not override a synthetic 404 inferred from a fake-200 response", async () => { + const error = new ProxyError("not supported by any configured account in this group", 404, { + body: '{"error":"model_not_found"}', + statusCodeInferred: true, + providerId: 96, + providerName: "provider-a", + }); + + expect(isProviderLocalModelUnavailableError(error)).toBe(false); + expect(await categorizeErrorAsync(error)).toBe(ErrorCategory.NON_RETRYABLE_CLIENT_ERROR); + expect(mocks.detectAsync).toHaveBeenCalledOnce(); + }); +}); diff --git a/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts b/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts index 835dd4eff..bfa2dc845 100644 --- a/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts +++ b/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts @@ -1558,6 +1558,82 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { } }); + test("provider-local model 404 should not cancel another in-flight hedge candidate", async () => { + vi.useFakeTimers(); + + try { + const provider1 = createProvider({ id: 1, name: "p1", firstByteTimeoutStreamingMs: 100 }); + const provider2 = createProvider({ id: 2, name: "p2", firstByteTimeoutStreamingMs: 100 }); + const session = createSession(); + session.setProvider(provider1); + + mocks.pickRandomProviderWithExclusion + .mockResolvedValueOnce(provider2) + .mockResolvedValueOnce(null); + mocks.categorizeErrorAsync.mockResolvedValue(ProxyErrorCategory.RESOURCE_NOT_FOUND); + + const doForward = vi.spyOn( + ProxyForwarder as unknown as { + doForward: (...args: unknown[]) => Promise; + }, + "doForward" + ); + const controller1 = new AbortController(); + const controller2 = new AbortController(); + const providerLocal404 = new UpstreamProxyError( + 'Model "gpt-5.6-sol" is not supported by any configured account in this group', + 404, + { + body: '{"error":{"type":"model_not_found","message":"invalid request: not supported by any configured account in this group"}}', + providerId: provider1.id, + providerName: provider1.name, + } + ); + + doForward.mockImplementationOnce(async (attemptSession) => { + const runtime = attemptSession as ProxySession & AttemptRuntime; + runtime.responseController = controller1; + runtime.clearResponseTimeout = vi.fn(); + return createDelayedFailure({ + delayMs: 150, + error: providerLocal404, + controller: controller1, + }); + }); + doForward.mockImplementationOnce(async (attemptSession) => { + const runtime = attemptSession as ProxySession & AttemptRuntime; + runtime.responseController = controller2; + runtime.clearResponseTimeout = vi.fn(); + return createStreamingResponse({ + label: "p2", + firstChunkDelayMs: 80, + controller: controller2, + }); + }); + + const responsePromise = ProxyForwarder.send(session); + + await vi.advanceTimersByTimeAsync(100); + expect(doForward).toHaveBeenCalledTimes(2); + await vi.advanceTimersByTimeAsync(50); + expect(controller2.signal.aborted).toBe(false); + await vi.advanceTimersByTimeAsync(30); + + const response = await responsePromise; + expect(await response.text()).toContain('"provider":"p2"'); + expect(controller2.signal.aborted).toBe(false); + expect(mocks.recordFailure).not.toHaveBeenCalledWith(provider1.id, providerLocal404); + expect(session.getProviderChain()).toEqual( + expect.arrayContaining([ + expect.objectContaining({ id: provider1.id, reason: "resource_not_found" }), + expect.objectContaining({ id: provider2.id, reason: "hedge_winner" }), + ]) + ); + } finally { + vi.useRealTimers(); + } + }); + test.each([ { name: "provider error", @@ -1570,14 +1646,18 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { }), }, { - name: "resource not found", + name: "provider-local resource not found", category: ProxyErrorCategory.RESOURCE_NOT_FOUND, errorFactory: (provider: Provider) => - new UpstreamProxyError("Provider returned 404: model not found", 404, { - body: '{"error":"model_not_found"}', - providerId: provider.id, - providerName: provider.name, - }), + new UpstreamProxyError( + 'Model "gpt-5.6-sol" is not supported by any configured account in this group', + 404, + { + body: '{"error":{"type":"model_not_found"}}', + providerId: provider.id, + providerName: provider.name, + } + ), }, { name: "system error", diff --git a/tests/unit/proxy/proxy-forwarder-retry-limit.test.ts b/tests/unit/proxy/proxy-forwarder-retry-limit.test.ts index 5f99a68a2..ff51303ba 100644 --- a/tests/unit/proxy/proxy-forwarder-retry-limit.test.ts +++ b/tests/unit/proxy/proxy-forwarder-retry-limit.test.ts @@ -354,6 +354,67 @@ describe("ProxyForwarder - retry limit enforcement", () => { ]); }); + test("provider-local model 404 skips same-provider retries and switches Provider", async () => { + const session = createSession(); + const provider1 = createProvider({ + id: 1, + name: "provider-without-model", + providerVendorId: null, + maxRetryAttempts: 3, + }); + const provider2 = createProvider({ + id: 2, + name: "provider-with-model", + providerVendorId: null, + }); + session.setProvider(provider1); + + mocks.getPreferredProviderEndpoints.mockResolvedValue([]); + vi.mocked(categorizeErrorAsync).mockResolvedValue(ErrorCategory.RESOURCE_NOT_FOUND); + + const doForward = vi.spyOn( + ProxyForwarder as unknown as { doForward: (...args: unknown[]) => unknown }, + "doForward" + ); + const selectAlternative = vi.spyOn( + ProxyForwarder as unknown as { selectAlternative: (...args: unknown[]) => unknown }, + "selectAlternative" + ); + const providerLocal404 = new ProxyError( + 'Model "gpt-5.6-sol" is not supported by any configured account in this group', + 404, + { + body: '{"error":{"type":"model_not_found","message":"invalid request: not supported by any configured account in this group"}}', + providerId: provider1.id, + providerName: provider1.name, + } + ); + + doForward.mockRejectedValueOnce(providerLocal404).mockResolvedValueOnce( + new Response("{}", { + status: 200, + headers: { "content-type": "application/json", "content-length": "2" }, + }) + ); + selectAlternative.mockResolvedValueOnce(provider2); + + const response = await ProxyForwarder.send(session); + + expect(response.status).toBe(200); + expect(doForward).toHaveBeenCalledTimes(2); + expect(selectAlternative).toHaveBeenCalledTimes(1); + expect(selectAlternative).toHaveBeenCalledWith(session, [provider1.id]); + expect(mocks.recordFailure).not.toHaveBeenCalled(); + expect(session.getProviderChain()).toEqual([ + expect.objectContaining({ + id: provider1.id, + reason: "resource_not_found", + attemptNumber: 1, + }), + expect.objectContaining({ id: provider2.id, reason: "retry_success", attemptNumber: 1 }), + ]); + }); + test("upstream storage-capacity 400 should retry, record failure, and switch provider", async () => { vi.useFakeTimers();