From 3fa65a2a8628480ddd62c29783e683405b3b6ba1 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 19 Aug 2026 03:01:27 +0000 Subject: [PATCH] fix(llm): retry rate-limited (429) batches instead of dropping them Add a bounded rate-limit retry policy (5s, 15s, 30s, 60s, 60s, honoring a numeric Retry-After up to 120s) to both the sync and async LLM batch retry loops, applied even for models with native provider retries. Exhausted retries record LLM_RATE_LIMIT_RETRIES_EXHAUSTED in the inspection ledger. Signed-off-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-Authored-By: Keshav Pradeep --- src/skillspector/inspection_ledger.py | 4 + src/skillspector/llm_analyzer_base.py | 207 +++++++++++++++++++------- tests/nodes/test_llm_analyzer_base.py | 182 ++++++++++++++++++++++ tests/test_inspection_ledger.py | 1 + 4 files changed, 344 insertions(+), 50 deletions(-) diff --git a/src/skillspector/inspection_ledger.py b/src/skillspector/inspection_ledger.py index 0b3cf204..4b6676d8 100644 --- a/src/skillspector/inspection_ledger.py +++ b/src/skillspector/inspection_ledger.py @@ -50,6 +50,7 @@ class LedgerReason(StrEnum): LLM_BATCH_FAILED = "llm_batch_failed" LLM_STRUCTURED_RESPONSE_INVALID = "llm_structured_response_invalid" LLM_CONNECTION_RETRIES_EXHAUSTED = "llm_connection_retries_exhausted" + LLM_RATE_LIMIT_RETRIES_EXHAUSTED = "llm_rate_limit_retries_exhausted" ANALYZER_RUNTIME_ERROR = "analyzer_runtime_error" UNACCOUNTED_WORK = "unaccounted_work" FINDING_ACCOUNTING_ERROR = "finding_accounting_error" @@ -81,6 +82,9 @@ class LedgerReason(StrEnum): "LLM returned a malformed structured response after bounded retries." ), LedgerReason.LLM_CONNECTION_RETRIES_EXHAUSTED: ("LLM connection failed after bounded retries."), + LedgerReason.LLM_RATE_LIMIT_RETRIES_EXHAUSTED: ( + "LLM provider rate limit persisted after bounded retries." + ), LedgerReason.ANALYZER_RUNTIME_ERROR: ("Analyzer failed after beginning applicable work."), LedgerReason.UNACCOUNTED_WORK: ("Planned inspection work has no unique terminal outcome."), LedgerReason.FINDING_ACCOUNTING_ERROR: ( diff --git a/src/skillspector/llm_analyzer_base.py b/src/skillspector/llm_analyzer_base.py index 0fbf8d23..0f24e989 100644 --- a/src/skillspector/llm_analyzer_base.py +++ b/src/skillspector/llm_analyzer_base.py @@ -34,9 +34,11 @@ from dataclasses import dataclass, field from typing import Any, Literal, cast +import httpx from langchain_anthropic import ChatAnthropic from langchain_core.messages import BaseMessage from langchain_openai import ChatOpenAI +from openai import APIStatusError from pydantic import BaseModel, Field, ValidationError, field_validator from skillspector.inspection_ledger import ( @@ -68,7 +70,13 @@ STRUCTURED_RESPONSE_MAX_RETRIES = 3 STRUCTURED_RESPONSE_MAX_ATTEMPTS = STRUCTURED_RESPONSE_MAX_RETRIES + 1 STRUCTURED_RESPONSE_RETRY_DELAYS_SECONDS = API_CONNECTION_RETRY_DELAYS_SECONDS -LLM_BATCH_MAX_ATTEMPTS = STRUCTURED_RESPONSE_MAX_ATTEMPTS + API_CONNECTION_MAX_RETRIES +RATE_LIMIT_MAX_RETRIES = 5 +RATE_LIMIT_RETRY_DELAYS_SECONDS = (5.0, 15.0, 30.0, 60.0, 60.0) +RATE_LIMIT_MAX_DELAY_SECONDS = 120.0 +HTTP_TOO_MANY_REQUESTS = 429 +LLM_BATCH_MAX_ATTEMPTS = ( + STRUCTURED_RESPONSE_MAX_ATTEMPTS + API_CONNECTION_MAX_RETRIES + RATE_LIMIT_MAX_RETRIES +) class _StructuredResponseValidationError(Exception): @@ -80,6 +88,82 @@ def _is_retryable_api_connection_error(exc: BaseException) -> bool: return type(exc).__name__ == "APIConnectionError" +def _is_retryable_rate_limit_error(exc: BaseException) -> bool: + """Return whether *exc* is a provider rate-limit (HTTP 429) rejection. + + Provider SDKs name this exception ``RateLimitError`` (OpenAI, Anthropic and + the OpenAI-compatible endpoints reached through ``ChatOpenAI``); raw HTTP + clients surface it as a 429 status error instead. + """ + if type(exc).__name__ == "RateLimitError": + return True + if isinstance(exc, (APIStatusError, httpx.HTTPStatusError)): + return exc.response.status_code == HTTP_TOO_MANY_REQUESTS + return False + + +def _rate_limit_retry_after_seconds(exc: BaseException) -> float | None: + """Return the provider-requested ``Retry-After`` delay in seconds, if usable.""" + if not isinstance(exc, (APIStatusError, httpx.HTTPStatusError)): + return None + header = exc.response.headers.get("retry-after") + if header is None: + return None + try: + seconds = float(header) + except ValueError: + # Retry-After may also be an HTTP-date; the schedule below covers it. + return None + if seconds <= 0: + return None + return min(seconds, RATE_LIMIT_MAX_DELAY_SECONDS) + + +def _rate_limit_retry_delay(exc: BaseException, retries_used: int) -> float: + """Return the wait before the next attempt after a rate-limit rejection.""" + scheduled = RATE_LIMIT_RETRY_DELAYS_SECONDS[retries_used] + requested = _rate_limit_retry_after_seconds(exc) + return max(scheduled, requested) if requested is not None else scheduled + + +def _resolve_retry_delay( + exc: BaseException, + *, + attempt: int, + connection_retries: int, + rate_limit_retries: int, + uses_native_connection_retries: bool, +) -> tuple[Literal["rate_limit", "connection"], float] | None: + """Return the retry kind and delay for *exc*, or ``None`` to fail the batch. + + Rate-limit rejections are retried even when the chat model has native retry + support: native budgets back off in milliseconds, which a provider quota + ceiling outlasts. + """ + if attempt == LLM_BATCH_MAX_ATTEMPTS: + return None + if _is_retryable_rate_limit_error(exc): + if rate_limit_retries >= len(RATE_LIMIT_RETRY_DELAYS_SECONDS): + return None + return "rate_limit", _rate_limit_retry_delay(exc, rate_limit_retries) + if ( + not _is_retryable_api_connection_error(exc) + or uses_native_connection_retries + or connection_retries >= len(API_CONNECTION_RETRY_DELAYS_SECONDS) + ): + return None + return "connection", API_CONNECTION_RETRY_DELAYS_SECONDS[connection_retries] + + +def _reason_for_batch_exception(exc: BaseException) -> LedgerReason: + """Return the ledger reason recorded for an unrecovered batch failure.""" + if _is_retryable_rate_limit_error(exc): + return LedgerReason.LLM_RATE_LIMIT_RETRIES_EXHAUSTED + if _is_retryable_api_connection_error(exc): + return LedgerReason.LLM_CONNECTION_RETRIES_EXHAUSTED + return LedgerReason.LLM_BATCH_FAILED + + def _uses_native_connection_retries(chat_model: object) -> bool: """Set the common native retry budget and report whether it is available.""" if isinstance(chat_model, ChatOpenAI): @@ -99,9 +183,10 @@ def resolve_max_concurrency() -> int: Defaults to :data:`DEFAULT_MAX_LLM_CONCURRENCY`. Users on rate-limited providers (free tiers with a low RPM) can set it to ``1`` to serialize requests instead of bursting up to 10 in parallel — a burst that otherwise - guarantees 429s, and 429'd batches are dropped from the result (see the - analyzer fan-out below). Invalid values fall back to the default; values - below 1 are clamped to 1. + guarantees 429s. Rate-limited batches are retried with the bounded + schedule in the analyzer fan-out below, so serializing trades wall time + for fewer retries rather than deciding whether a batch is analyzed at all. + Invalid values fall back to the default; values below 1 are clamped to 1. """ raw = os.environ.get("SKILLSPECTOR_MAX_LLM_CONCURRENCY", "").strip() if not raw: @@ -598,9 +683,10 @@ def _invoke_batch(self, batch: Batch, prompt: str) -> tuple[Batch, list]: return batch, self.parse_response(response, batch) def _invoke_batch_with_retries(self, batch: Batch, prompt: str) -> tuple[Batch, list]: - """Run one batch with bounded retries for malformed output and connection failures.""" + """Run one batch with bounded retries for malformed output, 429s and connection failures.""" structured_retries = 0 connection_retries = 0 + rate_limit_retries = 0 for attempt in range(1, LLM_BATCH_MAX_ATTEMPTS + 1): try: return self._invoke_batch(batch, prompt) @@ -621,22 +707,34 @@ def _invoke_batch_with_retries(self, batch: Batch, prompt: str) -> tuple[Batch, ) time.sleep(delay) except Exception as exc: - if ( - not _is_retryable_api_connection_error(exc) - or self._uses_native_connection_retries - or connection_retries >= len(API_CONNECTION_RETRY_DELAYS_SECONDS) - or attempt == LLM_BATCH_MAX_ATTEMPTS - ): - raise - delay = API_CONNECTION_RETRY_DELAYS_SECONDS[connection_retries] - connection_retries += 1 - logger.warning( - "LLM connection failed for %s; retrying in %.2fs (%d/%d)", - batch.file_label, - delay, - connection_retries, - API_CONNECTION_MAX_RETRIES, + decision = _resolve_retry_delay( + exc, + attempt=attempt, + connection_retries=connection_retries, + rate_limit_retries=rate_limit_retries, + uses_native_connection_retries=self._uses_native_connection_retries, ) + if decision is None: + raise + kind, delay = decision + if kind == "rate_limit": + rate_limit_retries += 1 + logger.warning( + "LLM provider rate limited %s; retrying in %.2fs (%d/%d)", + batch.file_label, + delay, + rate_limit_retries, + RATE_LIMIT_MAX_RETRIES, + ) + else: + connection_retries += 1 + logger.warning( + "LLM connection failed for %s; retrying in %.2fs (%d/%d)", + batch.file_label, + delay, + connection_retries, + API_CONNECTION_MAX_RETRIES, + ) time.sleep(delay) raise AssertionError("bounded retry loop must return or raise") @@ -664,9 +762,10 @@ async def _ainvoke_batch(self, batch: Batch, prompt: str) -> tuple[Batch, list]: return batch, self.parse_response(response, batch) async def _ainvoke_batch_with_retries(self, batch: Batch, prompt: str) -> tuple[Batch, list]: - """Asynchronously run one batch with bounded malformed-output and connection retries.""" + """Asynchronously run one batch with bounded malformed-output, 429 and connection retries.""" structured_retries = 0 connection_retries = 0 + rate_limit_retries = 0 for attempt in range(1, LLM_BATCH_MAX_ATTEMPTS + 1): try: return await self._ainvoke_batch(batch, prompt) @@ -687,22 +786,34 @@ async def _ainvoke_batch_with_retries(self, batch: Batch, prompt: str) -> tuple[ ) await asyncio.sleep(delay) except Exception as exc: - if ( - not _is_retryable_api_connection_error(exc) - or self._uses_native_connection_retries - or connection_retries >= len(API_CONNECTION_RETRY_DELAYS_SECONDS) - or attempt == LLM_BATCH_MAX_ATTEMPTS - ): - raise - delay = API_CONNECTION_RETRY_DELAYS_SECONDS[connection_retries] - connection_retries += 1 - logger.warning( - "LLM connection failed for %s; retrying in %.2fs (%d/%d)", - batch.file_label, - delay, - connection_retries, - API_CONNECTION_MAX_RETRIES, + decision = _resolve_retry_delay( + exc, + attempt=attempt, + connection_retries=connection_retries, + rate_limit_retries=rate_limit_retries, + uses_native_connection_retries=self._uses_native_connection_retries, ) + if decision is None: + raise + kind, delay = decision + if kind == "rate_limit": + rate_limit_retries += 1 + logger.warning( + "LLM provider rate limited %s; retrying in %.2fs (%d/%d)", + batch.file_label, + delay, + rate_limit_retries, + RATE_LIMIT_MAX_RETRIES, + ) + else: + connection_retries += 1 + logger.warning( + "LLM connection failed for %s; retrying in %.2fs (%d/%d)", + batch.file_label, + delay, + connection_retries, + API_CONNECTION_MAX_RETRIES, + ) await asyncio.sleep(delay) raise AssertionError("bounded retry loop must return or raise") @@ -755,11 +866,7 @@ def run_batches_detailed( BatchFailure( batch=batch, error_class=type(exc).__name__, - reason=( - LedgerReason.LLM_CONNECTION_RETRIES_EXHAUSTED - if _is_retryable_api_connection_error(exc) - else LedgerReason.LLM_BATCH_FAILED - ), + reason=_reason_for_batch_exception(exc), ) ) return outcome @@ -788,12 +895,16 @@ async def arun_batches( Anthropic chat models use their native three-retry policy instead; native retry timing remains provider-managed. Unrecovered errors cost only their own batch and are omitted from the result. + Rate-limit rejections (HTTP 429) receive five bounded retries (5s, 15s, + 30s, then 60s twice, or the provider's ``Retry-After`` when it asks for + longer, capped at 120s) regardless of native retry support, because a + provider quota ceiling outlasts the sub-second native budget. Malformed structured responses (Pydantic ``ValidationError`` or CLI JSON parse failures) receive three bounded exponential-backoff retries - and are then isolated to their batch. A batch makes at most seven outer - chat-model invocations even when both - retry policies apply; native provider retries can make additional HTTP - requests within one invocation. + and are then isolated to their batch. A batch makes at most twelve + outer chat-model invocations even when all retry policies apply; native + provider retries can make additional HTTP requests within one + invocation. Callers can detect partial results by comparing the returned batches against the submitted ones. Other ``ValueError`` instances and ``NotImplementedError`` signal misconfiguration rather than infra trouble @@ -849,11 +960,7 @@ async def _process(batch: Batch) -> tuple[Batch, list]: BatchFailure( batch=batch, error_class=type(result).__name__, - reason=( - LedgerReason.LLM_CONNECTION_RETRIES_EXHAUSTED - if _is_retryable_api_connection_error(result) - else LedgerReason.LLM_BATCH_FAILED - ), + reason=_reason_for_batch_exception(result), ) ) continue diff --git a/tests/nodes/test_llm_analyzer_base.py b/tests/nodes/test_llm_analyzer_base.py index a6fb0dae..8803aab4 100644 --- a/tests/nodes/test_llm_analyzer_base.py +++ b/tests/nodes/test_llm_analyzer_base.py @@ -25,12 +25,15 @@ from langchain_anthropic import ChatAnthropic from langchain_core.messages import AIMessage from langchain_openai import ChatOpenAI +from openai import RateLimitError as OpenAIRateLimitError from pydantic import ValidationError from skillspector.inspection_ledger import LedgerOutcome, LedgerReason, finalize_ledger from skillspector.llm_analyzer_base import ( API_CONNECTION_MAX_RETRIES, DEFAULT_MAX_LLM_CONCURRENCY, + RATE_LIMIT_MAX_DELAY_SECONDS, + RATE_LIMIT_RETRY_DELAYS_SECONDS, Batch, BatchExecutionResult, BatchFailure, @@ -211,6 +214,23 @@ class APIConnectionError(Exception): """Test double matching the provider exception name used by the retry policy.""" +class RateLimitError(Exception): + """Test double for a non-OpenAI provider SDK's rate-limit exception.""" + + +def _openai_rate_limit_error(retry_after: str | None = None) -> OpenAIRateLimitError: + """Build the HTTP 429 error the OpenAI-compatible clients raise.""" + response = httpx.Response( + 429, + headers={"retry-after": retry_after} if retry_after is not None else {}, + request=httpx.Request("POST", "https://provider.test/v1/chat/completions"), + ) + return OpenAIRateLimitError("rate limit reached", response=response, body=None) + + +RATE_LIMIT_SLEEP_SCHEDULE = [((delay,), {}) for delay in RATE_LIMIT_RETRY_DELAYS_SECONDS] + + class _RawTextAnalyzer(LLMAnalyzerBase): """Test analyzer for raw-string mode.""" @@ -726,6 +746,107 @@ def test_api_connection_error_isolated_after_four_attempts(self, sleep: MagicMoc ((2.0,), {}), ] + @patch(MOCK_PATCH_TARGET, _mock_get_chat_model) + @patch("skillspector.llm_analyzer_base.time.sleep") + def test_rate_limit_error_recovers_with_bounded_backoff(self, sleep: MagicMock) -> None: + analyzer = LLMAnalyzerBase(base_prompt="test", model=self.MODEL) + analyzer._structured_llm.invoke = MagicMock( + side_effect=[_openai_rate_limit_error(), LLMAnalysisResult(findings=[])] + ) + + outcome = analyzer.run_batches_detailed([Batch(file_path="a.py", content="code")]) + + assert len(outcome.successful) == 1 + assert outcome.failures == [] + assert analyzer._structured_llm.invoke.call_count == 2 + sleep.assert_called_once_with(RATE_LIMIT_RETRY_DELAYS_SECONDS[0]) + + @patch(MOCK_PATCH_TARGET, _mock_get_chat_model) + @patch("skillspector.llm_analyzer_base.time.sleep") + def test_rate_limit_error_waits_for_provider_retry_after(self, sleep: MagicMock) -> None: + analyzer = LLMAnalyzerBase(base_prompt="test", model=self.MODEL) + analyzer._structured_llm.invoke = MagicMock( + side_effect=[_openai_rate_limit_error(retry_after="42"), LLMAnalysisResult(findings=[])] + ) + + analyzer.run_batches_detailed([Batch(file_path="a.py", content="code")]) + + sleep.assert_called_once_with(42.0) + + @patch(MOCK_PATCH_TARGET, _mock_get_chat_model) + @patch("skillspector.llm_analyzer_base.time.sleep") + def test_rate_limit_retry_after_is_capped(self, sleep: MagicMock) -> None: + analyzer = LLMAnalyzerBase(base_prompt="test", model=self.MODEL) + analyzer._structured_llm.invoke = MagicMock( + side_effect=[ + _openai_rate_limit_error(retry_after="3600"), + LLMAnalysisResult(findings=[]), + ] + ) + + analyzer.run_batches_detailed([Batch(file_path="a.py", content="code")]) + + sleep.assert_called_once_with(RATE_LIMIT_MAX_DELAY_SECONDS) + + @patch(MOCK_PATCH_TARGET, _mock_get_chat_model) + @patch("skillspector.llm_analyzer_base.time.sleep") + def test_rate_limit_error_from_other_provider_sdk_is_retried(self, sleep: MagicMock) -> None: + analyzer = LLMAnalyzerBase(base_prompt="test", model=self.MODEL) + analyzer._structured_llm.invoke = MagicMock( + side_effect=[RateLimitError("429 too many requests"), LLMAnalysisResult(findings=[])] + ) + + outcome = analyzer.run_batches_detailed([Batch(file_path="a.py", content="code")]) + + assert len(outcome.successful) == 1 + sleep.assert_called_once_with(RATE_LIMIT_RETRY_DELAYS_SECONDS[0]) + + @patch(MOCK_PATCH_TARGET) + @patch("skillspector.llm_analyzer_base.time.sleep") + def test_native_openai_rate_limits_are_still_retried_by_coordinator( + self, sleep: MagicMock, get_chat_model: MagicMock + ) -> None: + """Native retry budgets back off in milliseconds, so 429s need the longer schedule.""" + get_chat_model.return_value = ChatOpenAI(model=self.MODEL, api_key="sk-test") + analyzer = LLMAnalyzerBase(base_prompt="test", model=self.MODEL) + analyzer._invoke_batch = MagicMock( + side_effect=[ + _openai_rate_limit_error(), + (Batch(file_path="a.py", content="code"), []), + ] + ) + + outcome = analyzer.run_batches_detailed([Batch(file_path="a.py", content="code")]) + + assert len(outcome.successful) == 1 + assert outcome.failures == [] + sleep.assert_called_once_with(RATE_LIMIT_RETRY_DELAYS_SECONDS[0]) + + @patch(MOCK_PATCH_TARGET, _mock_get_chat_model) + @patch("skillspector.llm_analyzer_base.time.sleep") + def test_rate_limit_error_isolated_after_retry_schedule(self, sleep: MagicMock) -> None: + analyzer = LLMAnalyzerBase(base_prompt="test", model=self.MODEL) + analyzer._structured_llm.invoke = MagicMock( + side_effect=[ + *[_openai_rate_limit_error() for _ in RATE_LIMIT_RETRY_DELAYS_SECONDS], + _openai_rate_limit_error(), + LLMAnalysisResult(findings=[]), + ] + ) + + outcome = analyzer.run_batches_detailed( + [ + Batch(file_path="failed.py", content="code"), + Batch(file_path="clean.py", content="code"), + ] + ) + + assert [batch.file_path for batch, _ in outcome.successful] == ["clean.py"] + assert [(failure.batch.file_path, failure.reason) for failure in outcome.failures] == [ + ("failed.py", LedgerReason.LLM_RATE_LIMIT_RETRIES_EXHAUSTED) + ] + assert sleep.call_args_list == RATE_LIMIT_SLEEP_SCHEDULE + @patch(MOCK_PATCH_TARGET, _mock_get_chat_model) @patch("skillspector.llm_analyzer_base.time.sleep") def test_structured_error_then_connection_errors_keeps_both_retry_policies( @@ -1006,6 +1127,67 @@ async def test_api_connection_error_isolated_after_four_attempts( ((2.0,), {}), ] + @patch(MOCK_PATCH_TARGET, _mock_get_chat_model) + @patch("skillspector.llm_analyzer_base.asyncio.sleep", new_callable=AsyncMock) + async def test_rate_limit_error_recovers_with_bounded_backoff(self, sleep: AsyncMock) -> None: + analyzer = LLMAnalyzerBase(base_prompt="test", model=self.MODEL) + analyzer._structured_llm.ainvoke = AsyncMock( + side_effect=[_openai_rate_limit_error(retry_after="7"), LLMAnalysisResult(findings=[])] + ) + + outcome = await analyzer.arun_batches_detailed([Batch(file_path="a.py", content="code")]) + + assert len(outcome.successful) == 1 + assert outcome.failures == [] + assert analyzer._structured_llm.ainvoke.call_count == 2 + sleep.assert_awaited_once_with(7.0) + + @patch(MOCK_PATCH_TARGET) + @patch("skillspector.llm_analyzer_base.asyncio.sleep", new_callable=AsyncMock) + async def test_native_openai_rate_limits_are_still_retried_by_coordinator( + self, sleep: AsyncMock, get_chat_model: MagicMock + ) -> None: + get_chat_model.return_value = ChatOpenAI(model=self.MODEL, api_key="sk-test") + analyzer = LLMAnalyzerBase(base_prompt="test", model=self.MODEL) + analyzer._ainvoke_batch = AsyncMock( + side_effect=[ + _openai_rate_limit_error(), + (Batch(file_path="a.py", content="code"), []), + ] + ) + + outcome = await analyzer.arun_batches_detailed([Batch(file_path="a.py", content="code")]) + + assert len(outcome.successful) == 1 + assert outcome.failures == [] + sleep.assert_awaited_once_with(RATE_LIMIT_RETRY_DELAYS_SECONDS[0]) + + @patch(MOCK_PATCH_TARGET, _mock_get_chat_model) + @patch("skillspector.llm_analyzer_base.asyncio.sleep", new_callable=AsyncMock) + async def test_rate_limit_error_isolated_after_retry_schedule(self, sleep: AsyncMock) -> None: + analyzer = LLMAnalyzerBase(base_prompt="test", model=self.MODEL) + analyzer._structured_llm.ainvoke = AsyncMock( + side_effect=[ + *[_openai_rate_limit_error() for _ in RATE_LIMIT_RETRY_DELAYS_SECONDS], + _openai_rate_limit_error(), + LLMAnalysisResult(findings=[]), + ] + ) + + outcome = await analyzer.arun_batches_detailed( + [ + Batch(file_path="failed.py", content="code"), + Batch(file_path="clean.py", content="code"), + ], + max_concurrency=1, + ) + + assert [batch.file_path for batch, _ in outcome.successful] == ["clean.py"] + assert [(failure.batch.file_path, failure.reason) for failure in outcome.failures] == [ + ("failed.py", LedgerReason.LLM_RATE_LIMIT_RETRIES_EXHAUSTED) + ] + assert sleep.await_args_list == RATE_LIMIT_SLEEP_SCHEDULE + @patch(MOCK_PATCH_TARGET, _mock_get_chat_model) @patch("skillspector.llm_analyzer_base.asyncio.sleep", new_callable=AsyncMock) async def test_structured_error_then_connection_errors_keeps_both_retry_policies( diff --git a/tests/test_inspection_ledger.py b/tests/test_inspection_ledger.py index e8d73cc9..dd7adcd1 100644 --- a/tests/test_inspection_ledger.py +++ b/tests/test_inspection_ledger.py @@ -75,6 +75,7 @@ def test_analyzer_status_for_events_summarizes_terminal_work() -> None: (LedgerReason.LLM_STRUCTURED_RESPONSE_INVALID, LedgerOutcome.SKIPPED), (LedgerReason.LLM_BATCH_FAILED, LedgerOutcome.FAILED), (LedgerReason.LLM_CONNECTION_RETRIES_EXHAUSTED, LedgerOutcome.FAILED), + (LedgerReason.LLM_RATE_LIMIT_RETRIES_EXHAUSTED, LedgerOutcome.FAILED), ], ) def test_outcome_for_llm_batch_failure_preserves_failure_policy(