diff --git a/lib/stack/ai/client.py b/lib/stack/ai/client.py index d539f16..52bd570 100644 --- a/lib/stack/ai/client.py +++ b/lib/stack/ai/client.py @@ -190,15 +190,24 @@ def from_env(cls, *, namespace: str | None = None, "No AI endpoint configured — set up AI with 'stack up ai'" ) key = os.environ.get("OPENAI_KEY", "") or "not-needed" - # The SDK default timeout is 600s — a hung local endpoint would - # freeze a caller for ten minutes. 300s is the compromise: a - # 30K-token prefill on a local model (a scanned multi-page PDF - # going into a vision classify) needs several minutes before the - # first token, and 120s cancelled those mid-prefill; past 300s - # it is a stuck server and should fail loudly. + # This is a read timeout, and the call is not streamed, so it + # covers the entire silent wait while the model reads a document + # and generates an answer. Prefill alone, measured against local + # oMLX (Qwen3.5-9B-MLX-4bit) on a dense 14-page PDF: + # + # 1,000 tokens -> 7.6s 15,000 tokens -> 58.4s + # 3,000 tokens -> 20.4s 28,400 tokens -> 163.6s + # 7,500 tokens -> 36.7s + # + # That machine had RAM to spare. A 16GB Mac running photos and + # documents alongside is several times slower, so 120s and then + # 300s were both cancelling healthy work partway through a big + # scan and discarding every minute already spent on it. 900s + # covers ~158K tokens here, past any context this serves, and + # still bounds a genuinely dead endpoint. client = AsyncOpenAI( base_url=url, api_key=key, max_retries=max_retries, - timeout=300.0, + timeout=900.0, ) return cls(client, namespace=namespace, capabilities=capabilities) diff --git a/stacklets/docs/bot/archivist.py b/stacklets/docs/bot/archivist.py index 943f840..f7c11c3 100644 --- a/stacklets/docs/bot/archivist.py +++ b/stacklets/docs/bot/archivist.py @@ -253,15 +253,23 @@ class ArchivistBot(MicroBot): # Filing is the slowest thing any famstack bot does, and the framework # default (180s) is a chat-bot budget, not a document-pipeline one. One - # archived email costs, in sequence: the Paperless upload + OCR wait + # archived document costs, in sequence: the Paperless upload + OCR wait # (up to 120s), a vision classify pass, a layout reformat pass, and the - # entity-enrichment pass, each a full LLM call whose prefill on a local - # model can run into minutes for a long scan. Cancelling at 180s threw - # away work that was still progressing. Eight minutes is a stuck-handler - # guard, not a latency target: the pipeline's own per-step timeouts bound - # normal slowness, and a handler that blows this budget now fails - # visibly (❌ + a threaded notice) and can be retried with 🔁. - HANDLER_TIMEOUT_SECONDS = 480 + # entity-enrichment pass — three separate LLM calls, each paying its own + # prefill before it emits a token. + # + # Those prefills are the whole budget. Measured on a dense 14-page PDF + # against local oMLX, one 28K-token pass took 163s on a machine with RAM + # to spare; a 16GB Mac running photos and documents alongside is several + # times slower. Three such calls plus the OCR wait clears eight minutes + # comfortably, which is why 480s still cancelled long documents and threw + # the work away. + # + # 30 minutes is deliberately far past any healthy filing. It is a + # stuck-handler guard, not a latency target: the pipeline's own per-step + # timeouts bound normal slowness, and a handler that does blow this now + # fails visibly (❌ + a threaded notice) and can be retried with 🔁. + HANDLER_TIMEOUT_SECONDS = 1800 def __init__(self, homeserver, user_id, password, session_dir, **settings): super().__init__(homeserver, user_id, password, session_dir, **settings)