From 5bce63fc309f8509973b199877622b9c35f4009c Mon Sep 17 00:00:00 2001 From: Madeuss Date: Mon, 31 Aug 2026 17:41:23 -0300 Subject: [PATCH 1/2] feat(api): put any model provider behind one port MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The veneer called the Anthropic SDK directly. That was right for getting it working and wrong to keep, because the provider is not decided — some API will be called, which one is open. Everything a model can be asked for now goes through one method: a system prompt, a user prompt, a schema, and back an instance of that schema. Two failures kept apart, because a refusal is a fact about the request that belongs in the evals, while unavailability is plumbing. The coupling worth avoiding was never the import. It is the vocabulary that would spread through everything built between now and the decision — a stance machine that knows what a stop_reason is, retry logic shaped by one vendor's error taxonomy. None of that announces itself until it has to be undone, so there is a test that fails when any module outside firenze.model imports a provider SDK. FakeModel fills any schema with deterministic, obviously-synthetic text, so the game can be built and a front end developed with no key and no bill. It fits schemas, never cases: domain validation rejects a fake veneer because the cast it invents belongs to no mystery. That rejection is the line between "this pipeline runs offline" and "this produces something a player could be shown". The provider defaults to none and raises rather than picking one, and model_name no longer defaults to a vendor's model. A default would be the decision being made quietly by whoever set it. This is affordable because of what the project already decided: nothing in the deduction path asks a model for anything (RN-023, RN-032). A model writes prose and proposes a stance; both come back as a validated schema. --- .env.example | 13 +- CLAUDE.md | 2 + apps/api/src/firenze/cli.py | 6 +- apps/api/src/firenze/config.py | 14 +- apps/api/src/firenze/model/__init__.py | 39 +++++ apps/api/src/firenze/model/anthropic.py | 70 +++++++++ apps/api/src/firenze/model/fake.py | 100 ++++++++++++ apps/api/src/firenze/model/port.py | 61 ++++++++ apps/api/src/firenze/veneer/writer.py | 70 ++------- apps/api/tests/test_model_port.py | 143 ++++++++++++++++++ apps/api/tests/test_veneer.py | 58 ++++--- docs/00-plano-de-projeto.md | 7 +- .../0007-one-port-for-any-model-provider.md | 96 ++++++++++++ 13 files changed, 589 insertions(+), 90 deletions(-) create mode 100644 apps/api/src/firenze/model/__init__.py create mode 100644 apps/api/src/firenze/model/anthropic.py create mode 100644 apps/api/src/firenze/model/fake.py create mode 100644 apps/api/src/firenze/model/port.py create mode 100644 apps/api/tests/test_model_port.py create mode 100644 docs/adr/0007-one-port-for-any-model-provider.md diff --git a/.env.example b/.env.example index facb774..835ac4d 100644 --- a/.env.example +++ b/.env.example @@ -10,9 +10,12 @@ FIRENZE_ENVIRONMENT=dev FIRENZE_DATABASE_URL=postgresql+psycopg://firenze:firenze@localhost:5433/firenze FIRENZE_REDIS_URL=redis://localhost:6379/0 -# Provedor de LLM. Nunca commitar valor real. -# O SDK lê ANTHROPIC_API_KEY sozinho — sem prefixo FIRENZE_. -# ANTHROPIC_API_KEY= +# Modelo: anthropic, fake ou none. O provedor ainda não foi escolhido, +# então o padrão é none e o verniz simplesmente não roda (ADR-0007). +# fake -> sem rede, sem chave, texto sintético marcado com [fake] +# anthropic -> exige ANTHROPIC_API_KEY, que o SDK lê sozinho +FIRENZE_MODEL_PROVIDER=none +FIRENZE_MODEL_NAME=claude-haiku-4-5 -# Qual modelo escreve o verniz. É o botão que move o custo por caso. -FIRENZE_VENEER_MODEL=claude-haiku-4-5 +# Nunca commitar valor real. +# ANTHROPIC_API_KEY= diff --git a/CLAUDE.md b/CLAUDE.md index 27b1b75..f3b0465 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,6 +20,8 @@ Next.js (front) + FastAPI/LangGraph (core de IA) + Postgres com pgvector. - Toda saída de LLM é validada por schema Pydantic versionado. Mudou o schema, roda os evals antes de commitar. - Prompts vivem em `prompts/`, versionados, nunca em string literal. +- Modelo só se chama pela porta em `firenze.model`. Nenhum outro módulo + importa SDK de fornecedor. (ADR-0007) - Código em inglês, docs em português, conteúdo do jogo em português. Domínio guarda estrutura; frase pronta só no catálogo. (ADR-0005/0006) - Conventional Commits. Nada direto na main. diff --git a/apps/api/src/firenze/cli.py b/apps/api/src/firenze/cli.py index eb0f2bd..03f2a5d 100644 --- a/apps/api/src/firenze/cli.py +++ b/apps/api/src/firenze/cli.py @@ -18,6 +18,7 @@ from firenze.domain import CaseWithSolution, FactKind, Role from firenze.generation import UnsolvableCase, generate, solve from firenze.i18n import DEFAULT_LOCALE, Catalog, UnknownLocale, available_locales, load +from firenze.model import ModelUnavailable, resolve from firenze.veneer import CaseVeneer, VeneerRejected, VeneerUnavailable, write @@ -111,8 +112,9 @@ def main(argv: Sequence[str] | None = None) -> int: veneer = None if args.veneer: try: - veneer = write(full.case, catalog, model=settings.veneer_model) - except (VeneerUnavailable, VeneerRejected) as failure: + model = resolve(settings.model_provider, settings.model_name) + veneer = write(full.case, catalog, model=model) + except (VeneerUnavailable, VeneerRejected, ModelUnavailable) as failure: # The case is playable without prose. Degrading beats failing. print(f"veneer skipped: {failure}", file=sys.stderr) diff --git a/apps/api/src/firenze/config.py b/apps/api/src/firenze/config.py index aceb17a..5ff7efb 100644 --- a/apps/api/src/firenze/config.py +++ b/apps/api/src/firenze/config.py @@ -13,9 +13,17 @@ class Settings(BaseSettings): environment: Environment = "dev" database_url: str = "postgresql+psycopg://firenze:firenze@localhost:5433/firenze" redis_url: str = "redis://localhost:6379/0" - veneer_model: str = "claude-haiku-4-5" - """Which model writes the veneer. Env-switchable because it is the one - knob that moves cost per case.""" + model_provider: str = "none" + """Which provider backs the model port: anthropic, fake, or none. + + `none` by default, deliberately. The provider is not decided yet, and a + default that picked one would make the decision quietly.""" + + model_name: str = "" + """Which model at that provider. + + Empty by default for the same reason the provider is `none`: naming one + would pick a vendor. Required once a real provider is configured.""" settings = Settings() diff --git a/apps/api/src/firenze/model/__init__.py b/apps/api/src/firenze/model/__init__.py new file mode 100644 index 0000000..659dd5e --- /dev/null +++ b/apps/api/src/firenze/model/__init__.py @@ -0,0 +1,39 @@ +"""Language models, behind one interface. + +`resolve()` is the only place that knows which providers exist. Everything else +depends on the port. +""" + +from firenze.model.anthropic import AnthropicModel +from firenze.model.fake import FakeModel +from firenze.model.port import ModelRefused, ModelUnavailable, StructuredModel + + +def resolve(provider: str, model: str) -> StructuredModel: + """Build the configured model, or raise `ModelUnavailable`. + + `none` is the default and is not an error state to be worked around: the + provider for this project is not chosen yet, and a build that quietly picked + one would be making the decision on the reader's behalf. + """ + if provider == "none": + raise ModelUnavailable( + "no model provider configured; set FIRENZE_MODEL_PROVIDER to one of: anthropic, fake" + ) + if provider == "fake": + return FakeModel() + if provider == "anthropic": + if not model: + raise ModelUnavailable("provider 'anthropic' needs FIRENZE_MODEL_NAME set") + return AnthropicModel(model=model) + raise ModelUnavailable(f"unknown model provider {provider!r}") + + +__all__ = [ + "AnthropicModel", + "FakeModel", + "ModelRefused", + "ModelUnavailable", + "StructuredModel", + "resolve", +] diff --git a/apps/api/src/firenze/model/anthropic.py b/apps/api/src/firenze/model/anthropic.py new file mode 100644 index 0000000..082975b --- /dev/null +++ b/apps/api/src/firenze/model/anthropic.py @@ -0,0 +1,70 @@ +"""Anthropic adapter. + +One of possibly several. Everything provider-specific lives here — the SDK +import, the client construction, the shape of a structured-output call — so that +choosing a different provider is a new file in this package and a line of +configuration, not a change anywhere in the game. + +The provider for this project is not decided yet. This adapter exists because it +was written first and works; it is a reference implementation of the port, not a +commitment. +""" + +from typing import Any, cast + +from firenze.model.port import ModelRefused, ModelUnavailable, Schema + +DEFAULT_MODEL = "claude-haiku-4-5" + + +class AnthropicModel: + """Calls the Anthropic Messages API with a schema as the output format.""" + + def __init__(self, model: str = DEFAULT_MODEL, client: Any | None = None) -> None: + self._model = model + self._client = client + + @property + def name(self) -> str: + return self._model + + def _connect(self) -> Any: + if self._client is not None: + return self._client + try: + import anthropic + except ImportError as missing: # pragma: no cover - the dependency is declared + raise ModelUnavailable("the anthropic sdk is not installed") from missing + try: + self._client = anthropic.Anthropic() + except Exception as failure: + raise ModelUnavailable(f"no usable credentials: {failure}") from failure + return self._client + + def complete( + self, + *, + system: str, + user: str, + schema: type[Schema], + max_tokens: int, + ) -> Schema: + client = self._connect() + try: + response = client.messages.parse( + model=self._model, + max_tokens=max_tokens, + system=system, + messages=[{"role": "user", "content": user}], + output_format=schema, + ) + except Exception as failure: + raise ModelUnavailable(f"the model call failed: {failure}") from failure + + if getattr(response, "stop_reason", None) == "refusal": + raise ModelRefused("the model declined to answer") + + parsed = getattr(response, "parsed_output", None) + if parsed is None: + raise ModelUnavailable("the model returned nothing that fits the schema") + return cast("Schema", parsed) diff --git a/apps/api/src/firenze/model/fake.py b/apps/api/src/firenze/model/fake.py new file mode 100644 index 0000000..4b612de --- /dev/null +++ b/apps/api/src/firenze/model/fake.py @@ -0,0 +1,100 @@ +"""A model that needs no network, no key and no money. + +It exists so the game can be built and played before a provider is chosen, and +so a front end can be developed against a running backend without anyone paying +per keystroke. It fills a schema with deterministic, obviously-synthetic text. + +Two rules keep it honest: + +- **It never pretends to be a real model.** `name` says `fake`, and that string + is recorded on everything it writes. A veneer produced here is traceable as + produced here. +- **It is deterministic.** Same prompt, same output. A fake that varied would + make a failing test look flaky, which is worse than no fake at all. + +It is not a mock of any provider and does not imitate one. It satisfies the +port, nothing more — and note what that means: it produces output that fits a +**schema**, never output that fits a **case**. Domain validation will reject a +fake veneer, because the cast it invents is not the cast of any real mystery, +and that rejection is correct rather than a shortcoming. The fake earns its keep +on pipelines whose correctness does not depend on the content: schema +validation, stance transitions, output filters, turn accounting. +""" + +import hashlib +from enum import Enum +from typing import Any, get_args, get_origin + +from pydantic import BaseModel + +from firenze.model.port import ModelRefused, Schema + +MARKER = "[fake]" + + +class FakeModel: + """Fills any schema with placeholder text derived from the prompt.""" + + def __init__(self, *, refuse: bool = False) -> None: + self._refuse = refuse + + @property + def name(self) -> str: + return "fake" + + def complete( + self, + *, + system: str, + user: str, + schema: type[Schema], + max_tokens: int, + ) -> Schema: + if self._refuse: + raise ModelRefused("the fake model was asked to refuse") + + seed = hashlib.blake2s(f"{system}{user}".encode(), digest_size=4).hexdigest() + return _fill(schema, seed=seed, path="") + + +def _fill(schema: type[Schema], *, seed: str, path: str) -> Schema: + values: dict[str, Any] = {} + for name, field in schema.model_fields.items(): + values[name] = _value_for(field.annotation, seed=seed, path=f"{path}.{name}") + return schema(**values) + + +def _value_for(annotation: Any, *, seed: str, path: str) -> Any: + if annotation is str: + return f"{MARKER} {path.lstrip('.')} {seed}" + if annotation is int: + return 0 + if annotation is float: + return 0.0 + if annotation is bool: + return False + + origin = get_origin(annotation) + args = get_args(annotation) + + if origin in (tuple, list, set, frozenset): + # One element, not zero: an empty collection would pass a schema and + # then fail a validation rule for a reason unrelated to the fake. + inner = args[0] if args else str + item = _value_for(inner, seed=seed, path=f"{path}[0]") + return origin([item]) if origin is not tuple else (item,) + + if isinstance(annotation, type) and issubclass(annotation, Enum): + # The first member, deterministically. A stance machine fed a random + # enum would fail for a reason that has nothing to do with the code. + return next(iter(annotation)) + + if isinstance(annotation, type) and issubclass(annotation, BaseModel): + return _fill(annotation, seed=seed, path=path) + + if args: # Optional[X], X | None, Literal[...] + for candidate in args: + if candidate is not type(None): + return _value_for(candidate, seed=seed, path=path) + + return f"{MARKER} {path.lstrip('.')}" diff --git a/apps/api/src/firenze/model/port.py b/apps/api/src/firenze/model/port.py new file mode 100644 index 0000000..e98fd26 --- /dev/null +++ b/apps/api/src/firenze/model/port.py @@ -0,0 +1,61 @@ +"""The only thing the rest of the codebase knows about language models. + +One method: given a system prompt, a user prompt and a schema, return an +instance of that schema. Everything a provider offers beyond that — streaming +shapes, tool calling, thinking budgets, cache controls — stays behind the +adapter, because the day the provider changes, whatever leaked through this +interface is what has to be rewritten. + +The narrowness is the point, and it is affordable here for a reason specific to +this project: nothing in the deduction path asks a model for anything. The +solver, the validator, the verdict and the scoring are code (RN-023, RN-032). +A model writes prose and proposes a stance, and both arrive as a validated +schema. An application whose business logic depended on tool calling could not +draw this line so tightly. +""" + +from typing import Protocol, TypeVar + +from pydantic import BaseModel + +Schema = TypeVar("Schema", bound=BaseModel) + + +class ModelUnavailable(RuntimeError): + """No usable model: no provider configured, no credentials, transport failed. + + Callers degrade rather than retry. Prose is a luxury; the mystery is not. + """ + + +class ModelRefused(RuntimeError): + """The provider declined to answer. + + Kept separate from unavailability on purpose. A refusal is a fact about the + request — worth surfacing, worth counting in the evals — and burying it in a + generic failure would hide it exactly when it matters. + """ + + +class StructuredModel(Protocol): + """What a provider must supply. Adapters live in this package, not elsewhere.""" + + @property + def name(self) -> str: + """Recorded alongside anything the model wrote, so output can be traced + back to what produced it. A veneer written by a fake must never be + mistaken for one written by a real model.""" + ... + + def complete( + self, + *, + system: str, + user: str, + schema: type[Schema], + max_tokens: int, + ) -> Schema: + """Return an instance of `schema`, or raise `ModelUnavailable` / + `ModelRefused`. Never returns partially valid data: a response that does + not fit the schema is a failure, not a value to repair.""" + ... diff --git a/apps/api/src/firenze/veneer/writer.py b/apps/api/src/firenze/veneer/writer.py index 63ead9e..f2c9bf3 100644 --- a/apps/api/src/firenze/veneer/writer.py +++ b/apps/api/src/firenze/veneer/writer.py @@ -12,24 +12,22 @@ output is not there to catch the model; it is there to catch us, because a canary in this output means context assembly upstream is broken (RN-010, RN-012). -The prompt lives in `prompts/veneer/`, versioned, never as a string literal. +The prompt lives in `prompts/veneer/`, versioned, never as a string literal, +and the model arrives through the port in `firenze.model` — this module has +never heard of a provider (ADR-0007). """ import os import re from pathlib import Path -from typing import Any, Protocol, cast from firenze.domain import Case from firenze.i18n import Catalog +from firenze.model import ModelRefused, ModelUnavailable, StructuredModel from firenze.veneer.models import CaseVeneer, VeneerDraft from firenze.veneer.validation import check PROMPT_VERSION = "v1" -# Haiku: the veneer is short, structured, and its failure modes are caught by -# validation rather than by model quality. ~US$ 0.0025 per case against a -# R$ 0,50 per-match budget that phase 2 will spend on six NPCs and a turn each. -DEFAULT_MODEL = "claude-haiku-4-5" MAX_TOKENS = 4000 @@ -37,17 +35,6 @@ class VeneerUnavailable(RuntimeError): """The veneer could not be produced. The case is still playable without it.""" -class _Parseable(Protocol): - """The slice of the SDK this module uses, so tests can supply their own.""" - - def parse(self, **kwargs: Any) -> Any: ... - - -class _Client(Protocol): - @property - def messages(self) -> _Parseable: ... - - def prompts_dir() -> Path: """Repository `prompts/`, or wherever `FIRENZE_PROMPTS_DIR` points.""" override = os.environ.get("FIRENZE_PROMPTS_DIR") @@ -86,41 +73,26 @@ def _render_prompt(case: Case, catalog: Catalog) -> tuple[str, str]: ) -def write( - case: Case, - catalog: Catalog, - *, - client: _Client | None = None, - model: str = DEFAULT_MODEL, -) -> CaseVeneer: +def write(case: Case, catalog: Catalog, *, model: StructuredModel) -> CaseVeneer: """Write the veneer for an approved case, or raise `VeneerUnavailable`. A rejected draft is discarded rather than repaired: a model that broke the cast list once will break it differently on a patch, and a half-corrected veneer is harder to reason about than none. """ - if client is None: - client = _default_client() - system, user = _render_prompt(case, catalog) try: - response = client.messages.parse( - model=model, - max_tokens=MAX_TOKENS, + draft = model.complete( system=system, - messages=[{"role": "user", "content": user}], - output_format=VeneerDraft, + user=user, + schema=VeneerDraft, + max_tokens=MAX_TOKENS, ) - except Exception as failure: - raise VeneerUnavailable(f"the model call failed: {failure}") from failure - - if getattr(response, "stop_reason", None) == "refusal": - raise VeneerUnavailable("the model declined to write this case") - - draft = response.parsed_output - if draft is None: - raise VeneerUnavailable("the model returned no parseable draft") + except ModelRefused as refusal: + raise VeneerUnavailable(f"the model declined to write this case: {refusal}") from refusal + except ModelUnavailable as unavailable: + raise VeneerUnavailable(str(unavailable)) from unavailable check(draft, case) @@ -130,21 +102,7 @@ def write( setting=case.setting, prompt_version=PROMPT_VERSION, locale=catalog.locale, - model=model, + model=model.name, scene=draft.scene, characters=draft.characters, ) - - -def _default_client() -> _Client: - try: - import anthropic - except ImportError as missing: # pragma: no cover - the dependency is declared - raise VeneerUnavailable("the anthropic sdk is not installed") from missing - - try: - # The SDK's own `parse` signature is narrower than the slice we use; - # the cast is the one place where that difference is acknowledged. - return cast("_Client", anthropic.Anthropic()) - except Exception as failure: - raise VeneerUnavailable(f"no usable credentials: {failure}") from failure diff --git a/apps/api/tests/test_model_port.py b/apps/api/tests/test_model_port.py new file mode 100644 index 0000000..2b73ca2 --- /dev/null +++ b/apps/api/tests/test_model_port.py @@ -0,0 +1,143 @@ +"""Port and adapter tests. + +The contract is one method, so these are short. What they protect is the +boundary: if an adapter starts leaking provider vocabulary through the port, or +the fake starts looking like a real model, the next provider change stops being +a new file and becomes a refactor. +""" + +from typing import Any + +import pytest +from pydantic import BaseModel + +from firenze.domain import Role +from firenze.model import AnthropicModel, FakeModel, ModelRefused, ModelUnavailable, resolve +from firenze.model.fake import MARKER + + +class Nested(BaseModel): + label: str + + +class Answer(BaseModel): + line: str + stance: Role + lied: bool + turns: int + parts: tuple[Nested, ...] + + +def test_the_fake_fills_any_schema() -> None: + answer = FakeModel().complete(system="s", user="u", schema=Answer, max_tokens=100) + + assert MARKER in answer.line + assert answer.stance is Role.victim # first member, deterministically + assert answer.lied is False + assert answer.turns == 0 + assert len(answer.parts) == 1, "collections get one element, never zero" + assert MARKER in answer.parts[0].label + + +def test_the_fake_is_deterministic() -> None: + """A fake that varied would make a failing test look flaky.""" + first = FakeModel().complete(system="s", user="u", schema=Answer, max_tokens=100) + second = FakeModel().complete(system="s", user="u", schema=Answer, max_tokens=100) + + assert first == second + + +def test_the_fake_answers_differently_to_a_different_prompt() -> None: + first = FakeModel().complete(system="s", user="one", schema=Answer, max_tokens=100) + second = FakeModel().complete(system="s", user="two", schema=Answer, max_tokens=100) + + assert first.line != second.line + + +def test_the_fake_never_claims_to_be_a_real_model() -> None: + """Its name is recorded on everything it writes, so output stays traceable.""" + assert FakeModel().name == "fake" + + +def test_the_fake_can_be_asked_to_refuse() -> None: + with pytest.raises(ModelRefused): + FakeModel(refuse=True).complete(system="s", user="u", schema=Answer, max_tokens=10) + + +def test_no_provider_configured_is_an_explicit_failure() -> None: + """Not a silent default. The provider for this project is not chosen yet.""" + with pytest.raises(ModelUnavailable, match="no model provider configured"): + resolve("none", "whatever") + + +def test_an_unknown_provider_says_so() -> None: + with pytest.raises(ModelUnavailable, match="unknown model provider"): + resolve("mistral-via-carrier-pigeon", "x") + + +@pytest.mark.parametrize( + ("provider", "expected"), + [("fake", "fake"), ("anthropic", "claude-haiku-4-5")], +) +def test_resolve_builds_what_it_was_asked_for(provider: str, expected: str) -> None: + assert resolve(provider, "claude-haiku-4-5").name == expected + + +def test_the_anthropic_adapter_translates_a_refusal() -> None: + class Refusing: + def parse(self, **kwargs: Any) -> Any: + return type("R", (), {"stop_reason": "refusal", "parsed_output": None}) + + model = AnthropicModel(client=type("C", (), {"messages": Refusing()})()) + + with pytest.raises(ModelRefused): + model.complete(system="s", user="u", schema=Answer, max_tokens=10) + + +def test_the_anthropic_adapter_translates_a_transport_failure() -> None: + """A provider exception must never escape the adapter as itself.""" + + class Exploding: + def parse(self, **kwargs: Any) -> Any: + raise ConnectionError("no route to host") + + model = AnthropicModel(client=type("C", (), {"messages": Exploding()})()) + + with pytest.raises(ModelUnavailable, match="model call failed"): + model.complete(system="s", user="u", schema=Answer, max_tokens=10) + + +def test_the_anthropic_adapter_rejects_an_unparseable_response() -> None: + class Empty: + def parse(self, **kwargs: Any) -> Any: + return type("R", (), {"stop_reason": "end_turn", "parsed_output": None}) + + model = AnthropicModel(client=type("C", (), {"messages": Empty()})()) + + with pytest.raises(ModelUnavailable, match="nothing that fits the schema"): + model.complete(system="s", user="u", schema=Answer, max_tokens=10) + + +def test_no_module_outside_the_port_imports_a_provider_sdk() -> None: + """The boundary this ADR draws has to survive people who have not read it. + + Provider vocabulary spreading module by module is the failure ADR-0007 + exists to prevent, and it never announces itself — it looks like one + reasonable import at a time. + """ + import pathlib + import re + + source = pathlib.Path(__file__).resolve().parents[1] / "src" / "firenze" + # Naming a provider as a valid config value is not coupling; importing its + # SDK is. So this looks for the import, not for the word. + sdks = "anthropic|openai|google|mistralai|cohere|ollama" + sdk_import = re.compile(rf"^\s*(?:import|from)\s+({sdks})\b", re.MULTILINE) + + offenders = [ + path.relative_to(source).as_posix() + for path in source.rglob("*.py") + if path.parent.name != "model" and sdk_import.search(path.read_text(encoding="utf-8")) + ] + + assert not offenders, f"provider SDK imported outside firenze.model: {offenders}" diff --git a/apps/api/tests/test_veneer.py b/apps/api/tests/test_veneer.py index 3473973..62f1ff7 100644 --- a/apps/api/tests/test_veneer.py +++ b/apps/api/tests/test_veneer.py @@ -14,6 +14,7 @@ from firenze.domain import Case from firenze.generation import generate from firenze.i18n import load +from firenze.model import ModelRefused, ModelUnavailable from firenze.veneer import ( CharacterVeneer, VeneerDraft, @@ -26,20 +27,23 @@ from firenze.veneer.writer import _render_prompt -class StubMessages: - def __init__(self, draft: VeneerDraft | None, stop_reason: str = "end_turn") -> None: - self.draft = draft - self.stop_reason = stop_reason - self.calls: list[dict[str, Any]] = [] +class ScriptedModel: + """A model that returns exactly what a test needs it to return.""" - def parse(self, **kwargs: Any) -> Any: - self.calls.append(kwargs) - return type("Response", (), {"parsed_output": self.draft, "stop_reason": self.stop_reason}) + def __init__(self, draft: VeneerDraft | None = None, failure: Exception | None = None) -> None: + self._draft = draft + self._failure = failure + self.calls: list[dict[str, Any]] = [] + @property + def name(self) -> str: + return "scripted" -class StubClient: - def __init__(self, draft: VeneerDraft | None, stop_reason: str = "end_turn") -> None: - self.messages = StubMessages(draft, stop_reason) + def complete(self, **kwargs: Any) -> Any: + self.calls.append(kwargs) + if self._failure is not None: + raise self._failure + return self._draft @pytest.fixture(scope="module") @@ -93,13 +97,14 @@ def test_nothing_in_the_prompt_singles_out_the_culprit(case: Case) -> None: def test_a_good_draft_becomes_a_veneer(case: Case) -> None: - client = StubClient(_good_draft(case)) + model = ScriptedModel(_good_draft(case)) - veneer = write(case, load("pt-BR"), client=client, model="stub-model") + veneer = write(case, load("pt-BR"), model=model) assert veneer.seed == case.seed assert veneer.locale == "pt-BR" assert veneer.prompt_version == "v1" + assert veneer.model == "scripted", "output records what wrote it" # Prose is written for one setting. Carrying it keeps a cached veneer from # being reused for a different world that happened to share a seed. assert veneer.setting == case.setting @@ -182,21 +187,30 @@ def test_a_blank_field_is_rejected(case: Case) -> None: def test_a_refusal_is_not_a_crash(case: Case) -> None: - client = StubClient(None, stop_reason="refusal") + model = ScriptedModel(failure=ModelRefused("policy")) with pytest.raises(VeneerUnavailable, match="declined"): - write(case, load("pt-BR"), client=client) + write(case, load("pt-BR"), model=model) -def test_a_transport_failure_becomes_veneer_unavailable(case: Case) -> None: - class Exploding: - def parse(self, **kwargs: Any) -> Any: - raise ConnectionError("no route to host") +def test_an_unavailable_model_is_not_a_crash(case: Case) -> None: + model = ScriptedModel(failure=ModelUnavailable("no route to host")) - client = type("C", (), {"messages": Exploding()})() + with pytest.raises(VeneerUnavailable, match="no route"): + write(case, load("pt-BR"), model=model) + + +def test_a_fake_veneer_is_rejected_by_domain_validation(case: Case) -> None: + """The fake fits the schema and not the case, and that is the right outcome. + + It invents a cast that belongs to no mystery, so `check` refuses it. Worth + asserting: it is the line between "this pipeline runs offline" and "this + pipeline produces something a player could be shown". + """ + from firenze.model import FakeModel - with pytest.raises(VeneerUnavailable, match="model call failed"): - write(case, load("pt-BR"), client=client) + with pytest.raises(VeneerRejected, match="cast"): + write(case, load("pt-BR"), model=FakeModel()) def test_the_prompt_file_is_the_source_of_truth() -> None: diff --git a/docs/00-plano-de-projeto.md b/docs/00-plano-de-projeto.md index ab80dea..434ceca 100644 --- a/docs/00-plano-de-projeto.md +++ b/docs/00-plano-de-projeto.md @@ -177,13 +177,16 @@ ADR escrita para convencer, não para decidir. estrutura, não frase; idioma é propriedade da partida - [`0006`](adr/0006-english-in-code-portuguese-in-the-product.md) — código em inglês, produto em português +- [`0007`](adr/0007-one-port-for-any-model-provider.md) — uma porta para + qualquer fornecedor de modelo, escolha do fornecedor adiada Todas as ADRs são escritas em inglês (ADR-0006) — as quatro primeiras foram traduzidas depois de escritas. **Na fila:** LangGraph vs orquestração própria · streaming SSE vs WebSocket · -API externa vs LLM self-hosted na Magalu · isolamento de contexto como fronteira -de segurança · VM+compose antes de Kubernetes. Numeração sai na ordem em que a +qual fornecedor de modelo (decisão adiada pela ADR-0007, com entrada nos evals +da fase 6) · API externa vs LLM self-hosted na Magalu · isolamento de contexto +como fronteira de segurança · VM+compose antes de Kubernetes. Numeração sai na ordem em que a decisão é tomada, não na ordem desta lista. ### 4.2 C4 (`docs/04-arquitetura.md`) diff --git a/docs/adr/0007-one-port-for-any-model-provider.md b/docs/adr/0007-one-port-for-any-model-provider.md new file mode 100644 index 0000000..4350387 --- /dev/null +++ b/docs/adr/0007-one-port-for-any-model-provider.md @@ -0,0 +1,96 @@ +# ADR-0007: One port for any model provider, and the provider chosen later + +## Status + +Accepted — 2026-08-31 + +## Context + +The veneer was written against the Anthropic SDK directly: the import, the +client, `messages.parse`, `output_format`, `stop_reason == "refusal"`. That was +the right way to get it working and the wrong thing to keep, because the +provider is not decided. Some API will be called; which one is open. + +Building the rest of the game on top of an undecided dependency has two failure +modes. The obvious one is a rewrite when the decision lands. The subtler one is +that provider vocabulary spreads: a stance machine that knows what a +`stop_reason` is, an eval suite that counts tokens the way one vendor reports +them, a turn pipeline whose retry logic assumes one provider's error taxonomy. +None of that announces itself as coupling until the day it has to be undone. + +There is also a practical constraint worth stating: the next phases have to be +built before any key exists. Most of a turn does not need a model — dossier +assembly (RN-010), schema validation (RN-022), the stance machine (RN-023), +canary and scope filtering (RN-042), turn accounting (RN-030) are all +deterministic. Only the prose needs one. + +## Decision + +Everything a model can be asked for goes through one interface in +`firenze.model`: + +```python +def complete(self, *, system: str, user: str, schema: type[Schema], max_tokens: int) -> Schema +``` + +A system prompt, a user prompt, a schema; an instance of that schema, or a +failure. Two failures, kept apart: `ModelUnavailable` (no provider, no +credentials, transport) and `ModelRefused` (the provider declined). A refusal is +a fact about the request and belongs in the evals; burying it in a generic +failure would hide it exactly when it matters. + +Adapters live in the same package and nowhere else. Two exist: + +- **`AnthropicModel`** — the code that was already written, now one + implementation rather than the implementation. Not a commitment. +- **`FakeModel`** — no network, no key, no money. Deterministic, and it names + itself `fake` so anything it wrote is traceable to it. + +`FIRENZE_MODEL_PROVIDER` defaults to `none`, which raises rather than picking +one. A default provider would be the decision being made quietly by whoever set +the default. + +**This interface is affordable because of what this project already decided.** +Nothing in the deduction path asks a model for anything: the solver, the +verdict, the scoring and contradiction detection are code (RN-023, RN-032). A +model writes prose and proposes a stance, and both come back as a validated +schema. An application whose business logic ran through tool calling could not +draw the line this tightly. + +## Consequences + ++ Choosing a provider is a new file in one package plus a line of configuration. ++ The game can be built and played before that choice, and a front end can be + developed against a running backend with no key and no bill. ++ Provider vocabulary cannot spread, because no other module can name it. ++ Failures arrive already classified, so callers degrade on a policy rather than + on a vendor's exception hierarchy. +− Provider features outside the interface are unreachable without widening it: + streaming, tool calling, prompt caching, thinking budgets. Streaming in + particular will need a second method when the front end arrives, and that is a + deliberate later decision rather than an oversight. +− The port cannot express per-provider tuning — cache breakpoints, effort + levels — so cost optimisation that depends on them is invisible from outside + the adapter. +− Two adapters to keep working, one of which nothing in production will use. +− A fake that satisfies schemas will still fail domain validation, because it + invents content that belongs to no case. That is correct, and it means the + fake proves pipelines, never output quality. + +## Alternatives considered + +- **Keep calling the SDK directly and abstract when the provider is chosen.** + Cheaper today. Rejected because the coupling that hurts is not the import — it + is the vocabulary that leaks into modules written between now and then, and by + the time it is visible it is spread across the phases that were built in the + meantime. +- **Adopt a framework's model abstraction (LangChain, LiteLLM).** A ready port + supporting many providers. Rejected for now: it brings an abstraction far + wider than one method, and its own release cadence, to solve a problem this + project has already narrowed to a single call. Worth revisiting if the port + ever needs streaming, tool calling and caching at once — at which point + reimplementing it would be the mistake. +- **Decide the provider now.** The honest blocker is that the decision has + inputs nobody has yet: cost per match against real usage, and injection + resistance per language measured on the eval suite from phase 6. Deciding + before those exist would mean deciding on vibes and then defending it. From 4aaa9c19a2b73ca6a7e88e1033444d5f7db71e9b Mon Sep 17 00:00:00 2001 From: Madeuss Date: Tue, 1 Sep 2026 21:09:55 -0300 Subject: [PATCH 2/2] feat(api): target magalu prosa through an openai-compatible adapter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the Anthropic adapter, which was dead code the moment the provider question was raised: an adapter for a provider this project will not call, carrying an SDK dependency for calls that will never happen. It was kept because moving working code behind the new port was easier than deleting it, which is not a reason. Prosa exposes an OpenAI-compatible API, so what replaces it is not a Prosa adapter. It is an OpenAI-compatible adapter with a base URL, which reaches Groq, Together, OpenRouter or a local vLLM by configuration rather than by code. Structured output is the part the documentation does not answer, so the adapter answers it at runtime: a server-enforced JSON schema, then JSON mode with the schema in the prompt, then a plain request with the JSON pulled out of whatever comes back — keeping whichever works. All three failing is a failure. A response that does not validate is discarded, never repaired (RN-022). ADR-0008 records the decision and what it costs. The grounds are credits already held, infrastructure already on Magalu, a monthly ceiling in the currency the eval plan is written in, and supporting a Brazilian cloud — not the eval evidence ADR-0007 said the decision would wait for, and the ADR says so rather than pretending the evidence arrived early. The cost is stated plainly: the catalog is open weights, and those models are weakest exactly where phase 3 is hardest — injection resistance and persona consistency, in Portuguese, which ADR-0005 already flagged as the harder language. What does not degrade is the canary gate: the model is never given a secret fact, and the filter is code. Provider defaults to none until the pilot opens. --- .env.example | 15 +- apps/api/pyproject.toml | 2 +- apps/api/src/firenze/cli.py | 7 +- apps/api/src/firenze/config.py | 17 +- apps/api/src/firenze/model/__init__.py | 27 ++- apps/api/src/firenze/model/anthropic.py | 70 ------- .../src/firenze/model/openai_compatible.py | 195 ++++++++++++++++++ apps/api/tests/test_model_port.py | 163 ++++++++++++--- apps/api/uv.lock | 48 ++--- docs/00-plano-de-projeto.md | 7 +- .../0007-one-port-for-any-model-provider.md | 22 +- ...0008-magalu-prosa-as-the-model-provider.md | 98 +++++++++ 12 files changed, 504 insertions(+), 167 deletions(-) delete mode 100644 apps/api/src/firenze/model/anthropic.py create mode 100644 apps/api/src/firenze/model/openai_compatible.py create mode 100644 docs/adr/0008-magalu-prosa-as-the-model-provider.md diff --git a/.env.example b/.env.example index 835ac4d..b5f9f20 100644 --- a/.env.example +++ b/.env.example @@ -10,12 +10,13 @@ FIRENZE_ENVIRONMENT=dev FIRENZE_DATABASE_URL=postgresql+psycopg://firenze:firenze@localhost:5433/firenze FIRENZE_REDIS_URL=redis://localhost:6379/0 -# Modelo: anthropic, fake ou none. O provedor ainda não foi escolhido, -# então o padrão é none e o verniz simplesmente não roda (ADR-0007). -# fake -> sem rede, sem chave, texto sintético marcado com [fake] -# anthropic -> exige ANTHROPIC_API_KEY, que o SDK lê sozinho +# Modelo: prosa, fake ou none (ADR-0007, ADR-0008). +# fake -> sem rede, sem chave, texto sintético marcado com [fake] +# prosa -> Magalu Prosa; base_url e chave saem do console da Magalu +# Padrão none porque o Prosa ainda está em piloto. FIRENZE_MODEL_PROVIDER=none -FIRENZE_MODEL_NAME=claude-haiku-4-5 +FIRENZE_MODEL_NAME= +FIRENZE_MODEL_BASE_URL= -# Nunca commitar valor real. -# ANTHROPIC_API_KEY= +# Nunca commitar valor real. O .env não é versionado. +FIRENZE_MODEL_API_KEY= diff --git a/apps/api/pyproject.toml b/apps/api/pyproject.toml index ac399ce..f761bf1 100644 --- a/apps/api/pyproject.toml +++ b/apps/api/pyproject.toml @@ -7,7 +7,7 @@ dependencies = [ "fastapi>=0.115", "uvicorn[standard]>=0.34", "pydantic-settings>=2.7", - "anthropic>=1.2.0", + "openai>=3.6.0", ] [project.scripts] diff --git a/apps/api/src/firenze/cli.py b/apps/api/src/firenze/cli.py index 03f2a5d..40e0470 100644 --- a/apps/api/src/firenze/cli.py +++ b/apps/api/src/firenze/cli.py @@ -112,7 +112,12 @@ def main(argv: Sequence[str] | None = None) -> int: veneer = None if args.veneer: try: - model = resolve(settings.model_provider, settings.model_name) + model = resolve( + settings.model_provider, + model=settings.model_name, + base_url=settings.model_base_url, + api_key=settings.model_api_key.get_secret_value(), + ) veneer = write(full.case, catalog, model=model) except (VeneerUnavailable, VeneerRejected, ModelUnavailable) as failure: # The case is playable without prose. Degrading beats failing. diff --git a/apps/api/src/firenze/config.py b/apps/api/src/firenze/config.py index 5ff7efb..a0a4329 100644 --- a/apps/api/src/firenze/config.py +++ b/apps/api/src/firenze/config.py @@ -1,5 +1,6 @@ from typing import Literal +from pydantic import SecretStr from pydantic_settings import BaseSettings, SettingsConfigDict Environment = Literal["dev", "staging", "prod"] @@ -14,16 +15,20 @@ class Settings(BaseSettings): database_url: str = "postgresql+psycopg://firenze:firenze@localhost:5433/firenze" redis_url: str = "redis://localhost:6379/0" model_provider: str = "none" - """Which provider backs the model port: anthropic, fake, or none. + """Which provider backs the model port: prosa, fake, or none. - `none` by default, deliberately. The provider is not decided yet, and a - default that picked one would make the decision quietly.""" + `none` by default even though Prosa is the decision (ADR-0008): the product + is still in pilot, and a default that tried to reach an endpoint nobody has + credentials for would turn a missing key into a confusing failure.""" model_name: str = "" - """Which model at that provider. + """Which model at that provider, from its catalog.""" - Empty by default for the same reason the provider is `none`: naming one - would pick a vendor. Required once a real provider is configured.""" + model_base_url: str = "" + """Endpoint of the OpenAI-compatible API. Prosa shows it beside the API key.""" + + model_api_key: SecretStr = SecretStr("") + """Secret so it does not land in a log by accident.""" settings = Settings() diff --git a/apps/api/src/firenze/model/__init__.py b/apps/api/src/firenze/model/__init__.py index 659dd5e..4f00304 100644 --- a/apps/api/src/firenze/model/__init__.py +++ b/apps/api/src/firenze/model/__init__.py @@ -4,36 +4,43 @@ depends on the port. """ -from firenze.model.anthropic import AnthropicModel from firenze.model.fake import FakeModel +from firenze.model.openai_compatible import OpenAICompatibleModel from firenze.model.port import ModelRefused, ModelUnavailable, StructuredModel -def resolve(provider: str, model: str) -> StructuredModel: +def resolve( + provider: str, + *, + model: str = "", + base_url: str = "", + api_key: str = "", +) -> StructuredModel: """Build the configured model, or raise `ModelUnavailable`. - `none` is the default and is not an error state to be worked around: the - provider for this project is not chosen yet, and a build that quietly picked - one would be making the decision on the reader's behalf. + `none` is not an error state to be worked around: a build that quietly + picked a provider would be making the decision on the reader's behalf. """ if provider == "none": raise ModelUnavailable( - "no model provider configured; set FIRENZE_MODEL_PROVIDER to one of: anthropic, fake" + "no model provider configured; set FIRENZE_MODEL_PROVIDER to one of: prosa, fake" ) if provider == "fake": return FakeModel() - if provider == "anthropic": + if provider == "prosa": + # Prosa speaks the OpenAI dialect (ADR-0008), so the adapter is generic + # and the provider name is only a label over a base URL. if not model: - raise ModelUnavailable("provider 'anthropic' needs FIRENZE_MODEL_NAME set") - return AnthropicModel(model=model) + raise ModelUnavailable("provider 'prosa' needs FIRENZE_MODEL_NAME set") + return OpenAICompatibleModel(model=model, base_url=base_url, api_key=api_key) raise ModelUnavailable(f"unknown model provider {provider!r}") __all__ = [ - "AnthropicModel", "FakeModel", "ModelRefused", "ModelUnavailable", + "OpenAICompatibleModel", "StructuredModel", "resolve", ] diff --git a/apps/api/src/firenze/model/anthropic.py b/apps/api/src/firenze/model/anthropic.py deleted file mode 100644 index 082975b..0000000 --- a/apps/api/src/firenze/model/anthropic.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Anthropic adapter. - -One of possibly several. Everything provider-specific lives here — the SDK -import, the client construction, the shape of a structured-output call — so that -choosing a different provider is a new file in this package and a line of -configuration, not a change anywhere in the game. - -The provider for this project is not decided yet. This adapter exists because it -was written first and works; it is a reference implementation of the port, not a -commitment. -""" - -from typing import Any, cast - -from firenze.model.port import ModelRefused, ModelUnavailable, Schema - -DEFAULT_MODEL = "claude-haiku-4-5" - - -class AnthropicModel: - """Calls the Anthropic Messages API with a schema as the output format.""" - - def __init__(self, model: str = DEFAULT_MODEL, client: Any | None = None) -> None: - self._model = model - self._client = client - - @property - def name(self) -> str: - return self._model - - def _connect(self) -> Any: - if self._client is not None: - return self._client - try: - import anthropic - except ImportError as missing: # pragma: no cover - the dependency is declared - raise ModelUnavailable("the anthropic sdk is not installed") from missing - try: - self._client = anthropic.Anthropic() - except Exception as failure: - raise ModelUnavailable(f"no usable credentials: {failure}") from failure - return self._client - - def complete( - self, - *, - system: str, - user: str, - schema: type[Schema], - max_tokens: int, - ) -> Schema: - client = self._connect() - try: - response = client.messages.parse( - model=self._model, - max_tokens=max_tokens, - system=system, - messages=[{"role": "user", "content": user}], - output_format=schema, - ) - except Exception as failure: - raise ModelUnavailable(f"the model call failed: {failure}") from failure - - if getattr(response, "stop_reason", None) == "refusal": - raise ModelRefused("the model declined to answer") - - parsed = getattr(response, "parsed_output", None) - if parsed is None: - raise ModelUnavailable("the model returned nothing that fits the schema") - return cast("Schema", parsed) diff --git a/apps/api/src/firenze/model/openai_compatible.py b/apps/api/src/firenze/model/openai_compatible.py new file mode 100644 index 0000000..c9d0ff4 --- /dev/null +++ b/apps/api/src/firenze/model/openai_compatible.py @@ -0,0 +1,195 @@ +"""Adapter for any endpoint that speaks the OpenAI chat-completions dialect. + +Written for Magalu Prosa (ADR-0008), which exposes exactly that. It is not a +Prosa adapter, though: the only thing that ties it to one provider is a base +URL, so pointing it at another compatible endpoint — or at a vLLM of your own — +is configuration rather than code. + +## Getting a schema back from a gateway that may not support schemas + +The port promises a validated instance or a failure. OpenAI-compatible gateways +vary in how much of that they help with, and Prosa's documentation does not say +which parts it implements. So this tries, in order: + +1. `response_format` with a JSON schema — the server enforces the shape; +2. `response_format: json_object` plus the schema in the prompt — the server + guarantees valid JSON, the shape is the model's problem; +3. a plain request with the schema in the prompt, extracting the first JSON + object out of whatever comes back. + +Whichever works first is remembered for the life of the adapter, so the cost of +not knowing is paid once. All three failing is a failure, never a repair: a +response that does not validate is discarded, because a half-parsed answer that +reaches the game is worse than no answer. +""" + +import json +import re +from typing import Any + +from pydantic import ValidationError + +from firenze.model.port import ModelRefused, ModelUnavailable, Schema + +MODES = ("json_schema", "json_object", "prompt") +JSON_OBJECT = re.compile(r"\{.*\}", re.DOTALL) + + +class OpenAICompatibleModel: + """Calls a chat-completions endpoint and returns a validated schema.""" + + def __init__( + self, + *, + model: str, + base_url: str, + api_key: str, + client: Any | None = None, + ) -> None: + self._model = model + self._base_url = base_url + self._api_key = api_key + self._client = client + self._mode: str | None = None + + @property + def name(self) -> str: + return self._model + + def _connect(self) -> Any: + if self._client is not None: + return self._client + try: + from openai import OpenAI + except ImportError as missing: # pragma: no cover - the dependency is declared + raise ModelUnavailable("the openai sdk is not installed") from missing + if not self._api_key: + raise ModelUnavailable("no api key: set FIRENZE_MODEL_API_KEY") + if not self._base_url: + raise ModelUnavailable("no base url: set FIRENZE_MODEL_BASE_URL") + self._client = OpenAI(api_key=self._api_key, base_url=self._base_url) + return self._client + + def complete( + self, + *, + system: str, + user: str, + schema: type[Schema], + max_tokens: int, + ) -> Schema: + client = self._connect() + attempts = (self._mode,) if self._mode else MODES + failures: list[str] = [] + + for mode in attempts: + try: + text = self._ask(client, mode, system, user, schema, max_tokens) + except ModelRefused: + raise + except Exception as failure: # the gateway rejected this mode, or the call failed + failures.append(f"{mode}: {failure}") + continue + + try: + parsed = schema.model_validate_json(_only_json(text)) + except (ValidationError, ValueError) as invalid: + failures.append(f"{mode}: response did not fit the schema ({invalid})") + continue + + self._mode = mode + return parsed + + raise ModelUnavailable("; ".join(failures) or "no usable response") + + def _ask( + self, + client: Any, + mode: str, + system: str, + user: str, + schema: type[Schema], + max_tokens: int, + ) -> str: + instructions = system + request: dict[str, Any] = {} + + if mode == "json_schema": + request["response_format"] = { + "type": "json_schema", + "json_schema": { + "name": schema.__name__, + "strict": True, + "schema": _strict(schema.model_json_schema()), + }, + } + else: + instructions = f"{system}\n\n{_schema_instructions(schema)}" + if mode == "json_object": + request["response_format"] = {"type": "json_object"} + + response = client.chat.completions.create( + model=self._model, + max_tokens=max_tokens, + messages=[ + {"role": "system", "content": instructions}, + {"role": "user", "content": user}, + ], + **request, + ) + + choice = response.choices[0] + if getattr(choice, "finish_reason", None) == "content_filter": + raise ModelRefused("the provider's content filter stopped the response") + content = choice.message.content + if not content: + raise ValueError("the response carried no content") + return str(content) + + +def _schema_instructions(schema: type[Schema]) -> str: + return ( + "Answer with a single JSON object and nothing else — no prose before it, " + "no code fence around it. It must validate against this JSON Schema:\n" + f"{json.dumps(schema.model_json_schema(), ensure_ascii=False)}" + ) + + +def _only_json(text: str) -> str: + """Pull the JSON object out of a reply that may be wrapped in prose or fences.""" + stripped = text.strip() + if stripped.startswith("{"): + return stripped + found = JSON_OBJECT.search(stripped) + if not found: + raise ValueError(f"no JSON object in the response: {stripped[:120]!r}") + return found.group(0) + + +def _strict(schema: dict[str, Any]) -> dict[str, Any]: + """Tighten a Pydantic schema into the shape strict mode expects. + + Every object closed to extra properties, every property required. Gateways + that enforce schemas tend to demand this, and Pydantic does not emit it. + """ + if schema.get("type") == "object" or "properties" in schema: + schema["additionalProperties"] = False + if "properties" in schema: + schema["required"] = list(schema["properties"]) + for key in ("properties", "$defs", "definitions"): + for value in schema.get(key, {}).values(): + if isinstance(value, dict): + _strict(value) + for key in ("items", "prefixItems"): + value = schema.get(key) + if isinstance(value, dict): + _strict(value) + elif isinstance(value, list): + for item in value: + if isinstance(item, dict): + _strict(item) + for combinator in ("anyOf", "oneOf", "allOf"): + for item in schema.get(combinator, []): + if isinstance(item, dict): + _strict(item) + return schema diff --git a/apps/api/tests/test_model_port.py b/apps/api/tests/test_model_port.py index 2b73ca2..75ebaf2 100644 --- a/apps/api/tests/test_model_port.py +++ b/apps/api/tests/test_model_port.py @@ -12,8 +12,15 @@ from pydantic import BaseModel from firenze.domain import Role -from firenze.model import AnthropicModel, FakeModel, ModelRefused, ModelUnavailable, resolve +from firenze.model import ( + FakeModel, + ModelRefused, + ModelUnavailable, + OpenAICompatibleModel, + resolve, +) from firenze.model.fake import MARKER +from firenze.model.openai_compatible import _strict class Nested(BaseModel): @@ -67,55 +74,149 @@ def test_the_fake_can_be_asked_to_refuse() -> None: def test_no_provider_configured_is_an_explicit_failure() -> None: """Not a silent default. The provider for this project is not chosen yet.""" with pytest.raises(ModelUnavailable, match="no model provider configured"): - resolve("none", "whatever") + resolve("none") def test_an_unknown_provider_says_so() -> None: with pytest.raises(ModelUnavailable, match="unknown model provider"): - resolve("mistral-via-carrier-pigeon", "x") + resolve("mistral-via-carrier-pigeon") -@pytest.mark.parametrize( - ("provider", "expected"), - [("fake", "fake"), ("anthropic", "claude-haiku-4-5")], -) -def test_resolve_builds_what_it_was_asked_for(provider: str, expected: str) -> None: - assert resolve(provider, "claude-haiku-4-5").name == expected +def test_resolve_builds_what_it_was_asked_for() -> None: + assert resolve("fake").name == "fake" + assert resolve("prosa", model="qwen-whatever", base_url="https://x").name == "qwen-whatever" -def test_the_anthropic_adapter_translates_a_refusal() -> None: - class Refusing: - def parse(self, **kwargs: Any) -> Any: - return type("R", (), {"stop_reason": "refusal", "parsed_output": None}) +def test_prosa_without_a_model_name_fails_loudly() -> None: + with pytest.raises(ModelUnavailable, match="needs FIRENZE_MODEL_NAME"): + resolve("prosa") - model = AnthropicModel(client=type("C", (), {"messages": Refusing()})()) - with pytest.raises(ModelRefused): - model.complete(system="s", user="u", schema=Answer, max_tokens=10) +class Reply: + """The slice of an OpenAI-shaped response this adapter reads.""" + + def __init__(self, content: str | None, finish_reason: str = "stop") -> None: + message = type("M", (), {"content": content}) + choice = type("C", (), {"message": message, "finish_reason": finish_reason}) + self.choices = [choice] + + +class Gateway: + """A chat-completions endpoint that supports only the modes it was told to.""" + + PADRAO = ( + '{"line": "ok", "stance": "victim", "lied": false, "turns": 1, "parts": [{"label": "x"}]}' + ) + + def __init__(self, supports: set[str], reply: str = PADRAO) -> None: + self._supports = supports + self._reply = reply + self.modes_tried: list[str] = [] + + @property + def chat(self) -> "Gateway": + return self + + @property + def completions(self) -> "Gateway": + return self + + def create(self, **kwargs: Any) -> Reply: + fmt = kwargs.get("response_format") or {} + mode = fmt.get("type", "prompt") + self.modes_tried.append(mode) + if mode not in self._supports: + raise ValueError(f"unsupported response_format: {mode}") + return Reply(self._reply) + + +def _model(gateway: Gateway) -> OpenAICompatibleModel: + return OpenAICompatibleModel( + model="qwen-whatever", base_url="https://x", api_key="k", client=gateway + ) + + +def test_it_uses_a_server_enforced_schema_when_the_gateway_supports_one() -> None: + gateway = Gateway(supports={"json_schema"}) + + answer = _model(gateway).complete(system="s", user="u", schema=Answer, max_tokens=10) + + assert gateway.modes_tried == ["json_schema"] + assert answer.line == "ok" -def test_the_anthropic_adapter_translates_a_transport_failure() -> None: - """A provider exception must never escape the adapter as itself.""" +def test_it_falls_back_to_json_mode_when_schemas_are_rejected() -> None: + """Prosa's documentation does not say which modes it implements.""" + gateway = Gateway(supports={"json_object"}) - class Exploding: - def parse(self, **kwargs: Any) -> Any: - raise ConnectionError("no route to host") + answer = _model(gateway).complete(system="s", user="u", schema=Answer, max_tokens=10) - model = AnthropicModel(client=type("C", (), {"messages": Exploding()})()) + assert gateway.modes_tried == ["json_schema", "json_object"] + assert answer.line == "ok" - with pytest.raises(ModelUnavailable, match="model call failed"): - model.complete(system="s", user="u", schema=Answer, max_tokens=10) +def test_it_falls_back_to_asking_in_the_prompt() -> None: + gateway = Gateway(supports={"prompt"}) + + answer = _model(gateway).complete(system="s", user="u", schema=Answer, max_tokens=10) + + assert gateway.modes_tried == ["json_schema", "json_object", "prompt"] + assert answer.line == "ok" + + +def test_the_working_mode_is_remembered() -> None: + """The cost of not knowing what the gateway supports is paid once.""" + gateway = Gateway(supports={"prompt"}) + model = _model(gateway) + + model.complete(system="s", user="u", schema=Answer, max_tokens=10) + model.complete(system="s", user="u", schema=Answer, max_tokens=10) + + assert gateway.modes_tried == ["json_schema", "json_object", "prompt", "prompt"] + + +def test_json_wrapped_in_prose_is_still_read() -> None: + """A model told to answer in JSON often says "here you go:" first.""" + corpo = ( + '{"line": "ok", "stance": "victim", "lied": false, "turns": 1, "parts": [{"label": "x"}]}' + ) + envelope = f"""Claro! ```json +{corpo} +``` espero ter ajudado""" + gateway = Gateway(supports={"prompt"}, reply=envelope) + + answer = _model(gateway).complete(system="s", user="u", schema=Answer, max_tokens=10) + + assert answer.line == "ok" + + +def test_a_response_that_does_not_fit_the_schema_is_discarded() -> None: + """Never repaired. A half-parsed answer reaching the game is worse than none.""" + gateway = Gateway(supports={"json_schema"}, reply='{"line": "ok"}') + + with pytest.raises(ModelUnavailable, match="did not fit the schema"): + _model(gateway).complete(system="s", user="u", schema=Answer, max_tokens=10) + + +def test_a_content_filter_is_a_refusal_not_a_failure() -> None: + class Filtered(Gateway): + def create(self, **kwargs: Any) -> Reply: + return Reply(None, finish_reason="content_filter") + + with pytest.raises(ModelRefused): + _model(Filtered(supports={"json_schema"})).complete( + system="s", user="u", schema=Answer, max_tokens=10 + ) -def test_the_anthropic_adapter_rejects_an_unparseable_response() -> None: - class Empty: - def parse(self, **kwargs: Any) -> Any: - return type("R", (), {"stop_reason": "end_turn", "parsed_output": None}) - model = AnthropicModel(client=type("C", (), {"messages": Empty()})()) +def test_strict_mode_closes_every_object_in_the_schema() -> None: + """Gateways that enforce schemas demand this; Pydantic does not emit it.""" + tightened = _strict(Answer.model_json_schema()) - with pytest.raises(ModelUnavailable, match="nothing that fits the schema"): - model.complete(system="s", user="u", schema=Answer, max_tokens=10) + assert tightened["additionalProperties"] is False + assert set(tightened["required"]) == set(tightened["properties"]) + nested = tightened["$defs"]["Nested"] + assert nested["additionalProperties"] is False def test_no_module_outside_the_port_imports_a_provider_sdk() -> None: diff --git a/apps/api/uv.lock b/apps/api/uv.lock index a32f4b8..b0e9f08 100644 --- a/apps/api/uv.lock +++ b/apps/api/uv.lock @@ -24,24 +24,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, ] -[[package]] -name = "anthropic" -version = "1.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "docstring-parser" }, - { name = "httpx2" }, - { name = "jiter" }, - { name = "pydantic" }, - { name = "sniffio" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/95/1a/b5af41cc1fa14da277ec20ca5554dd2fcbc09b8523ac59b7a97fbb88e452/anthropic-1.2.0.tar.gz", hash = "sha256:12f8eedee7b7fb5685837b1371b7bfae1b281703f62355f4632598ec2fc53b34", size = 1137443, upload-time = "2026-08-27T20:29:12.68Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ed/78/3f8b52708b03309e511990700bb8d0ec7a0c9db3d2a1e0d1c3ca417a4604/anthropic-1.2.0-py3-none-any.whl", hash = "sha256:b60642b3e3cd6b8e3e328a2d3f2863ad2b6e743f1037e42cc0143f7df99f63c6", size = 1289535, upload-time = "2026-08-27T20:29:11.01Z" }, -] - [[package]] name = "anyio" version = "4.14.2" @@ -145,15 +127,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] -[[package]] -name = "docstring-parser" -version = "0.18.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e0/4d/f332313098c1de1b2d2ff91cf2674415cc7cddab2ca1b01ae29774bd5fdf/docstring_parser-0.18.0.tar.gz", hash = "sha256:292510982205c12b1248696f44959db3cdd1740237a968ea1e2e7a900eeb2015", size = 29341, upload-time = "2026-04-14T04:09:19.867Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl", hash = "sha256:b3fcbed555c47d8479be0796ef7e19c2670d428d72e96da63f3a40122860374b", size = 22484, upload-time = "2026-04-14T04:09:18.638Z" }, -] - [[package]] name = "fastapi" version = "0.141.1" @@ -175,8 +148,8 @@ name = "firenze-api" version = "0.1.0" source = { editable = "." } dependencies = [ - { name = "anthropic" }, { name = "fastapi" }, + { name = "openai" }, { name = "pydantic-settings" }, { name = "uvicorn", extra = ["standard"] }, ] @@ -191,10 +164,10 @@ dev = [ [package.metadata] requires-dist = [ - { name = "anthropic", specifier = ">=1.2.0" }, { name = "fastapi", specifier = ">=0.115" }, { name = "httpx", marker = "extra == 'dev'", specifier = ">=0.28" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.14" }, + { name = "openai", specifier = ">=3.6.0" }, { name = "pydantic-settings", specifier = ">=2.7" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.9" }, @@ -517,6 +490,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, ] +[[package]] +name = "openai" +version = "3.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "httpx2" }, + { name = "jiter" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/76/913b755a1a6b54e2d9140eb8d488aa0d47c7359b1d7eac5e864cb7913bbf/openai-3.6.0.tar.gz", hash = "sha256:18fe3f6e96390ef41ee27b152fc9effefca321c33673bd9b956a572493d3ab9b", size = 1455376, upload-time = "2026-08-28T22:29:18.268Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a1/94/805b87ecc951c49ec8f247f5e8eb324ab064bd2ad73b6a0e704dd49aa073/openai-3.6.0-py3-none-any.whl", hash = "sha256:508e2158bf971687f953b62e44b02f207792c815aac306816386d7ba34d37f5f", size = 1699841, upload-time = "2026-08-28T22:29:16.436Z" }, +] + [[package]] name = "packaging" version = "26.3" diff --git a/docs/00-plano-de-projeto.md b/docs/00-plano-de-projeto.md index 434ceca..7760f78 100644 --- a/docs/00-plano-de-projeto.md +++ b/docs/00-plano-de-projeto.md @@ -178,14 +178,15 @@ ADR escrita para convencer, não para decidir. - [`0006`](adr/0006-english-in-code-portuguese-in-the-product.md) — código em inglês, produto em português - [`0007`](adr/0007-one-port-for-any-model-provider.md) — uma porta para - qualquer fornecedor de modelo, escolha do fornecedor adiada + qualquer fornecedor de modelo +- [`0008`](adr/0008-magalu-prosa-as-the-model-provider.md) — Magalu Prosa como + fornecedor, com adaptador OpenAI-compatible Todas as ADRs são escritas em inglês (ADR-0006) — as quatro primeiras foram traduzidas depois de escritas. **Na fila:** LangGraph vs orquestração própria · streaming SSE vs WebSocket · -qual fornecedor de modelo (decisão adiada pela ADR-0007, com entrada nos evals -da fase 6) · API externa vs LLM self-hosted na Magalu · isolamento de contexto +API externa vs LLM self-hosted na Magalu · isolamento de contexto como fronteira de segurança · VM+compose antes de Kubernetes. Numeração sai na ordem em que a decisão é tomada, não na ordem desta lista. diff --git a/docs/adr/0007-one-port-for-any-model-provider.md b/docs/adr/0007-one-port-for-any-model-provider.md index 4350387..5b12896 100644 --- a/docs/adr/0007-one-port-for-any-model-provider.md +++ b/docs/adr/0007-one-port-for-any-model-provider.md @@ -1,8 +1,8 @@ -# ADR-0007: One port for any model provider, and the provider chosen later +# ADR-0007: One port for any model provider ## Status -Accepted — 2026-08-31 +Accepted — 2026-08-31. The provider it deferred was chosen in ADR-0008. ## Context @@ -41,8 +41,9 @@ failure would hide it exactly when it matters. Adapters live in the same package and nowhere else. Two exist: -- **`AnthropicModel`** — the code that was already written, now one - implementation rather than the implementation. Not a commitment. +- **`OpenAICompatibleModel`** — any endpoint speaking the OpenAI + chat-completions dialect, which is one base URL away from being a different + provider. ADR-0008 points it at Magalu Prosa. - **`FakeModel`** — no network, no key, no money. Deterministic, and it names itself `fake` so anything it wrote is traceable to it. @@ -72,7 +73,8 @@ draw the line this tightly. − The port cannot express per-provider tuning — cache breakpoints, effort levels — so cost optimisation that depends on them is invisible from outside the adapter. -− Two adapters to keep working, one of which nothing in production will use. +− The port ships tested against fakes only. Its first contact with a real + endpoint will find something the design did not anticipate. − A fake that satisfies schemas will still fail domain validation, because it invents content that belongs to no case. That is correct, and it means the fake proves pipelines, never output quality. @@ -90,7 +92,9 @@ draw the line this tightly. project has already narrowed to a single call. Worth revisiting if the port ever needs streaming, tool calling and caching at once — at which point reimplementing it would be the mistake. -- **Decide the provider now.** The honest blocker is that the decision has - inputs nobody has yet: cost per match against real usage, and injection - resistance per language measured on the eval suite from phase 6. Deciding - before those exist would mean deciding on vibes and then defending it. +- **Decide the provider now.** Argued at the time that the decision needed + inputs nobody had: cost against real usage, injection resistance per language + from the phase-6 evals. It was then made anyway, days later and on entirely + different grounds — credits, a Brazilian cloud, a hard budget ceiling + (ADR-0008). The port is what made that safe: deciding early costs one file to + undo, so waiting bought less than it delayed. diff --git a/docs/adr/0008-magalu-prosa-as-the-model-provider.md b/docs/adr/0008-magalu-prosa-as-the-model-provider.md new file mode 100644 index 0000000..9e0b800 --- /dev/null +++ b/docs/adr/0008-magalu-prosa-as-the-model-provider.md @@ -0,0 +1,98 @@ +# ADR-0008: Magalu Prosa as the model provider + +## Status + +Accepted — 2026-08-31. Blocked on the product leaving pilot. + +## Context + +ADR-0007 put every model call behind one port and deliberately left the provider +open, saying the decision needed inputs that did not exist yet: cost measured +against real usage, and injection resistance per language measured on the phase-6 +eval suite. + +The decision is being made before those inputs exist, on different grounds. That +is worth stating plainly rather than pretending the evidence arrived early. + +The grounds that do exist: + +- **The credits are already there.** Magalu Cloud credits are held; an API + billed to a card is not. For a study project, the difference between spending + credits and spending money is the difference between experimenting freely and + rationing. +- **The infrastructure is already there.** ADR-0002 puts Postgres with pgvector + on Magalu, and phase 7 puts the application there. Inference in the same place + means one account, one bill, one network. +- **Prosa speaks the OpenAI dialect.** Its documentation is explicit: *"uma API + compatível com o padrão OpenAI"*. So the adapter is not a Prosa adapter — it + is an OpenAI-compatible adapter with a base URL, and every other compatible + endpoint is reachable by configuration. +- **A monthly budget in R$ that pauses consumption at the limit.** The eval plan + caps cost per match; a provider that enforces a ceiling is closer to that + requirement than one that emails an invoice. +- **Supporting a Brazilian cloud is a reason the author holds.** Not an + engineering argument, and it does not need to pretend to be one. + +## Decision + +Magalu Prosa, through a generic OpenAI-compatible adapter in +`firenze.model.openai_compatible`, configured by `FIRENZE_MODEL_BASE_URL`, +`FIRENZE_MODEL_NAME` and `FIRENZE_MODEL_API_KEY`. + +The default provider stays `none` until the product leaves pilot and credentials +exist. `fake` covers development in the meantime. + +**Structured output is handled by the adapter, because the documentation does +not say what the gateway supports.** It tries a server-enforced JSON schema, +then JSON mode with the schema in the prompt, then a plain request parsed out of +the reply — keeping whichever works. All three failing is a failure; a response +that does not validate is discarded, never repaired (RN-022). + +## Consequences + ++ Inference, database and hosting on one account, paid with credits already + held. ++ A hard monthly ceiling in the currency the eval plan is written in. ++ The adapter is generic, so a second provider — or a local vLLM — is a base URL + away. Nothing about this decision is expensive to reverse, which is what makes + it safe to take early. ++ The project gets an unusual measurement out of it: open-weights models, served + from a Brazilian cloud, answering in Portuguese, scored on an adversarial + suite. That is a more interesting result than passing with a frontier model. +− **The catalog is open-weights — Google, Meta, NVIDIA, Qwen — and those models + are weaker exactly where phase 3 is hardest.** Injection resistance ≥ 95% and + persona consistency are the axes where model strength shows most, and ADR-0005 + already records that resistance degrades outside English. Portuguese plus + open weights is likely the hardest combination this project could pick. +− **Pilot means no guarantees**: the catalog can change, rate limits are + undocumented, and there is no SLA. Acceptable for a study project and not + acceptable for anything else. +− Structured output may cost an extra round trip on every call if the gateway + supports neither schema enforcement nor JSON mode. +− The provider cannot be exercised until launch, so the adapter ships tested + against fakes only. Its first contact with a real endpoint will find something. + +### What does not degrade with a weaker model + +Worth being precise, because "weaker model" sounds like it threatens everything +and does not. The canary gate is 0% and stays 0%: isolation is a data boundary, +the model is never given a secret fact, and the filter is code (RN-010, RN-012). +The verdict, the score and contradiction detection are deterministic (RN-032). +What a weaker model costs is **quality** — a character who breaks voice, a reply +in the wrong shape, an injection classified wrong. Those are measurable, and the +eval suite exists to measure them. + +## Alternatives considered + +- **A frontier provider billed to a card.** Stronger on precisely the axes + phase 3 measures. Rejected for now: it spends money the project does not need + to spend, and it would leave R$ 300 of credits unused while making the + cheapest possible interesting experiment impossible. +- **Wait for the phase-6 evals, as ADR-0007 said.** Consistent, and it would + block phases 2 to 5 on a decision whose reversal costs one file. The port is + what makes deciding early cheap; refusing to use it would waste the design. +- **Self-hosted model on a Magalu GPU VM.** Investigated: the account currently + lists 50 machine types and none with a GPU, so it needs a quota request. Also + bills per hour of uptime rather than per token, which inverts the economics + for intermittent development use. Stays a phase-8 experiment, to be run + against the eval suite and written up.