diff --git a/.env.example b/.env.example index facb774..b5f9f20 100644 --- a/.env.example +++ b/.env.example @@ -10,9 +10,13 @@ FIRENZE_ENVIRONMENT=dev FIRENZE_DATABASE_URL=postgresql+psycopg://firenze:firenze@localhost:5433/firenze FIRENZE_REDIS_URL=redis://localhost:6379/0 -# Provedor de LLM. Nunca commitar valor real. -# O SDK lê ANTHROPIC_API_KEY sozinho — sem prefixo FIRENZE_. -# ANTHROPIC_API_KEY= +# Modelo: prosa, fake ou none (ADR-0007, ADR-0008). +# fake -> sem rede, sem chave, texto sintético marcado com [fake] +# prosa -> Magalu Prosa; base_url e chave saem do console da Magalu +# Padrão none porque o Prosa ainda está em piloto. +FIRENZE_MODEL_PROVIDER=none +FIRENZE_MODEL_NAME= +FIRENZE_MODEL_BASE_URL= -# Qual modelo escreve o verniz. É o botão que move o custo por caso. -FIRENZE_VENEER_MODEL=claude-haiku-4-5 +# Nunca commitar valor real. O .env não é versionado. +FIRENZE_MODEL_API_KEY= diff --git a/CLAUDE.md b/CLAUDE.md index 27b1b75..f3b0465 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,6 +20,8 @@ Next.js (front) + FastAPI/LangGraph (core de IA) + Postgres com pgvector. - Toda saída de LLM é validada por schema Pydantic versionado. Mudou o schema, roda os evals antes de commitar. - Prompts vivem em `prompts/`, versionados, nunca em string literal. +- Modelo só se chama pela porta em `firenze.model`. Nenhum outro módulo + importa SDK de fornecedor. (ADR-0007) - Código em inglês, docs em português, conteúdo do jogo em português. Domínio guarda estrutura; frase pronta só no catálogo. (ADR-0005/0006) - Conventional Commits. Nada direto na main. diff --git a/apps/api/pyproject.toml b/apps/api/pyproject.toml index ac399ce..f761bf1 100644 --- a/apps/api/pyproject.toml +++ b/apps/api/pyproject.toml @@ -7,7 +7,7 @@ dependencies = [ "fastapi>=0.115", "uvicorn[standard]>=0.34", "pydantic-settings>=2.7", - "anthropic>=1.2.0", + "openai>=3.6.0", ] [project.scripts] diff --git a/apps/api/src/firenze/cli.py b/apps/api/src/firenze/cli.py index eb0f2bd..40e0470 100644 --- a/apps/api/src/firenze/cli.py +++ b/apps/api/src/firenze/cli.py @@ -18,6 +18,7 @@ from firenze.domain import CaseWithSolution, FactKind, Role from firenze.generation import UnsolvableCase, generate, solve from firenze.i18n import DEFAULT_LOCALE, Catalog, UnknownLocale, available_locales, load +from firenze.model import ModelUnavailable, resolve from firenze.veneer import CaseVeneer, VeneerRejected, VeneerUnavailable, write @@ -111,8 +112,14 @@ def main(argv: Sequence[str] | None = None) -> int: veneer = None if args.veneer: try: - veneer = write(full.case, catalog, model=settings.veneer_model) - except (VeneerUnavailable, VeneerRejected) as failure: + model = resolve( + settings.model_provider, + model=settings.model_name, + base_url=settings.model_base_url, + api_key=settings.model_api_key.get_secret_value(), + ) + veneer = write(full.case, catalog, model=model) + except (VeneerUnavailable, VeneerRejected, ModelUnavailable) as failure: # The case is playable without prose. Degrading beats failing. print(f"veneer skipped: {failure}", file=sys.stderr) diff --git a/apps/api/src/firenze/config.py b/apps/api/src/firenze/config.py index aceb17a..a0a4329 100644 --- a/apps/api/src/firenze/config.py +++ b/apps/api/src/firenze/config.py @@ -1,5 +1,6 @@ from typing import Literal +from pydantic import SecretStr from pydantic_settings import BaseSettings, SettingsConfigDict Environment = Literal["dev", "staging", "prod"] @@ -13,9 +14,21 @@ class Settings(BaseSettings): environment: Environment = "dev" database_url: str = "postgresql+psycopg://firenze:firenze@localhost:5433/firenze" redis_url: str = "redis://localhost:6379/0" - veneer_model: str = "claude-haiku-4-5" - """Which model writes the veneer. Env-switchable because it is the one - knob that moves cost per case.""" + model_provider: str = "none" + """Which provider backs the model port: prosa, fake, or none. + + `none` by default even though Prosa is the decision (ADR-0008): the product + is still in pilot, and a default that tried to reach an endpoint nobody has + credentials for would turn a missing key into a confusing failure.""" + + model_name: str = "" + """Which model at that provider, from its catalog.""" + + model_base_url: str = "" + """Endpoint of the OpenAI-compatible API. Prosa shows it beside the API key.""" + + model_api_key: SecretStr = SecretStr("") + """Secret so it does not land in a log by accident.""" settings = Settings() diff --git a/apps/api/src/firenze/model/__init__.py b/apps/api/src/firenze/model/__init__.py new file mode 100644 index 0000000..4f00304 --- /dev/null +++ b/apps/api/src/firenze/model/__init__.py @@ -0,0 +1,46 @@ +"""Language models, behind one interface. + +`resolve()` is the only place that knows which providers exist. Everything else +depends on the port. +""" + +from firenze.model.fake import FakeModel +from firenze.model.openai_compatible import OpenAICompatibleModel +from firenze.model.port import ModelRefused, ModelUnavailable, StructuredModel + + +def resolve( + provider: str, + *, + model: str = "", + base_url: str = "", + api_key: str = "", +) -> StructuredModel: + """Build the configured model, or raise `ModelUnavailable`. + + `none` is not an error state to be worked around: a build that quietly + picked a provider would be making the decision on the reader's behalf. + """ + if provider == "none": + raise ModelUnavailable( + "no model provider configured; set FIRENZE_MODEL_PROVIDER to one of: prosa, fake" + ) + if provider == "fake": + return FakeModel() + if provider == "prosa": + # Prosa speaks the OpenAI dialect (ADR-0008), so the adapter is generic + # and the provider name is only a label over a base URL. + if not model: + raise ModelUnavailable("provider 'prosa' needs FIRENZE_MODEL_NAME set") + return OpenAICompatibleModel(model=model, base_url=base_url, api_key=api_key) + raise ModelUnavailable(f"unknown model provider {provider!r}") + + +__all__ = [ + "FakeModel", + "ModelRefused", + "ModelUnavailable", + "OpenAICompatibleModel", + "StructuredModel", + "resolve", +] diff --git a/apps/api/src/firenze/model/fake.py b/apps/api/src/firenze/model/fake.py new file mode 100644 index 0000000..4b612de --- /dev/null +++ b/apps/api/src/firenze/model/fake.py @@ -0,0 +1,100 @@ +"""A model that needs no network, no key and no money. + +It exists so the game can be built and played before a provider is chosen, and +so a front end can be developed against a running backend without anyone paying +per keystroke. It fills a schema with deterministic, obviously-synthetic text. + +Two rules keep it honest: + +- **It never pretends to be a real model.** `name` says `fake`, and that string + is recorded on everything it writes. A veneer produced here is traceable as + produced here. +- **It is deterministic.** Same prompt, same output. A fake that varied would + make a failing test look flaky, which is worse than no fake at all. + +It is not a mock of any provider and does not imitate one. It satisfies the +port, nothing more — and note what that means: it produces output that fits a +**schema**, never output that fits a **case**. Domain validation will reject a +fake veneer, because the cast it invents is not the cast of any real mystery, +and that rejection is correct rather than a shortcoming. The fake earns its keep +on pipelines whose correctness does not depend on the content: schema +validation, stance transitions, output filters, turn accounting. +""" + +import hashlib +from enum import Enum +from typing import Any, get_args, get_origin + +from pydantic import BaseModel + +from firenze.model.port import ModelRefused, Schema + +MARKER = "[fake]" + + +class FakeModel: + """Fills any schema with placeholder text derived from the prompt.""" + + def __init__(self, *, refuse: bool = False) -> None: + self._refuse = refuse + + @property + def name(self) -> str: + return "fake" + + def complete( + self, + *, + system: str, + user: str, + schema: type[Schema], + max_tokens: int, + ) -> Schema: + if self._refuse: + raise ModelRefused("the fake model was asked to refuse") + + seed = hashlib.blake2s(f"{system}{user}".encode(), digest_size=4).hexdigest() + return _fill(schema, seed=seed, path="") + + +def _fill(schema: type[Schema], *, seed: str, path: str) -> Schema: + values: dict[str, Any] = {} + for name, field in schema.model_fields.items(): + values[name] = _value_for(field.annotation, seed=seed, path=f"{path}.{name}") + return schema(**values) + + +def _value_for(annotation: Any, *, seed: str, path: str) -> Any: + if annotation is str: + return f"{MARKER} {path.lstrip('.')} {seed}" + if annotation is int: + return 0 + if annotation is float: + return 0.0 + if annotation is bool: + return False + + origin = get_origin(annotation) + args = get_args(annotation) + + if origin in (tuple, list, set, frozenset): + # One element, not zero: an empty collection would pass a schema and + # then fail a validation rule for a reason unrelated to the fake. + inner = args[0] if args else str + item = _value_for(inner, seed=seed, path=f"{path}[0]") + return origin([item]) if origin is not tuple else (item,) + + if isinstance(annotation, type) and issubclass(annotation, Enum): + # The first member, deterministically. A stance machine fed a random + # enum would fail for a reason that has nothing to do with the code. + return next(iter(annotation)) + + if isinstance(annotation, type) and issubclass(annotation, BaseModel): + return _fill(annotation, seed=seed, path=path) + + if args: # Optional[X], X | None, Literal[...] + for candidate in args: + if candidate is not type(None): + return _value_for(candidate, seed=seed, path=path) + + return f"{MARKER} {path.lstrip('.')}" diff --git a/apps/api/src/firenze/model/openai_compatible.py b/apps/api/src/firenze/model/openai_compatible.py new file mode 100644 index 0000000..c9d0ff4 --- /dev/null +++ b/apps/api/src/firenze/model/openai_compatible.py @@ -0,0 +1,195 @@ +"""Adapter for any endpoint that speaks the OpenAI chat-completions dialect. + +Written for Magalu Prosa (ADR-0008), which exposes exactly that. It is not a +Prosa adapter, though: the only thing that ties it to one provider is a base +URL, so pointing it at another compatible endpoint — or at a vLLM of your own — +is configuration rather than code. + +## Getting a schema back from a gateway that may not support schemas + +The port promises a validated instance or a failure. OpenAI-compatible gateways +vary in how much of that they help with, and Prosa's documentation does not say +which parts it implements. So this tries, in order: + +1. `response_format` with a JSON schema — the server enforces the shape; +2. `response_format: json_object` plus the schema in the prompt — the server + guarantees valid JSON, the shape is the model's problem; +3. a plain request with the schema in the prompt, extracting the first JSON + object out of whatever comes back. + +Whichever works first is remembered for the life of the adapter, so the cost of +not knowing is paid once. All three failing is a failure, never a repair: a +response that does not validate is discarded, because a half-parsed answer that +reaches the game is worse than no answer. +""" + +import json +import re +from typing import Any + +from pydantic import ValidationError + +from firenze.model.port import ModelRefused, ModelUnavailable, Schema + +MODES = ("json_schema", "json_object", "prompt") +JSON_OBJECT = re.compile(r"\{.*\}", re.DOTALL) + + +class OpenAICompatibleModel: + """Calls a chat-completions endpoint and returns a validated schema.""" + + def __init__( + self, + *, + model: str, + base_url: str, + api_key: str, + client: Any | None = None, + ) -> None: + self._model = model + self._base_url = base_url + self._api_key = api_key + self._client = client + self._mode: str | None = None + + @property + def name(self) -> str: + return self._model + + def _connect(self) -> Any: + if self._client is not None: + return self._client + try: + from openai import OpenAI + except ImportError as missing: # pragma: no cover - the dependency is declared + raise ModelUnavailable("the openai sdk is not installed") from missing + if not self._api_key: + raise ModelUnavailable("no api key: set FIRENZE_MODEL_API_KEY") + if not self._base_url: + raise ModelUnavailable("no base url: set FIRENZE_MODEL_BASE_URL") + self._client = OpenAI(api_key=self._api_key, base_url=self._base_url) + return self._client + + def complete( + self, + *, + system: str, + user: str, + schema: type[Schema], + max_tokens: int, + ) -> Schema: + client = self._connect() + attempts = (self._mode,) if self._mode else MODES + failures: list[str] = [] + + for mode in attempts: + try: + text = self._ask(client, mode, system, user, schema, max_tokens) + except ModelRefused: + raise + except Exception as failure: # the gateway rejected this mode, or the call failed + failures.append(f"{mode}: {failure}") + continue + + try: + parsed = schema.model_validate_json(_only_json(text)) + except (ValidationError, ValueError) as invalid: + failures.append(f"{mode}: response did not fit the schema ({invalid})") + continue + + self._mode = mode + return parsed + + raise ModelUnavailable("; ".join(failures) or "no usable response") + + def _ask( + self, + client: Any, + mode: str, + system: str, + user: str, + schema: type[Schema], + max_tokens: int, + ) -> str: + instructions = system + request: dict[str, Any] = {} + + if mode == "json_schema": + request["response_format"] = { + "type": "json_schema", + "json_schema": { + "name": schema.__name__, + "strict": True, + "schema": _strict(schema.model_json_schema()), + }, + } + else: + instructions = f"{system}\n\n{_schema_instructions(schema)}" + if mode == "json_object": + request["response_format"] = {"type": "json_object"} + + response = client.chat.completions.create( + model=self._model, + max_tokens=max_tokens, + messages=[ + {"role": "system", "content": instructions}, + {"role": "user", "content": user}, + ], + **request, + ) + + choice = response.choices[0] + if getattr(choice, "finish_reason", None) == "content_filter": + raise ModelRefused("the provider's content filter stopped the response") + content = choice.message.content + if not content: + raise ValueError("the response carried no content") + return str(content) + + +def _schema_instructions(schema: type[Schema]) -> str: + return ( + "Answer with a single JSON object and nothing else — no prose before it, " + "no code fence around it. It must validate against this JSON Schema:\n" + f"{json.dumps(schema.model_json_schema(), ensure_ascii=False)}" + ) + + +def _only_json(text: str) -> str: + """Pull the JSON object out of a reply that may be wrapped in prose or fences.""" + stripped = text.strip() + if stripped.startswith("{"): + return stripped + found = JSON_OBJECT.search(stripped) + if not found: + raise ValueError(f"no JSON object in the response: {stripped[:120]!r}") + return found.group(0) + + +def _strict(schema: dict[str, Any]) -> dict[str, Any]: + """Tighten a Pydantic schema into the shape strict mode expects. + + Every object closed to extra properties, every property required. Gateways + that enforce schemas tend to demand this, and Pydantic does not emit it. + """ + if schema.get("type") == "object" or "properties" in schema: + schema["additionalProperties"] = False + if "properties" in schema: + schema["required"] = list(schema["properties"]) + for key in ("properties", "$defs", "definitions"): + for value in schema.get(key, {}).values(): + if isinstance(value, dict): + _strict(value) + for key in ("items", "prefixItems"): + value = schema.get(key) + if isinstance(value, dict): + _strict(value) + elif isinstance(value, list): + for item in value: + if isinstance(item, dict): + _strict(item) + for combinator in ("anyOf", "oneOf", "allOf"): + for item in schema.get(combinator, []): + if isinstance(item, dict): + _strict(item) + return schema diff --git a/apps/api/src/firenze/model/port.py b/apps/api/src/firenze/model/port.py new file mode 100644 index 0000000..e98fd26 --- /dev/null +++ b/apps/api/src/firenze/model/port.py @@ -0,0 +1,61 @@ +"""The only thing the rest of the codebase knows about language models. + +One method: given a system prompt, a user prompt and a schema, return an +instance of that schema. Everything a provider offers beyond that — streaming +shapes, tool calling, thinking budgets, cache controls — stays behind the +adapter, because the day the provider changes, whatever leaked through this +interface is what has to be rewritten. + +The narrowness is the point, and it is affordable here for a reason specific to +this project: nothing in the deduction path asks a model for anything. The +solver, the validator, the verdict and the scoring are code (RN-023, RN-032). +A model writes prose and proposes a stance, and both arrive as a validated +schema. An application whose business logic depended on tool calling could not +draw this line so tightly. +""" + +from typing import Protocol, TypeVar + +from pydantic import BaseModel + +Schema = TypeVar("Schema", bound=BaseModel) + + +class ModelUnavailable(RuntimeError): + """No usable model: no provider configured, no credentials, transport failed. + + Callers degrade rather than retry. Prose is a luxury; the mystery is not. + """ + + +class ModelRefused(RuntimeError): + """The provider declined to answer. + + Kept separate from unavailability on purpose. A refusal is a fact about the + request — worth surfacing, worth counting in the evals — and burying it in a + generic failure would hide it exactly when it matters. + """ + + +class StructuredModel(Protocol): + """What a provider must supply. Adapters live in this package, not elsewhere.""" + + @property + def name(self) -> str: + """Recorded alongside anything the model wrote, so output can be traced + back to what produced it. A veneer written by a fake must never be + mistaken for one written by a real model.""" + ... + + def complete( + self, + *, + system: str, + user: str, + schema: type[Schema], + max_tokens: int, + ) -> Schema: + """Return an instance of `schema`, or raise `ModelUnavailable` / + `ModelRefused`. Never returns partially valid data: a response that does + not fit the schema is a failure, not a value to repair.""" + ... diff --git a/apps/api/src/firenze/veneer/writer.py b/apps/api/src/firenze/veneer/writer.py index 63ead9e..f2c9bf3 100644 --- a/apps/api/src/firenze/veneer/writer.py +++ b/apps/api/src/firenze/veneer/writer.py @@ -12,24 +12,22 @@ output is not there to catch the model; it is there to catch us, because a canary in this output means context assembly upstream is broken (RN-010, RN-012). -The prompt lives in `prompts/veneer/`, versioned, never as a string literal. +The prompt lives in `prompts/veneer/`, versioned, never as a string literal, +and the model arrives through the port in `firenze.model` — this module has +never heard of a provider (ADR-0007). """ import os import re from pathlib import Path -from typing import Any, Protocol, cast from firenze.domain import Case from firenze.i18n import Catalog +from firenze.model import ModelRefused, ModelUnavailable, StructuredModel from firenze.veneer.models import CaseVeneer, VeneerDraft from firenze.veneer.validation import check PROMPT_VERSION = "v1" -# Haiku: the veneer is short, structured, and its failure modes are caught by -# validation rather than by model quality. ~US$ 0.0025 per case against a -# R$ 0,50 per-match budget that phase 2 will spend on six NPCs and a turn each. -DEFAULT_MODEL = "claude-haiku-4-5" MAX_TOKENS = 4000 @@ -37,17 +35,6 @@ class VeneerUnavailable(RuntimeError): """The veneer could not be produced. The case is still playable without it.""" -class _Parseable(Protocol): - """The slice of the SDK this module uses, so tests can supply their own.""" - - def parse(self, **kwargs: Any) -> Any: ... - - -class _Client(Protocol): - @property - def messages(self) -> _Parseable: ... - - def prompts_dir() -> Path: """Repository `prompts/`, or wherever `FIRENZE_PROMPTS_DIR` points.""" override = os.environ.get("FIRENZE_PROMPTS_DIR") @@ -86,41 +73,26 @@ def _render_prompt(case: Case, catalog: Catalog) -> tuple[str, str]: ) -def write( - case: Case, - catalog: Catalog, - *, - client: _Client | None = None, - model: str = DEFAULT_MODEL, -) -> CaseVeneer: +def write(case: Case, catalog: Catalog, *, model: StructuredModel) -> CaseVeneer: """Write the veneer for an approved case, or raise `VeneerUnavailable`. A rejected draft is discarded rather than repaired: a model that broke the cast list once will break it differently on a patch, and a half-corrected veneer is harder to reason about than none. """ - if client is None: - client = _default_client() - system, user = _render_prompt(case, catalog) try: - response = client.messages.parse( - model=model, - max_tokens=MAX_TOKENS, + draft = model.complete( system=system, - messages=[{"role": "user", "content": user}], - output_format=VeneerDraft, + user=user, + schema=VeneerDraft, + max_tokens=MAX_TOKENS, ) - except Exception as failure: - raise VeneerUnavailable(f"the model call failed: {failure}") from failure - - if getattr(response, "stop_reason", None) == "refusal": - raise VeneerUnavailable("the model declined to write this case") - - draft = response.parsed_output - if draft is None: - raise VeneerUnavailable("the model returned no parseable draft") + except ModelRefused as refusal: + raise VeneerUnavailable(f"the model declined to write this case: {refusal}") from refusal + except ModelUnavailable as unavailable: + raise VeneerUnavailable(str(unavailable)) from unavailable check(draft, case) @@ -130,21 +102,7 @@ def write( setting=case.setting, prompt_version=PROMPT_VERSION, locale=catalog.locale, - model=model, + model=model.name, scene=draft.scene, characters=draft.characters, ) - - -def _default_client() -> _Client: - try: - import anthropic - except ImportError as missing: # pragma: no cover - the dependency is declared - raise VeneerUnavailable("the anthropic sdk is not installed") from missing - - try: - # The SDK's own `parse` signature is narrower than the slice we use; - # the cast is the one place where that difference is acknowledged. - return cast("_Client", anthropic.Anthropic()) - except Exception as failure: - raise VeneerUnavailable(f"no usable credentials: {failure}") from failure diff --git a/apps/api/tests/test_model_port.py b/apps/api/tests/test_model_port.py new file mode 100644 index 0000000..75ebaf2 --- /dev/null +++ b/apps/api/tests/test_model_port.py @@ -0,0 +1,244 @@ +"""Port and adapter tests. + +The contract is one method, so these are short. What they protect is the +boundary: if an adapter starts leaking provider vocabulary through the port, or +the fake starts looking like a real model, the next provider change stops being +a new file and becomes a refactor. +""" + +from typing import Any + +import pytest +from pydantic import BaseModel + +from firenze.domain import Role +from firenze.model import ( + FakeModel, + ModelRefused, + ModelUnavailable, + OpenAICompatibleModel, + resolve, +) +from firenze.model.fake import MARKER +from firenze.model.openai_compatible import _strict + + +class Nested(BaseModel): + label: str + + +class Answer(BaseModel): + line: str + stance: Role + lied: bool + turns: int + parts: tuple[Nested, ...] + + +def test_the_fake_fills_any_schema() -> None: + answer = FakeModel().complete(system="s", user="u", schema=Answer, max_tokens=100) + + assert MARKER in answer.line + assert answer.stance is Role.victim # first member, deterministically + assert answer.lied is False + assert answer.turns == 0 + assert len(answer.parts) == 1, "collections get one element, never zero" + assert MARKER in answer.parts[0].label + + +def test_the_fake_is_deterministic() -> None: + """A fake that varied would make a failing test look flaky.""" + first = FakeModel().complete(system="s", user="u", schema=Answer, max_tokens=100) + second = FakeModel().complete(system="s", user="u", schema=Answer, max_tokens=100) + + assert first == second + + +def test_the_fake_answers_differently_to_a_different_prompt() -> None: + first = FakeModel().complete(system="s", user="one", schema=Answer, max_tokens=100) + second = FakeModel().complete(system="s", user="two", schema=Answer, max_tokens=100) + + assert first.line != second.line + + +def test_the_fake_never_claims_to_be_a_real_model() -> None: + """Its name is recorded on everything it writes, so output stays traceable.""" + assert FakeModel().name == "fake" + + +def test_the_fake_can_be_asked_to_refuse() -> None: + with pytest.raises(ModelRefused): + FakeModel(refuse=True).complete(system="s", user="u", schema=Answer, max_tokens=10) + + +def test_no_provider_configured_is_an_explicit_failure() -> None: + """Not a silent default. The provider for this project is not chosen yet.""" + with pytest.raises(ModelUnavailable, match="no model provider configured"): + resolve("none") + + +def test_an_unknown_provider_says_so() -> None: + with pytest.raises(ModelUnavailable, match="unknown model provider"): + resolve("mistral-via-carrier-pigeon") + + +def test_resolve_builds_what_it_was_asked_for() -> None: + assert resolve("fake").name == "fake" + assert resolve("prosa", model="qwen-whatever", base_url="https://x").name == "qwen-whatever" + + +def test_prosa_without_a_model_name_fails_loudly() -> None: + with pytest.raises(ModelUnavailable, match="needs FIRENZE_MODEL_NAME"): + resolve("prosa") + + +class Reply: + """The slice of an OpenAI-shaped response this adapter reads.""" + + def __init__(self, content: str | None, finish_reason: str = "stop") -> None: + message = type("M", (), {"content": content}) + choice = type("C", (), {"message": message, "finish_reason": finish_reason}) + self.choices = [choice] + + +class Gateway: + """A chat-completions endpoint that supports only the modes it was told to.""" + + PADRAO = ( + '{"line": "ok", "stance": "victim", "lied": false, "turns": 1, "parts": [{"label": "x"}]}' + ) + + def __init__(self, supports: set[str], reply: str = PADRAO) -> None: + self._supports = supports + self._reply = reply + self.modes_tried: list[str] = [] + + @property + def chat(self) -> "Gateway": + return self + + @property + def completions(self) -> "Gateway": + return self + + def create(self, **kwargs: Any) -> Reply: + fmt = kwargs.get("response_format") or {} + mode = fmt.get("type", "prompt") + self.modes_tried.append(mode) + if mode not in self._supports: + raise ValueError(f"unsupported response_format: {mode}") + return Reply(self._reply) + + +def _model(gateway: Gateway) -> OpenAICompatibleModel: + return OpenAICompatibleModel( + model="qwen-whatever", base_url="https://x", api_key="k", client=gateway + ) + + +def test_it_uses_a_server_enforced_schema_when_the_gateway_supports_one() -> None: + gateway = Gateway(supports={"json_schema"}) + + answer = _model(gateway).complete(system="s", user="u", schema=Answer, max_tokens=10) + + assert gateway.modes_tried == ["json_schema"] + assert answer.line == "ok" + + +def test_it_falls_back_to_json_mode_when_schemas_are_rejected() -> None: + """Prosa's documentation does not say which modes it implements.""" + gateway = Gateway(supports={"json_object"}) + + answer = _model(gateway).complete(system="s", user="u", schema=Answer, max_tokens=10) + + assert gateway.modes_tried == ["json_schema", "json_object"] + assert answer.line == "ok" + + +def test_it_falls_back_to_asking_in_the_prompt() -> None: + gateway = Gateway(supports={"prompt"}) + + answer = _model(gateway).complete(system="s", user="u", schema=Answer, max_tokens=10) + + assert gateway.modes_tried == ["json_schema", "json_object", "prompt"] + assert answer.line == "ok" + + +def test_the_working_mode_is_remembered() -> None: + """The cost of not knowing what the gateway supports is paid once.""" + gateway = Gateway(supports={"prompt"}) + model = _model(gateway) + + model.complete(system="s", user="u", schema=Answer, max_tokens=10) + model.complete(system="s", user="u", schema=Answer, max_tokens=10) + + assert gateway.modes_tried == ["json_schema", "json_object", "prompt", "prompt"] + + +def test_json_wrapped_in_prose_is_still_read() -> None: + """A model told to answer in JSON often says "here you go:" first.""" + corpo = ( + '{"line": "ok", "stance": "victim", "lied": false, "turns": 1, "parts": [{"label": "x"}]}' + ) + envelope = f"""Claro! ```json +{corpo} +``` espero ter ajudado""" + gateway = Gateway(supports={"prompt"}, reply=envelope) + + answer = _model(gateway).complete(system="s", user="u", schema=Answer, max_tokens=10) + + assert answer.line == "ok" + + +def test_a_response_that_does_not_fit_the_schema_is_discarded() -> None: + """Never repaired. A half-parsed answer reaching the game is worse than none.""" + gateway = Gateway(supports={"json_schema"}, reply='{"line": "ok"}') + + with pytest.raises(ModelUnavailable, match="did not fit the schema"): + _model(gateway).complete(system="s", user="u", schema=Answer, max_tokens=10) + + +def test_a_content_filter_is_a_refusal_not_a_failure() -> None: + class Filtered(Gateway): + def create(self, **kwargs: Any) -> Reply: + return Reply(None, finish_reason="content_filter") + + with pytest.raises(ModelRefused): + _model(Filtered(supports={"json_schema"})).complete( + system="s", user="u", schema=Answer, max_tokens=10 + ) + + +def test_strict_mode_closes_every_object_in_the_schema() -> None: + """Gateways that enforce schemas demand this; Pydantic does not emit it.""" + tightened = _strict(Answer.model_json_schema()) + + assert tightened["additionalProperties"] is False + assert set(tightened["required"]) == set(tightened["properties"]) + nested = tightened["$defs"]["Nested"] + assert nested["additionalProperties"] is False + + +def test_no_module_outside_the_port_imports_a_provider_sdk() -> None: + """The boundary this ADR draws has to survive people who have not read it. + + Provider vocabulary spreading module by module is the failure ADR-0007 + exists to prevent, and it never announces itself — it looks like one + reasonable import at a time. + """ + import pathlib + import re + + source = pathlib.Path(__file__).resolve().parents[1] / "src" / "firenze" + # Naming a provider as a valid config value is not coupling; importing its + # SDK is. So this looks for the import, not for the word. + sdks = "anthropic|openai|google|mistralai|cohere|ollama" + sdk_import = re.compile(rf"^\s*(?:import|from)\s+({sdks})\b", re.MULTILINE) + + offenders = [ + path.relative_to(source).as_posix() + for path in source.rglob("*.py") + if path.parent.name != "model" and sdk_import.search(path.read_text(encoding="utf-8")) + ] + + assert not offenders, f"provider SDK imported outside firenze.model: {offenders}" diff --git a/apps/api/tests/test_veneer.py b/apps/api/tests/test_veneer.py index 3473973..62f1ff7 100644 --- a/apps/api/tests/test_veneer.py +++ b/apps/api/tests/test_veneer.py @@ -14,6 +14,7 @@ from firenze.domain import Case from firenze.generation import generate from firenze.i18n import load +from firenze.model import ModelRefused, ModelUnavailable from firenze.veneer import ( CharacterVeneer, VeneerDraft, @@ -26,20 +27,23 @@ from firenze.veneer.writer import _render_prompt -class StubMessages: - def __init__(self, draft: VeneerDraft | None, stop_reason: str = "end_turn") -> None: - self.draft = draft - self.stop_reason = stop_reason - self.calls: list[dict[str, Any]] = [] +class ScriptedModel: + """A model that returns exactly what a test needs it to return.""" - def parse(self, **kwargs: Any) -> Any: - self.calls.append(kwargs) - return type("Response", (), {"parsed_output": self.draft, "stop_reason": self.stop_reason}) + def __init__(self, draft: VeneerDraft | None = None, failure: Exception | None = None) -> None: + self._draft = draft + self._failure = failure + self.calls: list[dict[str, Any]] = [] + @property + def name(self) -> str: + return "scripted" -class StubClient: - def __init__(self, draft: VeneerDraft | None, stop_reason: str = "end_turn") -> None: - self.messages = StubMessages(draft, stop_reason) + def complete(self, **kwargs: Any) -> Any: + self.calls.append(kwargs) + if self._failure is not None: + raise self._failure + return self._draft @pytest.fixture(scope="module") @@ -93,13 +97,14 @@ def test_nothing_in_the_prompt_singles_out_the_culprit(case: Case) -> None: def test_a_good_draft_becomes_a_veneer(case: Case) -> None: - client = StubClient(_good_draft(case)) + model = ScriptedModel(_good_draft(case)) - veneer = write(case, load("pt-BR"), client=client, model="stub-model") + veneer = write(case, load("pt-BR"), model=model) assert veneer.seed == case.seed assert veneer.locale == "pt-BR" assert veneer.prompt_version == "v1" + assert veneer.model == "scripted", "output records what wrote it" # Prose is written for one setting. Carrying it keeps a cached veneer from # being reused for a different world that happened to share a seed. assert veneer.setting == case.setting @@ -182,21 +187,30 @@ def test_a_blank_field_is_rejected(case: Case) -> None: def test_a_refusal_is_not_a_crash(case: Case) -> None: - client = StubClient(None, stop_reason="refusal") + model = ScriptedModel(failure=ModelRefused("policy")) with pytest.raises(VeneerUnavailable, match="declined"): - write(case, load("pt-BR"), client=client) + write(case, load("pt-BR"), model=model) -def test_a_transport_failure_becomes_veneer_unavailable(case: Case) -> None: - class Exploding: - def parse(self, **kwargs: Any) -> Any: - raise ConnectionError("no route to host") +def test_an_unavailable_model_is_not_a_crash(case: Case) -> None: + model = ScriptedModel(failure=ModelUnavailable("no route to host")) - client = type("C", (), {"messages": Exploding()})() + with pytest.raises(VeneerUnavailable, match="no route"): + write(case, load("pt-BR"), model=model) + + +def test_a_fake_veneer_is_rejected_by_domain_validation(case: Case) -> None: + """The fake fits the schema and not the case, and that is the right outcome. + + It invents a cast that belongs to no mystery, so `check` refuses it. Worth + asserting: it is the line between "this pipeline runs offline" and "this + pipeline produces something a player could be shown". + """ + from firenze.model import FakeModel - with pytest.raises(VeneerUnavailable, match="model call failed"): - write(case, load("pt-BR"), client=client) + with pytest.raises(VeneerRejected, match="cast"): + write(case, load("pt-BR"), model=FakeModel()) def test_the_prompt_file_is_the_source_of_truth() -> None: diff --git a/apps/api/uv.lock b/apps/api/uv.lock index a32f4b8..b0e9f08 100644 --- a/apps/api/uv.lock +++ b/apps/api/uv.lock @@ -24,24 +24,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, ] -[[package]] -name = "anthropic" -version = "1.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "docstring-parser" }, - { name = "httpx2" }, - { name = "jiter" }, - { name = "pydantic" }, - { name = "sniffio" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/95/1a/b5af41cc1fa14da277ec20ca5554dd2fcbc09b8523ac59b7a97fbb88e452/anthropic-1.2.0.tar.gz", hash = "sha256:12f8eedee7b7fb5685837b1371b7bfae1b281703f62355f4632598ec2fc53b34", size = 1137443, upload-time = "2026-08-27T20:29:12.68Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ed/78/3f8b52708b03309e511990700bb8d0ec7a0c9db3d2a1e0d1c3ca417a4604/anthropic-1.2.0-py3-none-any.whl", hash = "sha256:b60642b3e3cd6b8e3e328a2d3f2863ad2b6e743f1037e42cc0143f7df99f63c6", size = 1289535, upload-time = "2026-08-27T20:29:11.01Z" }, -] - [[package]] name = "anyio" version = "4.14.2" @@ -145,15 +127,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] -[[package]] -name = "docstring-parser" -version = "0.18.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e0/4d/f332313098c1de1b2d2ff91cf2674415cc7cddab2ca1b01ae29774bd5fdf/docstring_parser-0.18.0.tar.gz", hash = "sha256:292510982205c12b1248696f44959db3cdd1740237a968ea1e2e7a900eeb2015", size = 29341, upload-time = "2026-04-14T04:09:19.867Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl", hash = "sha256:b3fcbed555c47d8479be0796ef7e19c2670d428d72e96da63f3a40122860374b", size = 22484, upload-time = "2026-04-14T04:09:18.638Z" }, -] - [[package]] name = "fastapi" version = "0.141.1" @@ -175,8 +148,8 @@ name = "firenze-api" version = "0.1.0" source = { editable = "." } dependencies = [ - { name = "anthropic" }, { name = "fastapi" }, + { name = "openai" }, { name = "pydantic-settings" }, { name = "uvicorn", extra = ["standard"] }, ] @@ -191,10 +164,10 @@ dev = [ [package.metadata] requires-dist = [ - { name = "anthropic", specifier = ">=1.2.0" }, { name = "fastapi", specifier = ">=0.115" }, { name = "httpx", marker = "extra == 'dev'", specifier = ">=0.28" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.14" }, + { name = "openai", specifier = ">=3.6.0" }, { name = "pydantic-settings", specifier = ">=2.7" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.9" }, @@ -517,6 +490,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, ] +[[package]] +name = "openai" +version = "3.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "httpx2" }, + { name = "jiter" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/76/913b755a1a6b54e2d9140eb8d488aa0d47c7359b1d7eac5e864cb7913bbf/openai-3.6.0.tar.gz", hash = "sha256:18fe3f6e96390ef41ee27b152fc9effefca321c33673bd9b956a572493d3ab9b", size = 1455376, upload-time = "2026-08-28T22:29:18.268Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a1/94/805b87ecc951c49ec8f247f5e8eb324ab064bd2ad73b6a0e704dd49aa073/openai-3.6.0-py3-none-any.whl", hash = "sha256:508e2158bf971687f953b62e44b02f207792c815aac306816386d7ba34d37f5f", size = 1699841, upload-time = "2026-08-28T22:29:16.436Z" }, +] + [[package]] name = "packaging" version = "26.3" diff --git a/docs/00-plano-de-projeto.md b/docs/00-plano-de-projeto.md index ab80dea..7760f78 100644 --- a/docs/00-plano-de-projeto.md +++ b/docs/00-plano-de-projeto.md @@ -177,13 +177,17 @@ ADR escrita para convencer, não para decidir. estrutura, não frase; idioma é propriedade da partida - [`0006`](adr/0006-english-in-code-portuguese-in-the-product.md) — código em inglês, produto em português +- [`0007`](adr/0007-one-port-for-any-model-provider.md) — uma porta para + qualquer fornecedor de modelo +- [`0008`](adr/0008-magalu-prosa-as-the-model-provider.md) — Magalu Prosa como + fornecedor, com adaptador OpenAI-compatible Todas as ADRs são escritas em inglês (ADR-0006) — as quatro primeiras foram traduzidas depois de escritas. **Na fila:** LangGraph vs orquestração própria · streaming SSE vs WebSocket · -API externa vs LLM self-hosted na Magalu · isolamento de contexto como fronteira -de segurança · VM+compose antes de Kubernetes. Numeração sai na ordem em que a +API externa vs LLM self-hosted na Magalu · isolamento de contexto +como fronteira de segurança · VM+compose antes de Kubernetes. Numeração sai na ordem em que a decisão é tomada, não na ordem desta lista. ### 4.2 C4 (`docs/04-arquitetura.md`) diff --git a/docs/adr/0007-one-port-for-any-model-provider.md b/docs/adr/0007-one-port-for-any-model-provider.md new file mode 100644 index 0000000..5b12896 --- /dev/null +++ b/docs/adr/0007-one-port-for-any-model-provider.md @@ -0,0 +1,100 @@ +# ADR-0007: One port for any model provider + +## Status + +Accepted — 2026-08-31. The provider it deferred was chosen in ADR-0008. + +## Context + +The veneer was written against the Anthropic SDK directly: the import, the +client, `messages.parse`, `output_format`, `stop_reason == "refusal"`. That was +the right way to get it working and the wrong thing to keep, because the +provider is not decided. Some API will be called; which one is open. + +Building the rest of the game on top of an undecided dependency has two failure +modes. The obvious one is a rewrite when the decision lands. The subtler one is +that provider vocabulary spreads: a stance machine that knows what a +`stop_reason` is, an eval suite that counts tokens the way one vendor reports +them, a turn pipeline whose retry logic assumes one provider's error taxonomy. +None of that announces itself as coupling until the day it has to be undone. + +There is also a practical constraint worth stating: the next phases have to be +built before any key exists. Most of a turn does not need a model — dossier +assembly (RN-010), schema validation (RN-022), the stance machine (RN-023), +canary and scope filtering (RN-042), turn accounting (RN-030) are all +deterministic. Only the prose needs one. + +## Decision + +Everything a model can be asked for goes through one interface in +`firenze.model`: + +```python +def complete(self, *, system: str, user: str, schema: type[Schema], max_tokens: int) -> Schema +``` + +A system prompt, a user prompt, a schema; an instance of that schema, or a +failure. Two failures, kept apart: `ModelUnavailable` (no provider, no +credentials, transport) and `ModelRefused` (the provider declined). A refusal is +a fact about the request and belongs in the evals; burying it in a generic +failure would hide it exactly when it matters. + +Adapters live in the same package and nowhere else. Two exist: + +- **`OpenAICompatibleModel`** — any endpoint speaking the OpenAI + chat-completions dialect, which is one base URL away from being a different + provider. ADR-0008 points it at Magalu Prosa. +- **`FakeModel`** — no network, no key, no money. Deterministic, and it names + itself `fake` so anything it wrote is traceable to it. + +`FIRENZE_MODEL_PROVIDER` defaults to `none`, which raises rather than picking +one. A default provider would be the decision being made quietly by whoever set +the default. + +**This interface is affordable because of what this project already decided.** +Nothing in the deduction path asks a model for anything: the solver, the +verdict, the scoring and contradiction detection are code (RN-023, RN-032). A +model writes prose and proposes a stance, and both come back as a validated +schema. An application whose business logic ran through tool calling could not +draw the line this tightly. + +## Consequences + ++ Choosing a provider is a new file in one package plus a line of configuration. ++ The game can be built and played before that choice, and a front end can be + developed against a running backend with no key and no bill. ++ Provider vocabulary cannot spread, because no other module can name it. ++ Failures arrive already classified, so callers degrade on a policy rather than + on a vendor's exception hierarchy. +− Provider features outside the interface are unreachable without widening it: + streaming, tool calling, prompt caching, thinking budgets. Streaming in + particular will need a second method when the front end arrives, and that is a + deliberate later decision rather than an oversight. +− The port cannot express per-provider tuning — cache breakpoints, effort + levels — so cost optimisation that depends on them is invisible from outside + the adapter. +− The port ships tested against fakes only. Its first contact with a real + endpoint will find something the design did not anticipate. +− A fake that satisfies schemas will still fail domain validation, because it + invents content that belongs to no case. That is correct, and it means the + fake proves pipelines, never output quality. + +## Alternatives considered + +- **Keep calling the SDK directly and abstract when the provider is chosen.** + Cheaper today. Rejected because the coupling that hurts is not the import — it + is the vocabulary that leaks into modules written between now and then, and by + the time it is visible it is spread across the phases that were built in the + meantime. +- **Adopt a framework's model abstraction (LangChain, LiteLLM).** A ready port + supporting many providers. Rejected for now: it brings an abstraction far + wider than one method, and its own release cadence, to solve a problem this + project has already narrowed to a single call. Worth revisiting if the port + ever needs streaming, tool calling and caching at once — at which point + reimplementing it would be the mistake. +- **Decide the provider now.** Argued at the time that the decision needed + inputs nobody had: cost against real usage, injection resistance per language + from the phase-6 evals. It was then made anyway, days later and on entirely + different grounds — credits, a Brazilian cloud, a hard budget ceiling + (ADR-0008). The port is what made that safe: deciding early costs one file to + undo, so waiting bought less than it delayed. diff --git a/docs/adr/0008-magalu-prosa-as-the-model-provider.md b/docs/adr/0008-magalu-prosa-as-the-model-provider.md new file mode 100644 index 0000000..9e0b800 --- /dev/null +++ b/docs/adr/0008-magalu-prosa-as-the-model-provider.md @@ -0,0 +1,98 @@ +# ADR-0008: Magalu Prosa as the model provider + +## Status + +Accepted — 2026-08-31. Blocked on the product leaving pilot. + +## Context + +ADR-0007 put every model call behind one port and deliberately left the provider +open, saying the decision needed inputs that did not exist yet: cost measured +against real usage, and injection resistance per language measured on the phase-6 +eval suite. + +The decision is being made before those inputs exist, on different grounds. That +is worth stating plainly rather than pretending the evidence arrived early. + +The grounds that do exist: + +- **The credits are already there.** Magalu Cloud credits are held; an API + billed to a card is not. For a study project, the difference between spending + credits and spending money is the difference between experimenting freely and + rationing. +- **The infrastructure is already there.** ADR-0002 puts Postgres with pgvector + on Magalu, and phase 7 puts the application there. Inference in the same place + means one account, one bill, one network. +- **Prosa speaks the OpenAI dialect.** Its documentation is explicit: *"uma API + compatível com o padrão OpenAI"*. So the adapter is not a Prosa adapter — it + is an OpenAI-compatible adapter with a base URL, and every other compatible + endpoint is reachable by configuration. +- **A monthly budget in R$ that pauses consumption at the limit.** The eval plan + caps cost per match; a provider that enforces a ceiling is closer to that + requirement than one that emails an invoice. +- **Supporting a Brazilian cloud is a reason the author holds.** Not an + engineering argument, and it does not need to pretend to be one. + +## Decision + +Magalu Prosa, through a generic OpenAI-compatible adapter in +`firenze.model.openai_compatible`, configured by `FIRENZE_MODEL_BASE_URL`, +`FIRENZE_MODEL_NAME` and `FIRENZE_MODEL_API_KEY`. + +The default provider stays `none` until the product leaves pilot and credentials +exist. `fake` covers development in the meantime. + +**Structured output is handled by the adapter, because the documentation does +not say what the gateway supports.** It tries a server-enforced JSON schema, +then JSON mode with the schema in the prompt, then a plain request parsed out of +the reply — keeping whichever works. All three failing is a failure; a response +that does not validate is discarded, never repaired (RN-022). + +## Consequences + ++ Inference, database and hosting on one account, paid with credits already + held. ++ A hard monthly ceiling in the currency the eval plan is written in. ++ The adapter is generic, so a second provider — or a local vLLM — is a base URL + away. Nothing about this decision is expensive to reverse, which is what makes + it safe to take early. ++ The project gets an unusual measurement out of it: open-weights models, served + from a Brazilian cloud, answering in Portuguese, scored on an adversarial + suite. That is a more interesting result than passing with a frontier model. +− **The catalog is open-weights — Google, Meta, NVIDIA, Qwen — and those models + are weaker exactly where phase 3 is hardest.** Injection resistance ≥ 95% and + persona consistency are the axes where model strength shows most, and ADR-0005 + already records that resistance degrades outside English. Portuguese plus + open weights is likely the hardest combination this project could pick. +− **Pilot means no guarantees**: the catalog can change, rate limits are + undocumented, and there is no SLA. Acceptable for a study project and not + acceptable for anything else. +− Structured output may cost an extra round trip on every call if the gateway + supports neither schema enforcement nor JSON mode. +− The provider cannot be exercised until launch, so the adapter ships tested + against fakes only. Its first contact with a real endpoint will find something. + +### What does not degrade with a weaker model + +Worth being precise, because "weaker model" sounds like it threatens everything +and does not. The canary gate is 0% and stays 0%: isolation is a data boundary, +the model is never given a secret fact, and the filter is code (RN-010, RN-012). +The verdict, the score and contradiction detection are deterministic (RN-032). +What a weaker model costs is **quality** — a character who breaks voice, a reply +in the wrong shape, an injection classified wrong. Those are measurable, and the +eval suite exists to measure them. + +## Alternatives considered + +- **A frontier provider billed to a card.** Stronger on precisely the axes + phase 3 measures. Rejected for now: it spends money the project does not need + to spend, and it would leave R$ 300 of credits unused while making the + cheapest possible interesting experiment impossible. +- **Wait for the phase-6 evals, as ADR-0007 said.** Consistent, and it would + block phases 2 to 5 on a decision whose reversal costs one file. The port is + what makes deciding early cheap; refusing to use it would waste the design. +- **Self-hosted model on a Magalu GPU VM.** Investigated: the account currently + lists 50 machine types and none with a GPU, so it needs a quota request. Also + bills per hour of uptime rather than per token, which inverts the economics + for intermittent development use. Stays a phase-8 experiment, to be run + against the eval suite and written up.