diff --git a/.clusterfuzzlite/sql_safety_fuzzer.py b/.clusterfuzzlite/sql_safety_fuzzer.py index 926def13..cd5114f7 100644 --- a/.clusterfuzzlite/sql_safety_fuzzer.py +++ b/.clusterfuzzlite/sql_safety_fuzzer.py @@ -11,6 +11,7 @@ known-shape attacks; this harness covers unknown-shape inputs). """ +import contextlib import sys import atheris @@ -27,10 +28,8 @@ def test_one_input(data: bytes) -> None: fdp = atheris.FuzzedDataProvider(data) query = fdp.ConsumeUnicodeNoSurrogates(fdp.remaining_bytes()) - try: + with contextlib.suppress(ValueError): # expected: malformed or policy-disallowed SQL is rejected _driver._validate(query) # fuzzing the private validator directly - except ValueError: - pass # expected: malformed or policy-disallowed SQL is rejected atheris.Setup(sys.argv, test_one_input) diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..32fa564d --- /dev/null +++ b/.editorconfig @@ -0,0 +1,21 @@ +root = true + +[*] +indent_style = space +indent_size = 4 +charset = utf-8 +end_of_line = lf +trim_trailing_whitespace = true +insert_final_newline = true + +[*.py] +indent_size = 4 + +[*.md] +trim_trailing_whitespace = false + +[*.{yml,yaml,json,toml}] +indent_size = 2 + +[*.{bat,cmd}] +end_of_line = crlf diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..3100640d --- /dev/null +++ b/.env.example @@ -0,0 +1,149 @@ +# MCPg configuration template — copy to .env and fill in real values. +# Every MCPG_* variable this server reads; see docs/user-guide.md for full detail on each. + +# --- Database (required) --- +MCPG_DATABASE_URL=postgresql://user:password@localhost:5432/dbname + +# --- Secondary databases (optional) --- +# MCPG_SECONDARY_DATABASE_URLS=db1=postgresql://user:password@localhost:5432/db1,db2=postgresql://user:password@localhost:5432/db2 + +# --- Read replicas (optional) --- +# MCPG_REPLICA_URLS=postgresql://user:password@replica1:5432/dbname,postgresql://user:password@replica2:5432/dbname + +# --- Access mode --- +# MCPG_ACCESS_MODE=read-only # read-only | restricted | unrestricted + +# --- Capability gates (unrestricted mode only) --- +# MCPG_ALLOW_DDL=false +# MCPG_ALLOW_SHELL=false +# MCPG_ALLOW_LISTEN=false +# MCPG_ALLOW_INSECURE_TLS=false + +# --- Connection pool --- +# MCPG_POOL_MIN_SIZE=1 +# MCPG_POOL_MAX_SIZE=5 + +# --- HTTP transport --- +# MCPG_TRANSPORT=stdio # stdio | streamable-http | sse +# MCPG_HTTP_HOST=127.0.0.1 +# MCPG_HTTP_PORT=8000 +# MCPG_HTTP_AUTH_TOKEN=your-bearer-token +# MCPG_HTTP_MAX_BODY_BYTES=1048576 +# MCPG_HTTP_ALLOWED_ORIGINS=http://localhost:3000,https://example.com +# MCPG_HTTP_IP_ALLOWLIST=127.0.0.1,192.168.1.0/24 +# MCPG_HTTP_HSTS_MAX_AGE=63072000 +# MCPG_HTTP_REQUEST_TIMEOUT_SECONDS=0 +# MCPG_HTTP_TRUSTED_HOSTS=api.example.com,*.example.com + +# --- HTTP transport TLS/mTLS --- +# MCPG_HTTP_TLS_CERTFILE=/path/to/cert.pem +# MCPG_HTTP_TLS_KEYFILE=/path/to/key.pem +# MCPG_HTTP_TLS_CA_CERTS=/path/to/ca-certs.pem +# MCPG_HTTP_TLS_CLIENT_CERT_REQUIRED=false + +# --- Authentication --- +# MCPG_AUTH_MODE=static # static | oidc +# MCPG_OIDC_ISSUER=https://example.auth0.com/ +# MCPG_OIDC_AUDIENCE=https://api.example.com +# MCPG_OIDC_JWKS_URL=https://example.auth0.com/.well-known/jwks.json +# MCPG_OIDC_ROLE_CLAIM=pg_role + +# --- Multi-tenancy via PG roles --- +# MCPG_DEFAULT_ROLE=app_user +# MCPG_ALLOWED_ROLES=app_user,analyst,admin + +# --- Logging --- +# MCPG_LOG_LEVEL=INFO # DEBUG | INFO | WARNING | ERROR | CRITICAL +# MCPG_LOG_FORMAT=text # text | json + +# --- Shell (PG binaries: pg_dump, pg_restore, psql) --- +# MCPG_SHELL_TIMEOUT_SEC=60 +# MCPG_SHELL_MAX_OUTPUT_BYTES=67108864 +# MCPG_SUBPROCESS_BIN_ALLOWLIST=/usr/lib/postgresql/bin:/usr/local/bin +# MCPG_SUBPROCESS_CPU_SECONDS=300 +# MCPG_SUBPROCESS_MEMORY_MB=512 + +# --- Query execution --- +# MCPG_STATEMENT_TIMEOUT_MS=30000 +# MCPG_LOCK_TIMEOUT_MS=5000 +# MCPG_SLOW_CALL_THRESHOLD_MS=1000 +# MCPG_ELICIT_CONFIRM_WRITES=false +# MCPG_LISTEN_QUEUE_MAX=1000 + +# --- Analytical queries --- +# MCPG_ENABLE_ANALYTICAL_QUERIES=true +# MCPG_ANALYTICAL_TIMEOUT_MS=120000 +# MCPG_ANALYTICAL_MAX_TIMEOUT_MS=600000 +# MCPG_ANALYTICAL_MAX_CONCURRENCY=2 + +# --- Audit trail --- +# MCPG_AUDIT_PERSIST=false +# MCPG_AUDIT_REDACT_KEYS=password,secret,token # Comma-separated keys to redact in audit logs +# MCPG_AUDIT_EVENTS_BACKEND=timescaledb # timescaledb | pg_partman | native +# MCPG_AUDIT_EVENTS_RETENTION_DAYS=90 +# MCPG_AUDIT_EVENTS_CHUNK_INTERVAL=1 day +# MCPG_AUDIT_EVENTS_COMPRESS_AFTER=7 days +# MCPG_AUDIT_EVENTS_RLS=true +# MCPG_AUDIT_EVENTS_READER_ROLE=audit_reader + +# --- RAG telemetry --- +# MCPG_RAG_TELEMETRY_BACKEND=timescaledb # timescaledb | pg_partman | native +# MCPG_RAG_TELEMETRY_RETENTION_DAYS=90 +# MCPG_RAG_TELEMETRY_CHUNK_INTERVAL=1 day +# MCPG_RAG_TELEMETRY_COMPRESS_AFTER=7 days +# MCPG_RAG_TELEMETRY_RLS=true +# MCPG_RAG_TELEMETRY_READER_ROLE=rag_reader + +# --- NL→SQL (natural language to SQL) --- +# MCPG_NL2SQL_PROVIDER=anthropic # anthropic | openai | gemini | deepinfra | etc. +# MCPG_NL2SQL_API_KEY=your-nl2sql-provider-api-key +# MCPG_NL2SQL_MODEL=claude-3-5-sonnet-20241022 +# MCPG_NL2SQL_BASE_URL=https://api.anthropic.com/v1 +# MCPG_NL2SQL_MAX_TOKENS=2048 +# MCPG_NL2SQL_CUSTOM_PROVIDERS=ollama=http://localhost:11434|llama2 + +# --- NL→SQL audit --- +# MCPG_NL2SQL_AUDIT_PERSIST=false +# MCPG_NL2SQL_AUDIT_BACKEND=timescaledb # timescaledb | pg_partman | native +# MCPG_NL2SQL_AUDIT_RETENTION_DAYS=90 +# MCPG_NL2SQL_AUDIT_CHUNK_INTERVAL=1 day +# MCPG_NL2SQL_AUDIT_COMPRESS_AFTER=7 days +# MCPG_NL2SQL_AUDIT_RLS=true +# MCPG_NL2SQL_AUDIT_READER_ROLE=nl2sql_audit_reader + +# --- Rate limiting --- +# MCPG_RATE_LIMIT_ENABLED=false +# MCPG_RATE_LIMIT_MAX_REQUESTS=60 +# MCPG_RATE_LIMIT_WINDOW_SECONDS=60 +# MCPG_RATE_LIMIT_HEAVY_MAX=5 +# MCPG_RATE_LIMIT_HEAVY_WINDOW=60 + +# --- Caching --- +# MCPG_CACHE_ENABLED=true +# MCPG_CACHE_TTL_SECONDS=300 +# MCPG_CACHE_MAXSIZE=1024 +# MCPG_REDIS_URL=redis://localhost:6379/0 + +# --- OpenTelemetry (OTEL) tracing --- +# MCPG_OTEL_ENABLED=false +# MCPG_OTEL_SERVICE_NAME=mcpg + +# --- Session intent --- +# MCPG_SESSION_INTENT=lookup,migration,vector_rag,monitor,admin +# MCPG_DYNAMIC_SESSION_INTENT=false + +# --- Observability and diagnostics --- +# MCPG_ENABLE_HEAVY_DIAGNOSTICS=true + +# --- Migrations --- +# MCPG_MIGRATION_SCRIPTS_ROOTS=/app/migrations + +# --- Audit integrity --- +# MCPG_AUDIT_HMAC_KEY=your-secret-hmac-key +# MCPG_AUDIT_INTEGRITY=false + +# --- Secrets backend --- +# MCPG_SECRETS_BACKEND=env # env | file | vault | aws | gcp + +# --- Server lifecycle --- +# MCPG_SHUTDOWN_DRAIN_SECONDS=30 diff --git a/.github/workflows/auto-merge-bot-prs.yml b/.github/workflows/auto-merge-bot-prs.yml index be423136..037d83ab 100644 --- a/.github/workflows/auto-merge-bot-prs.yml +++ b/.github/workflows/auto-merge-bot-prs.yml @@ -13,6 +13,11 @@ jobs: auto-merge-bot-prs: name: Auto-merge unsigned bot PRs runs-on: ubuntu-latest + # The `[bot]`-suffix username namespace is reserved for GitHub App identities + # (confirmed via `gh api users/dependabot%5Bbot%5D` — `type: Bot`); a human account + # cannot register one, so the `endsWith(github.actor, '[bot]')` clause below + # doesn't broaden trust beyond the three named bots in practice. Re-verify this + # assumption if GitHub's account-namespace rules ever change. if: | github.actor == 'dependabot[bot]' || github.actor == 'renovate[bot]' || diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd43f306..553320b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,6 +70,14 @@ jobs: uv run pip-audit --strict --disable-pip -r /tmp/mcpg-reqs.txt - name: Bandit static analysis (SAST) run: uv run bandit -r src/mcpg --skip B101,B608,B110 -ll + # Non-blocking license inventory: report the dependency tree's + # license mix so a maintainer can review it, without gating merge + # on any particular license yet (no `--fail-on`). Promote to a + # blocking check once the current mix has been reviewed and an + # explicit allow/deny policy is written. + - name: License enumeration (pip-licenses) + continue-on-error: true + run: uv run pip-licenses --format=markdown --with-urls --order=license test: name: Tests (PG ${{ matrix.postgres }}) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77009934..5fa2880b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,69 @@ adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Added + +- Dev dependencies: `pytest-mock`, `pytest-randomly`, `pytest-socket`, `time-machine`, + `pytest-rerunfailures`. Of these, only `pytest-randomly` is active today — it auto-enables and + randomizes test order by default. The other four (`pytest-mock`, `pytest-socket`, `time-machine`, + `pytest-rerunfailures`) are staged for planned future test-hygiene work and are not yet wired + into any test: no `mocker` fixture, `time_machine` call, `--reruns`/`@pytest.mark.flaky` usage, + or `disable_socket`/`socket_enabled` usage exists anywhere in `tests/` as of this writing. + +- License enumeration (`pip-licenses`) added to CI as a non-blocking report step. + +- `/readyz` readiness endpoint on the HTTP transport — reports 503 when the DB pool can't currently serve + a connection, distinct from `/healthz`'s liveness-only check. Previously reserved in the auth-exemption + set but never mounted. Reads `Database.is_connected` (no fresh connection attempted on every poll); + `create_server` now stashes the primary `Database` on the server object (`server.mcpg_database`, + alongside the existing `mcpg_settings`/`otel_tracer`/`rate_limiter`) so `build_http_app` can reach it. + +- `mcpg.errors.MCPgError`, a common base class every domain-specific exception (`ConfigError`, + `DatabaseError`, `CursorError`, and 62 others — 65 exception classes across 64 modules in total) + now subclasses — lets calling code catch "any MCPg-internal error" with one type instead of an + enumerated list. No existing exception's name, message, or call sites changed. Two of the 65 + (`TenancyError`, `DynamicIntentError`) previously subclassed `ValueError` rather than `Exception`; + they now subclass both `MCPgError` and `ValueError`, so anything catching them as `ValueError` + today keeps working. + +- Optional `TrustedHostMiddleware` support via `MCPG_HTTP_TRUSTED_HOSTS` (comma-separated), off by + default, matching the existing `MCPG_HTTP_ALLOWED_ORIGINS` convention. + +- `TranslationResult` (NL→SQL) now records `schema_context` — the schema evidence actually sent to the + model for that translation — so a generated query's provenance is traceable, not just which + model/provider produced it. + +- Circuit breaker (`circuitbreaker`) around each NL→SQL provider's `complete()` call and the OIDC + discovery-document fetch (`OIDCVerifier._resolve_jwks_url`) — after 5 consecutive failures against a + degraded vendor/IdP, further calls fail fast for 30s instead of each one separately paying the full + request timeout. A tripped breaker still surfaces as the module's existing error type (`NL2SQLError` / + `OIDCError`), never a bare `CircuitBreakerError`, so calling code's exception handling is unaffected. + +- Retry with exponential backoff + jitter (`tenacity`) around the same NL→SQL provider calls and OIDC + discovery fetch, layered *inside* the circuit breaker added above (up to 3 attempts, ~0.1-2s apart) — + a single dropped connection or transient 5xx is retried transparently instead of surfacing as an + error. Because retry sits inside the breaker, an exhausted retry cycle counts as exactly one breaker + failure (not one per retry attempt), and a call that fails twice then succeeds never touches the + breaker's failure count at all. + ### Fixed +- **`run_select` / `run_select_tuned` fully materialized a query's entire result set into Python + `dict`/`RowResult` objects before truncating to `max_rows`,** rather than bounding the fetch itself — + a query without its own `LIMIT` against a large table could build millions of row objects into memory + before the truncation line ever ran. `SqlDriver.execute_query` / `SafeSqlDriver.execute_query` (and the + tenancy-role-wrapped execute path) now take an optional `row_limit` parameter and use + `cursor.fetchmany(row_limit)` instead of `cursor.fetchall()`; both call sites in `query.py` now pass + `max_rows + 1`. Note the precise scope: `psycopg`'s regular (client-side) cursor already pulls the whole + result set into libpq's buffer during `cursor.execute()`, so this bounds how many rows are converted to + Python objects, not the server-side network transfer — a true wire-level bound would need a named + server-side cursor or an injected `LIMIT`, which is out of scope here (a server-side cursor can't + support `run_select_tuned`'s `SET LOCAL ...; SELECT ...` two-statement pattern or the SHOW/EXPLAIN/VACUUM + paths `SafeSqlDriver` also allows). +- **`py.typed` marker was missing despite the `Typing :: Typed` classifier.** Added `src/mcpg/py.typed` + and verified it ships in the built wheel. +- **`license-files` in `pyproject.toml` pointed at `src/mcpg/_vendor/LICENSE`, which hasn't existed since + the SQL kernel was de-vendored (ADR-0007).** Removed the stale entry. - **Broken readiness probe on the `warehousepg-latest` CI lane was burning ~6 minutes of dead time on every run.** The lane's readiness poll used `docker exec mcpg-db pg_isready -U gpadmin`, but @@ -21,13 +82,116 @@ adheres to [Semantic Versioning](https://semver.org/). step accounted for ~6-6.5 min of the lane's ~9-10 min total vs. ~20-30s on every other PG version; the `pytest` step itself was already the same duration as any other lane). +- **Seven `except Exception: pass` sites now log at `debug` level with `exc_info=True`** instead of + swallowing silently — `advisors.py`, `audit.py`, `audit_trail.py` (×3), `listen.py`, `migrations.py`. + No control flow changed; these were already best-effort/cleanup paths and remain so, now with + observability into how often they actually fire. +- **Error-logging call sites inside `except` blocks now preserve tracebacks** (`exc_info=True`, added + to the existing `logger.error`/`logger.warning` calls) where they previously logged only the + exception's string form and lost the traceback — audited every `logger.error`/`logger.warning` call + across `src/mcpg` (33 candidate sites), 23 sites fixed across `audit_nl2sql.py` (×2), `cache.py` (×6), + `cursors.py`, `database.py` (×2), `graph_diagram.py` (×2), `http_runtime.py` (×2), `nl2sql.py` (×3), + `otel_tracing.py`, `replicas.py` (×3), and `tenancy.py`. The remaining 10 sites were left unchanged: + one (`listen.py`) already carried `exc_info=True`; nine log about something other than the exception + just caught, or aren't inside an `except` block at all (e.g. a slow-call warning, an audit-event + record, an egress notice, a config-collision warning). + +### Changed + +- **Task 23 (ruff sweep, Parts A-E + Step 11) complete.** `pyproject.toml`'s `[tool.ruff.lint] select` + list now includes `C90`, `ASYNC`, `C4`, `SIM`, `PTH`, `PYI`, and `FBT` alongside the pre-existing + `E`/`F`/`I`/`B`/`W`/`N`/`UP`/`RUF`, and `uv run ruff check .` is clean with zero outstanding + violations across all 7 categories. The `external = ["ASYNC", "C90"]` workaround added mid-sweep (so + `RUF100` wouldn't flag the mandated per-violation `# noqa` justifications as unused before these + categories were selected) is removed now that both are enforced; `RUF100` polices those ~113 `noqa` + comments normally and none are stale. No `[tool.ruff.lint.mccabe]` section was added — `C901` + findings were measured against ruff's unconfigured default `max-complexity` of 10, corroborated by + several findings at exactly 11 (`audit_database`, `tenancy._execute_with_role`, + `data_movement.dump_database`, `http_runtime.build_http_app`) that a higher threshold would have + missed. + + **418 violations addressed in total:** + - Part A — 74 mechanical fixes: `PTH` (9), `C4` (1), `SIM` (48), `PYI` (16). + - Part B — 84 `ASYNC`/`C901` findings individually assessed: `ASYNC` (18, all justified-suppressed — + each is a pass-through `timeout=` parameter already enforced by a lower layer, e.g. httpx's own + per-request timeout or `SafeSqlDriver`'s `asyncio.timeout()`); `C901` (66: 7 refactored, 59 + justified-suppressed, including the security-critical `SafeSqlDriver._validate_node`, suppressed + rather than refactored per its existing adversarial/fuzz-pinning rationale). + - Parts C+D+E+Step 11 — 260 `FBT` (flake8-boolean-trap) violations, now clean repo-wide: 103 in + `src/mcpg/tools.py` (Part C — all 43 affected functions are registered MCP tools), 30 in the rest + of `src/` + `tools/` (Part D), 126 in `tests/` (Part E), and 1 in `benchmarks/`, found only once + `FBT` was selected repo-wide in Step 11 — a scope gap (`benchmarks/` sat outside every part's glob), + fixed directly as a trivial single-call-site change. + + Assessed but deliberately left disabled, per the rescoping decision recorded at Task 23's start: `D` + (2,284 violations, outside the public `tools.py` surface), `ANN` (475, redundant with `mypy --strict`, + already clean), `TC` (165), `PT` (75, test-only) — baseline counts kept for a possible future pass. + + The substantive code change inside the `FBT` sweep is in the SQL kernel: `force_readonly` is now + keyword-only across the whole `execute_query` / `_execute_with_connection` override family + (`sql/driver.py`, `sql/safety.py`, `multidb.py`, `replicas.py`, `tenancy.py`), so a security-relevant + read-only flag can no longer be passed in the wrong positional slot. **No public MCP tool signature or + contract snapshot moved anywhere in the sweep** — a `*` marker before each signature's first boolean + parameter (never reordered) is transparent to pydantic's JSON-Schema derivation, so `tests/contract/` + passes unmodified throughout and both snapshots (`tool_surface.snapshot.json`, + `tool_return_shapes.snapshot.json`) regenerate byte-for-byte. Most non-`tools.py` `FBT` hits were + invisible to the linter itself (`FBT003` fires only on boolean *literals*, never on a boolean + *variable* forwarded positionally), so an AST arity sweep was used instead — and it caught two + genuine near-miss bugs the linter alone would have missed: a positionally-mismatched + `TenantTimeoutSqlDriver` call in `replicas.py` (would have raised `TypeError` at runtime on the + multi-tenant + timeout path), and the mandatory Part D -> Part E test-fake handoff + (`tests/unit/_fakes.py`'s three standalone fakes — `FakeDriver`, `FakeRoutingDriver`, + `FakeParamRoutingDriver` — plus their 5 positional call sites across 4 test files), verified with + `grep -rn "force_readonly" tests/` before and after so no `TypeError` slipped through. + + **Live FBT breakdown:** of the 157 hits that remained once `tools.py`'s 103 were fixed, 26 were in + `src/`, 126 in `tests/`, 4 in `tools/` dev scripts, and 1 in `benchmarks/` (the last not discovered + until Step 11 selected `FBT` repo-wide). Full per-part, per-file detail — the granular + commit-by-commit narrative this entry summarizes — is preserved in + `.superpowers/sdd/2026-08-25-audit-remediation/batch-ruff*.md`. +- Pinned `hatchling>=1.26` as the build-system floor (previously unpinned). +- HSTS `max-age` default bumped from 31536000 (1 year) to 63072000 (2 years), OWASP's current + recommendation — the old value remains the `hstspreload.org` minimum-eligibility floor, not the target. +- NL→SQL providers (`AnthropicProvider`, `OpenAIProvider`, `GeminiProvider`) now reuse one process-wide + `httpx.AsyncClient` instead of opening `async with httpx.AsyncClient(...)` fresh on every + `translate_nl_to_sql` call. A new provider instance is still built per call (provider selection can + vary per-request via `provider=`), so the client lives at module scope rather than on the instance — + closed via `mcpg.nl2sql.aclose_shared_client()`, wired into `make_lifespan`'s existing shutdown path. + The OIDC discovery-document fetch similarly moves from a per-fetch client to one held for + `OIDCVerifier`'s lifetime (`aclose()` closes it); since the verifier is constructed after + `make_lifespan`'s closure already exists, `build_http_app` wraps the ASGI app's own lifespan to close it + on shutdown instead. Each avoids paying a fresh TCP/TLS handshake per call/fetch. ### Security +- **Fixed a SQL injection defect in `describe_graph` and `generate_graph_diagram`.** Both read Apache + AGE vertex/edge label names back from `ag_catalog.ag_label` and interpolated them directly into + f-string SQL (`FROM "{graph}"."{label}"`) with no identifier validation. Postgres/AGE quoted + identifiers can contain arbitrary characters, including embedded double quotes, so a label name + created via a prior `run_cypher` write could carry attacker-controlled SQL into that later + re-interpolation. Every catalog-derived label name is now identifier-validated + (`[A-Za-z_][A-Za-z0-9_]*`) before it reaches a generated SQL string or the diagram's Mermaid output; + an invalid name aborts the call with a `GraphError` instead of being silently used, matching the + existing `graph_projection` precedent for catalog-derived identifiers. Both tools' backing count/fetch + queries also now run under `force_readonly=True`, and `describe_graph` gained the same + `Capability.READ` access-mode gate its sibling read tools (`run_cypher`'s read path, + `generate_graph_diagram`) already had. +- **BREAKING: rate limiting (`MCPG_RATE_LIMIT_ENABLED`) now defaults to `true`** (previously `false`). + Set it to `false` explicitly to restore the previous unlimited behavior. +- **BREAKING: the HTTP transport now refuses to start unauthenticated by default.** Previously it started + anyway and only logged a warning if neither `MCPG_HTTP_AUTH_TOKEN` nor `MCPG_AUTH_MODE=oidc` was set. + Deployments that relied on the unauthenticated default must now either configure auth or set + `MCPG_HTTP_ALLOW_UNAUTHENTICATED=true` to explicitly opt back in (loudly logged when set). The default + `stdio` transport is unaffected. - **Bumped transitive `pip` 26.1.2 → 26.2.1 (PYSEC-2026-3721).** `pip-audit`'s own `pip_api` dependency pulled in a `pip` version with a known vulnerability, flagged by the local pre-commit hook's dependency audit. +- Added a centralized log-redaction filter (`RedactionFilter`) as a backstop for a log call's message + or `%`-style arguments reaching a handler without having explicitly redacted a connection string + first — complements, doesn't replace, the existing per-call-site `obfuscate_password()` discipline. + Does not cover exception tracebacks attached via `exc_info` (e.g. `logger.exception(...)`) — those + are rendered separately by `logging.Formatter.formatException()`, which bypasses this filter. ## [0.8.0] - 2026-08-19 diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..70ed9585 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,83 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, caste, color, religion, or sexual identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience +* Focusing on what is best not just for us as individuals, but for the overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for moderation decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at devopam@gmail.com. All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing clarity around the nature of the violation and an explanation of why the behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series of actions. + +**Consequence**: A warning with consequences for continued behavior. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding interactions in community spaces as well as external channels like social media. Violating these terms may lead to a temporary or permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public communication with the community for a specified period of time. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.1, available at [https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct enforcement ladder][Mozilla CoC]. + +For answers to common questions about this code of conduct, see the FAQ at [https://www.contributor-covenant.org/faq][FAQ]. Translations are available at [https://www.contributor-covenant.org/translations][translations]. + +[homepage]: https://www.contributor-covenant.org +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html +[Mozilla CoC]: https://github.com/mozilla/diversity +[FAQ]: https://www.contributor-covenant.org/faq +[translations]: https://www.contributor-covenant.org/translations diff --git a/README.md b/README.md index d149387d..c7be55bf 100644 --- a/README.md +++ b/README.md @@ -210,9 +210,11 @@ mcpg ``` Then point any MCP-aware client at `http://localhost:8000/mcp` (or -`/sse` for the SSE transport). Set -`MCPG_HTTP_AUTH_TOKEN=...` for a static bearer, or -`MCPG_AUTH_MODE=oidc` for full JWT validation against an OIDC issuer. +`/sse` for the SSE transport). The HTTP transport refuses to start +unless it's authenticated — set `MCPG_HTTP_AUTH_TOKEN=...` for a static +bearer, or `MCPG_AUTH_MODE=oidc` for full JWT validation against an +OIDC issuer. To deliberately run without auth (not recommended), set +`MCPG_HTTP_ALLOW_UNAUTHENTICATED=true`. --- @@ -262,7 +264,8 @@ are one-shot commands, not configuration). The only required one is | Variable | Default | Description | |---|---|---| | `MCPG_AUTH_MODE` | `static` | `static` (compare bearer to `MCPG_HTTP_AUTH_TOKEN`) \| `oidc` (full JWT validation). | -| `MCPG_HTTP_AUTH_TOKEN` | — | Required bearer token when `MCPG_AUTH_MODE=static`. Constant-time compare. | +| `MCPG_HTTP_AUTH_TOKEN` | — | Required bearer token when `MCPG_AUTH_MODE=static`. Constant-time compare. The HTTP transport refuses to start (`ConfigError`) unless this, `MCPG_AUTH_MODE=oidc`, or `MCPG_HTTP_ALLOW_UNAUTHENTICATED=true` is set. | +| `MCPG_HTTP_ALLOW_UNAUTHENTICATED` | `false` | Explicit opt-out of the HTTP transport's fail-closed auth check. Loudly logged on every startup when set; not recommended. | | `MCPG_OIDC_ISSUER` | — | OIDC issuer URL (required when `MCPG_AUTH_MODE=oidc`). | | `MCPG_OIDC_AUDIENCE` | — | Expected `aud` claim (required when `MCPG_AUTH_MODE=oidc`). | | `MCPG_OIDC_JWKS_URL` | discovered | Override JWKS endpoint (auto-discovered from issuer's `.well-known` otherwise). | @@ -274,8 +277,9 @@ are one-shot commands, not configuration). The only required one is |---|---|---| | `MCPG_HTTP_MAX_BODY_BYTES` | `1048576` | (1 MiB) Request bodies above this get a `413`. Counts streamed bytes, so a missing/lying `Content-Length` can't bypass it. | | `MCPG_HTTP_ALLOWED_ORIGINS` | — | Comma-separated CORS allowlist. Unset = no CORS middleware (no cross-origin headers emitted). | -| `MCPG_HTTP_HSTS_MAX_AGE` | `31536000` | `Strict-Transport-Security` max-age. `0` disables the HSTS header. Security headers (CSP, X-Frame-Options, X-Content-Type-Options, Referrer-Policy) are always added unless the app already set them. | +| `MCPG_HTTP_HSTS_MAX_AGE` | `63072000` | `Strict-Transport-Security` max-age (2 years, OWASP's current recommendation). `0` disables the HSTS header. Security headers (CSP, X-Frame-Options, X-Content-Type-Options, Referrer-Policy) are always added unless the app already set them. | | `MCPG_HTTP_REQUEST_TIMEOUT_SECONDS` | `0` | Per-request wall-clock cap (`504` on expiry). `0` = disabled. Leave off if you rely on long-lived SSE / streamable-http streams — a hard cap also severs those. | +| `MCPG_HTTP_TRUSTED_HOSTS` | — | Comma-separated list of allowed `Host` header values (wildcards like `*.example.com` supported). Unset = no host-header validation (current behaviour). When set, requests with a non-matching Host get a `400` via Starlette's `TrustedHostMiddleware`. | #### Multi-tenancy (`SET ROLE`) @@ -352,7 +356,7 @@ falls back to the env var, so partial files work. | Variable | Default | Description | |---|---|---| -| `MCPG_RATE_LIMIT_ENABLED` | `false` | Enable token-bucket per-tool rate limiting. | +| `MCPG_RATE_LIMIT_ENABLED` | `true` | Token-bucket per-tool rate limiting. Set to `false` to restore the pre-breaking-change unlimited behavior. | | `MCPG_RATE_LIMIT_MAX_REQUESTS` | `60` | Global cap per window across all tools. | | `MCPG_RATE_LIMIT_WINDOW_SECONDS` | `60` | Window length for the global quota. | | `MCPG_RATE_LIMIT_HEAVY_MAX` | `5` | Cap for heavy tools (`run_write`, `run_ddl`, `dump_database`, etc.). | diff --git a/benchmarks/dashboard/generate.py b/benchmarks/dashboard/generate.py index 87e18691..0673e6c7 100644 --- a/benchmarks/dashboard/generate.py +++ b/benchmarks/dashboard/generate.py @@ -248,7 +248,11 @@ def _waterfall_chart(rows: list[dict[str, Any]]) -> str: return legend + "".join(parts) -def _throughput_chart(results: list[dict[str, Any]]) -> str: +# C901 rationale: hand-rolled SVG chart renderer (grid lines, axis ticks, +# per-path polylines with coordinate-mapping closures) -- operator-only +# benchmark tooling, not part of the shipped package; the branching is +# inline SVG geometry construction, not business logic. +def _throughput_chart(results: list[dict[str, Any]]) -> str: # noqa: C901 """Lines: throughput (queries/sec) vs concurrency, per path. Empty if no sweep.""" conc = [r for r in results if r.get("throughput_rps") is not None and r.get("concurrency", 1) >= 1] if not conc: diff --git a/benchmarks/datasets/load_tpch.py b/benchmarks/datasets/load_tpch.py index 8cbed9cb..671d8d41 100644 --- a/benchmarks/datasets/load_tpch.py +++ b/benchmarks/datasets/load_tpch.py @@ -56,11 +56,10 @@ async def load(database_url: str, scale_factor: int) -> None: for table in _TABLES: csv_path = Path(tmp) / f"{table}.csv" duck.execute(f"COPY {table} TO '{csv_path}' (FORMAT csv, HEADER false)") - async with conn.cursor() as cur: - async with cur.copy(f"COPY {table} FROM STDIN (FORMAT csv)") as copy: - with csv_path.open("rb") as fh: - while chunk := fh.read(1 << 20): - await copy.write(chunk) + async with conn.cursor() as cur, cur.copy(f"COPY {table} FROM STDIN (FORMAT csv)") as copy: + with csv_path.open("rb") as fh: + while chunk := fh.read(1 << 20): + await copy.write(chunk) csv_path.unlink() print(f" loaded {table}") duck.close() diff --git a/benchmarks/perf/decompose.py b/benchmarks/perf/decompose.py index 768872ff..2715f704 100644 --- a/benchmarks/perf/decompose.py +++ b/benchmarks/perf/decompose.py @@ -27,6 +27,7 @@ from __future__ import annotations +import contextlib import time from dataclasses import dataclass @@ -107,10 +108,8 @@ async def run_once(self, sql: str, *, max_rows: int) -> SegmentSample: # A failing sample must not return the pooled connection mid # transaction and poison later checkouts. Best-effort close, # then re-raise the original error. - try: + with contextlib.suppress(Exception): await cur.execute("ROLLBACK") - except Exception: - pass raise return SegmentSample( diff --git a/benchmarks/perf/runner.py b/benchmarks/perf/runner.py index b58677a6..0b23b09b 100644 --- a/benchmarks/perf/runner.py +++ b/benchmarks/perf/runner.py @@ -132,7 +132,11 @@ def _conc_row(path: str, query: BenchQuery, cr: ConcurrencyResult) -> ResultRow: ) -async def _run_concurrency(database_url: str, iterations: int, timeout: float) -> list[ResultRow]: +async def _run_concurrency( + database_url: str, + iterations: int, + timeout: float, # noqa: ASYNC109 -- converted to a statement_timeout_ms setting, not a manual reimplementation +) -> list[ResultRow]: """Sweep the **ultralight** queries across the concurrency levels. Throughput-under-load exists to expose the *pool + per-call* overhead, so it @@ -255,7 +259,13 @@ def _checkpoint( return run -async def _run(args: argparse.Namespace) -> PerfRun: +# C901 rationale: the CLI orchestrator wiring together native/server-side/ +# optional e2e/concurrency-sweep paths with teardown-order-sensitive +# resource management (each runner registered for cleanup before it's +# started, per the inline comment) -- operator-only benchmark tooling, not +# part of the shipped package, but the teardown ordering is load-bearing +# for not leaking connections on a partial failure. +async def _run(args: argparse.Namespace) -> PerfRun: # noqa: C901 # MCPG_STATEMENT_TIMEOUT_MS sets Postgres's own statement_timeout GUC — a # second, independent ceiling from the asyncio guard --timeout controls # (mcpg.query.run_select's own `timeout` kwarg). Both must be raised diff --git a/benchmarks/tokens/tier_b/experiments/real_harness_comparison.py b/benchmarks/tokens/tier_b/experiments/real_harness_comparison.py index 48335848..263fed3b 100644 --- a/benchmarks/tokens/tier_b/experiments/real_harness_comparison.py +++ b/benchmarks/tokens/tier_b/experiments/real_harness_comparison.py @@ -131,7 +131,13 @@ def _build_mcp_config(database_url: str, worktree_dir: Path, nl2sql_api_key: str } -async def _invoke_claude( +# C901 rationale: one-off diagnostic benchmark script (never run in CI, per +# its module docstring) driving a real `claude -p` subprocess with +# stdin-piping worked around Windows-specific argv-quoting bugs (see the +# docstring below) and parsing streamed JSON events -- the branching is +# platform-workaround + streamed-event-parsing plumbing for a script that +# only a human runs interactively. +async def _invoke_claude( # noqa: C901 prompt: str, *, model: str, @@ -223,7 +229,7 @@ async def _invoke_claude( def _result_to_trial( - task_id: str, arm: str, trial: int, raw: dict[str, Any], tool_names: list[str], passed: bool + task_id: str, arm: str, trial: int, raw: dict[str, Any], tool_names: list[str], *, passed: bool ) -> TrialResult: """Map a `claude -p --output-format stream-json` result event onto the shared TrialResult schema. @@ -356,7 +362,7 @@ async def _run(args: argparse.Namespace) -> TierBReport: timeout_seconds=args.invocation_timeout_seconds, ) passed = task.grade(str(raw.get("result") or "")) - result = _result_to_trial(task.id, arm, trial, raw, tool_names, passed) + result = _result_to_trial(task.id, arm, trial, raw, tool_names, passed=passed) except Exception as exc: # record and continue, same as runner.py's pattern result = TrialResult( task_id=task.id, @@ -379,7 +385,10 @@ async def _run(args: argparse.Namespace) -> TierBReport: ) _checkpoint(args, trials, complete=False) finally: - mcp_config_path.unlink(missing_ok=True) + # ASYNC240 rationale: one-off benchmark script, single sequential coroutine + # (no concurrent tasks to starve); a single unlink() at teardown after + # trials that each take seconds to minutes is not a meaningful blocking cost. + mcp_config_path.unlink(missing_ok=True) # noqa: ASYNC240 return _checkpoint(args, trials, complete=True) diff --git a/docs/architecture.md b/docs/architecture.md index 3d3aac31..a49d7442 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -64,7 +64,7 @@ flowchart TD --- -## Module map (105 modules) +## Module map (106 modules) Every `mcpg.*` module and what it owns, alphabetical. The layered request path through these lives in the [Overview](#overview) diagram; @@ -100,6 +100,7 @@ this table is the exhaustive index. Regenerate with | `mcpg.dynamic_session_intent` | Dynamic session-intent — grow a session's visible tool surface at runtime. | | `mcpg.ecto` | Schema → Ecto (Elixir) schema exporter. | | `mcpg.ent` | Schema → Ent (Go) schema exporter. | +| `mcpg.errors` | The common ancestor for every MCPg-raised exception. | | `mcpg.extensions` | PostgreSQL extension management. | | `mcpg.graph` | Apache AGE Graph Introspection and Parsing. | | `mcpg.graph_diagram` | Apache AGE Graph Schema Visualisation. | diff --git a/docs/assets/icon-1024.png b/docs/assets/icon-1024.png new file mode 100644 index 00000000..bad72e80 Binary files /dev/null and b/docs/assets/icon-1024.png differ diff --git a/docs/assets/icon-128.png b/docs/assets/icon-128.png new file mode 100644 index 00000000..4c928145 Binary files /dev/null and b/docs/assets/icon-128.png differ diff --git a/docs/assets/icon-16.png b/docs/assets/icon-16.png new file mode 100644 index 00000000..a6604e63 Binary files /dev/null and b/docs/assets/icon-16.png differ diff --git a/docs/assets/icon-192.png b/docs/assets/icon-192.png new file mode 100644 index 00000000..265e0fe3 Binary files /dev/null and b/docs/assets/icon-192.png differ diff --git a/docs/assets/icon-256.png b/docs/assets/icon-256.png new file mode 100644 index 00000000..ca651cca Binary files /dev/null and b/docs/assets/icon-256.png differ diff --git a/docs/assets/icon-32.png b/docs/assets/icon-32.png new file mode 100644 index 00000000..a59ea062 Binary files /dev/null and b/docs/assets/icon-32.png differ diff --git a/docs/assets/icon-48.png b/docs/assets/icon-48.png new file mode 100644 index 00000000..072636ce Binary files /dev/null and b/docs/assets/icon-48.png differ diff --git a/docs/assets/icon-512.png b/docs/assets/icon-512.png index d8d97086..eb2e08bb 100644 Binary files a/docs/assets/icon-512.png and b/docs/assets/icon-512.png differ diff --git a/docs/assets/icon-64.png b/docs/assets/icon-64.png new file mode 100644 index 00000000..b62b5f89 Binary files /dev/null and b/docs/assets/icon-64.png differ diff --git a/docs/assets/logo-400.png b/docs/assets/logo-400.png index 78031552..fedcd079 100644 Binary files a/docs/assets/logo-400.png and b/docs/assets/logo-400.png differ diff --git a/docs/assets/logo-horizontal-1200.png b/docs/assets/logo-horizontal-1200.png new file mode 100644 index 00000000..522139ed Binary files /dev/null and b/docs/assets/logo-horizontal-1200.png differ diff --git a/docs/assets/logo-horizontal-800.png b/docs/assets/logo-horizontal-800.png new file mode 100644 index 00000000..8a40dd43 Binary files /dev/null and b/docs/assets/logo-horizontal-800.png differ diff --git a/docs/assets/logo-horizontal-full.png b/docs/assets/logo-horizontal-full.png new file mode 100644 index 00000000..abbb5a2b Binary files /dev/null and b/docs/assets/logo-horizontal-full.png differ diff --git a/docs/assets/logo-horizontal-master.png b/docs/assets/logo-horizontal-master.png new file mode 100644 index 00000000..f5f99637 Binary files /dev/null and b/docs/assets/logo-horizontal-master.png differ diff --git a/docs/installation.md b/docs/installation.md index 03486ff3..bf6cd093 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -150,13 +150,22 @@ specific client. For HTTP-based clients: +The HTTP transport (`streamable-http` / `sse`) refuses to start unless it +is authenticated: set `MCPG_HTTP_AUTH_TOKEN` (static bearer, shown below) +or `MCPG_AUTH_MODE=oidc` (see [security.md](security.md)). If you +genuinely need to run it without auth (e.g. behind a trusted network +proxy that terminates auth itself), set +`MCPG_HTTP_ALLOW_UNAUTHENTICATED=true` to explicitly opt out — MCPg logs +a loud warning every time it starts with that opt-out set. The default +`stdio` transport is unaffected and needs none of this. + **Linux / macOS (bash/zsh)** ```bash export MCPG_DATABASE_URL=postgresql://postgres:postgres@localhost:5432/mydb export MCPG_TRANSPORT=streamable-http export MCPG_HTTP_PORT=8000 -export MCPG_HTTP_AUTH_TOKEN=... # optional but strongly recommended +export MCPG_HTTP_AUTH_TOKEN=... # required unless MCPG_AUTH_MODE=oidc or MCPG_HTTP_ALLOW_UNAUTHENTICATED=true mcpg ``` @@ -166,7 +175,7 @@ mcpg $env:MCPG_DATABASE_URL = "postgresql://postgres:postgres@localhost:5432/mydb" $env:MCPG_TRANSPORT = "streamable-http" $env:MCPG_HTTP_PORT = "8000" -$env:MCPG_HTTP_AUTH_TOKEN = "..." # optional but strongly recommended +$env:MCPG_HTTP_AUTH_TOKEN = "..." # required unless MCPG_AUTH_MODE=oidc or MCPG_HTTP_ALLOW_UNAUTHENTICATED=true mcpg ``` @@ -268,6 +277,7 @@ the minimum set per common scenario. | **`LISTEN/NOTIFY`** event streams | `MCPG_ACCESS_MODE=unrestricted` + `MCPG_ALLOW_LISTEN=true` | | **HTTP transport** with static bearer | `MCPG_TRANSPORT=streamable-http` + `MCPG_HTTP_AUTH_TOKEN=…` | | **HTTP transport** with OIDC | `MCPG_TRANSPORT=streamable-http` + `MCPG_AUTH_MODE=oidc` + `MCPG_OIDC_ISSUER=…` + `MCPG_OIDC_AUDIENCE=…` | +| **HTTP transport**, deliberately unauthenticated | `MCPG_TRANSPORT=streamable-http` + `MCPG_HTTP_ALLOW_UNAUTHENTICATED=true` (not recommended; refuses to start otherwise) | | **HTTP transport** with IP allowlist | `MCPG_HTTP_IP_ALLOWLIST=10.0.0.0/8,192.168.1.0/24` (applied before auth; matched against the immediate peer — `X-Forwarded-For` is **not** honoured, so deployments behind a reverse proxy must enforce the allowlist at the proxy layer) | | **HTTP transport** with TLS | `MCPG_HTTP_TLS_CERTFILE=/etc/mcpg/cert.pem` + `MCPG_HTTP_TLS_KEYFILE=/etc/mcpg/key.pem` | | **HTTP transport** with mTLS | the TLS pair above + `MCPG_HTTP_TLS_CA_CERTS=/etc/mcpg/ca.pem` + `MCPG_HTTP_TLS_CLIENT_CERT_REQUIRED=true` | diff --git a/docs/project-incubation-baseline.md b/docs/project-incubation-baseline.md new file mode 100644 index 00000000..6ec0eccc --- /dev/null +++ b/docs/project-incubation-baseline.md @@ -0,0 +1,100 @@ +# Project Incubation Baseline + +**Project:** MCPg +**Baseline created:** 2026-08-25 +**Last audited:** 2026-08-25 (this invocation — baseline and audit run together, since no prior baseline existed) +**project-incubation skill version:** 0.2.0 (from `.claude-plugin/plugin.json` at incubation time) + +## Project shape + +- **Path:** software +- **Purpose:** A production-grade PostgreSQL Model Context Protocol (MCP) server — 254 MCP tools exposing + database introspection, query execution, migrations, observability, and administration to LLM agents, + with a first-party AST-validated SQL-safety kernel. +- **Team size at incubation:** solo / small team (single primary author + CODEOWNERS; user-confirmed) +- **Expected scale / lifespan:** production, long-lived (user-confirmed) — corroborated by the repo itself: + PyPI/GHCR/MCP-registry/Smithery releases, an active CI/CD and release pipeline, and multiple dated + security-review documents. +- **Compliance / regulatory constraints:** none stated in the repo (no PCI/HIPAA/GDPR references found in + README, SECURITY.md, or docs/) — not independently verified beyond a repo-content search. + +*Note: this baseline was written retroactively against an established repo, not at true inception — see +[Audit mode](#step-1-re-check-llmagent-component-status) results below for what was actually verified +against the existing codebase, since Phases 1–6 here are reconstructed from repo evidence and the user's +confirmation, not a live inception Q&A.* + +## Stack category (software path only) + +- **Primary category:** Agentic & MCP Platforms +- **Reasoning:** The project *is* an MCP server (the `mcp[cli]` SDK is a direct dependency, `server.py` + implements the MCP protocol surface, `packaging/mcpb/` ships a Claude Desktop extension bundle, + `publish-mcp-registry` in `publish.yml` registers it with the MCP Registry). User-confirmed. + +## Architecture template (software path only) + +- **Primary pattern:** Modular monolith — single deployable (`mcpg` package/process), ~100 source modules + with clear internal boundaries, no service-boundary splits. +- **Overlays / composed elements:** + - **Hexagonal-flavored core for the SQL-safety kernel** — `sql/allowlist.py` (policy-as-data) is + explicitly separated from `sql/safety.py` (mechanism: the `pglast` parse/validate path) and + `sql/driver.py` (pool/execution), per the project's own ADR-0007 and `CLAUDE.md`. This is a real + ports/adapters boundary, not an incidental module split — the project's own documentation frames it + that way. + - **Hexagonal-flavored core for secrets** — `secrets.py`'s `SecretsProvider` protocol with five + interchangeable backends (env/file/vault/aws/gcp) is a textbook swappable-adapter pattern behind one + port. + - **Event-driven overlay for LISTEN/NOTIFY** — `listen.py`'s pub-sub tool surface is the one genuinely + asynchronous, decoupled-communication subdomain in an otherwise request/response tool-call system. +- **Reasoning:** Single small team + single deployable target (a process an MCP host spawns/connects to) + rule out microservices outright — there's no multi-team release-cadence problem to solve. Domain + complexity in the SQL-safety kernel specifically (an AST allowlist that must be provably correct, with + its own fuzz-tested threat model) is exactly the "protect this from infrastructure churn" signal the + decision framework names for applying hexagonal to *that* core, not the whole system. This matches the + stack-category pairing table's own bias for Agentic & MCP Platforms ("hexagonal core... + event-driven + overlay") closely, arrived at independently from the repo's own structure and ADRs rather than from that + table. +- **ADR:** Not written as part of *this* skill invocation (the pattern was inferred from existing structure + and ADRs, not decided fresh) — the project's own `docs/adr/0007-first-party-sql-kernel.md` is the closest + existing document recording the hexagonal-core decision for the SQL kernel specifically. + +## Common architecture principles applied + +- **Principles doc version referenced:** `references/architecture-principles.md` as shipped in + agent-skills v0.2.0. +- **LLM/agent component:** yes + - **Basis:** asked directly (user-confirmed) — MCPg's NL→SQL feature (`nl2sql.py`) calls + Anthropic/OpenAI/Gemini directly, and the project's entire purpose is serving an LLM agent as an MCP + tool provider. + - **The LLM-conditional principles section applies.** Spot-checked against the codebase in this same + audit pass — see Step 3 below. +- **Notable deviations from the standard principle set:** none identified as a hard deviation; see Step 3 + findings for gaps found (not deviations by design). + +## Preferred libraries snapshot (software path only) + +- **Category reference used:** `references/preferred-libraries/agentic-mcp-platforms.md` +- **Snapshot date at incubation:** 2026-08-25 (this audit — the reference doc's own "last reviewed" dates + per entry are checked in Step 4 below) +- **Key library choices (from the repo, not prescribed by this baseline):** `mcp[cli]` (MCP SDK), + `psycopg[binary]` + `psycopg-pool` (async Postgres driver/pool), `pglast` (SQL AST parsing — the + safety-kernel's core dependency), `pyjwt[crypto]` (OIDC/JWT), `httpx` (LLM provider + OIDC discovery + calls), `hatchling` (build backend), `ruff`/`mypy --strict`/`bandit`/`pip-audit` (quality/security + tooling). + +## License + +- **Chosen license:** MIT +- **Reasoning:** Already the repo's license (`LICENSE`, `pyproject.toml` `license = "MIT"`) — permissive, + matches the MIT-licensed upstream MCPg forked from (per ADR-0001), no reasoning re-derived here since the + choice predates this baseline. + +## Drift log + +- 2026-08-25: Baseline created retroactively (no prior `docs/project-incubation-baseline.md` existed) and + first audit run in the same invocation. Findings from that audit (structure gaps: no root + `CODE_OF_CONDUCT.md`/`.editorconfig`; ~20+ exception classes with no shared base, a Maintainability/DRY + finding also raised independently by the same-day `python-code-review` run; NL→SQL provenance gap — + `TranslationResult` records which model/provider produced a translation but not the schema-context + evidence it saw; preferred-libraries snapshot confirmed current, 5 days old at audit time) were reported + to the user in conversation, not duplicated into this file. No fixes were auto-applied per this skill's + "never bulk-apply" rule — re-run the audit after any of them are addressed to update this log. diff --git a/docs/security-hardening.md b/docs/security-hardening.md index 47486316..90ed8d99 100644 --- a/docs/security-hardening.md +++ b/docs/security-hardening.md @@ -167,7 +167,9 @@ unconditionally (operators can disable per header via env). New **Env vars to add:** `MCPG_HTTP_MAX_BODY_BYTES`, `MCPG_HTTP_REQUEST_TIMEOUT_SECONDS`, `MCPG_HTTP_ALLOWED_ORIGINS`, -`MCPG_HTTP_HSTS_MAX_AGE` (default 31536000). +`MCPG_HTTP_HSTS_MAX_AGE` (default `63072000` — 2 years, OWASP's +current recommendation; bumped from the originally-shipped +`31536000`). **Effort:** medium (one new middleware module + 6-8 tests). diff --git a/docs/security.md b/docs/security.md index afcfe335..e0fca3e3 100644 --- a/docs/security.md +++ b/docs/security.md @@ -189,6 +189,11 @@ and every call is validated and audited. present a valid cert chaining to the configured CA. Setting the flag without `CA_CERTS` is rejected — there'd be nothing to verify against. +- **Fail closed by default.** `build_http_app` raises `ConfigError` at + startup — it refuses to serve — when neither `MCPG_HTTP_AUTH_TOKEN` + nor `MCPG_AUTH_MODE=oidc` is configured. `MCPG_HTTP_ALLOW_UNAUTHENTICATED=true` + is the only way to opt back into serving unauthenticated, and doing + so logs a warning on every startup. `stdio` is unaffected. - **Static bearer.** `MCPG_HTTP_AUTH_TOKEN` enforces `Authorization: Bearer ` with `hmac.compare_digest` constant-time comparison. `/metrics`, `/healthz`, `/readyz` are diff --git a/docs/superpowers/plans/2026-08-25-audit-remediation.md b/docs/superpowers/plans/2026-08-25-audit-remediation.md new file mode 100644 index 00000000..66788696 --- /dev/null +++ b/docs/superpowers/plans/2026-08-25-audit-remediation.md @@ -0,0 +1,2289 @@ +# Audit Remediation (python-code-review + project-incubation) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Close out every finding from the 2026-08-25 `python-code-review` and `project-incubation` skill +runs against MCPg, commit the maintainer's remodeled icon set, and leave `ruff check .` (all categories +enabled), `ruff format --check .`, `mypy --strict src/mcpg`, and the unit test suite all green. + +**Architecture:** No architectural change — this is remediation across existing module boundaries +(`sql/`, `config.py`, `http_runtime.py`, `nl2sql.py`, `oidc.py`, `query.py`, `cache.py`, `obs_logging.py`, +per-module exception classes) plus CI/tooling config (`pyproject.toml`, `.github/workflows/*`) and repo +governance files. Two genuinely behavior-changing defaults are included per explicit maintainer sign-off: +HTTP-transport auth fails closed by default, and rate limiting defaults to enabled. + +**Tech Stack:** Python 3.12+, `pytest`/`pytest-asyncio`/`pytest-cov`, `ruff`, `mypy --strict`, `bandit`, +`pip-audit`, `uv`. New dev/runtime deps this plan adds: `pytest-mock`, `pytest-randomly`, `pytest-socket`, +`time-machine`, `pytest-rerunfailures`, `circuitbreaker`, `tenacity`, `pip-licenses` (dev-only). + +**Spec:** The two audit reports produced earlier in this session — `python-code-review`'s scorecard/findings +(Critical/Important/Minor/Not-Implemented, all 11 domains) and `project-incubation`'s Step 2–4 audit +checklist — plus the maintainer's scoping decisions recorded in this same conversation: +HTTP auth fails closed (breaking, `MCPG_HTTP_ALLOW_UNAUTHENTICATED=true` is the opt-out), rate limiting +defaults on, all 11 opt-in ruff categories get enabled **and** their existing violations fixed (not +deferred), the review report itself stays out of the repo. + +## Global Constraints + +- Python floor: `>=3.12` (`pyproject.toml`) — no syntax requiring a newer floor. +- `mypy --strict` must stay clean on every task — run it after every task, not just at the end. +- **Never hand-edit `src/mcpg/_vendor/`** (CLAUDE.md, project-wide rule). +- Coverage gate is `fail_under = 90` (`[tool.coverage.report]`) — every new code path (the `/readyz` route, + the bounded-fetch change, the auth-fail-closed path, the rate-limit-default path, the redaction filter, + the `MCPgError` base) needs a test, not just an implementation. +- CHANGELOG entries go under `[Unreleased]`, Keep-a-Changelog categories (`Added`/`Changed`/`Deprecated`/ + `Removed`/`Fixed`/`Security`), ISO dates only when a version is actually cut (not for `[Unreleased]` + entries). +- Commit per logical slice (CLAUDE.md) — one task = one commit, in the order below, not squashed together. +- PR checklist (`.github/PULL_REQUEST_TEMPLATE.md`) requires a roadmap-row citation or `N/A — `; + this PR is infra/quality, not a roadmap feature — use `N/A — internal audit remediation + (python-code-review + project-incubation skill runs, 2026-08-25)`. +- Docstrings added anywhere in this plan must describe what the function actually does, verified by + reading it — CLAUDE.md's "verify before you write" rule applies to docstrings as much as to any other + documentation claim in this repo. No lazily-templated docstrings. +- `git add` explicit paths only, never `-A` — `.entire/` (self-ignoring tool metadata) and `reports/` (this + session's own working artifact, staying untracked per maintainer decision) must not be swept in. + +--- + +### Task 1: Commit the remodeled icon set + +**Files:** +- Modify (already on disk, currently `M`): `docs/assets/icon-512.png`, `docs/assets/logo-400.png` +- Add (already on disk, currently untracked): `docs/assets/icon-1024.png`, `icon-128.png`, `icon-16.png`, + `icon-192.png`, `icon-256.png`, `icon-32.png`, `icon-48.png`, `icon-64.png`, + `logo-horizontal-1200.png`, `logo-horizontal-800.png`, `logo-horizontal-full.png`, + `logo-horizontal-master.png` + +**Interfaces:** None — static assets, no code consumes them yet beyond whatever already references +`docs/assets/*.png` (check `packaging/mcpb/manifest.json` and `README.md` for existing references before +assuming these are net-new files with no consumers). + +- [ ] **Step 1: Confirm no consumer expects a different filename** + +```bash +grep -rn "docs/assets" --include="*.md" --include="*.json" --include="*.yaml" --include="*.yml" . +``` + +Expected: existing references (README badges, `packaging/mcpb/manifest.json` icon path, `glama.json`) name +files that already exist in this set (`icon-512.png`, `logo-400.png`, etc.) — no dangling reference to a +filename this set doesn't provide. If a consumer expects a filename not in the new set, stop and flag it +to the maintainer before committing (a broken icon reference is worse than an uncommitted icon). + +- [ ] **Step 2: Stage and commit the asset set only** + +```bash +git add docs/assets/icon-1024.png docs/assets/icon-128.png docs/assets/icon-16.png \ + docs/assets/icon-192.png docs/assets/icon-256.png docs/assets/icon-32.png \ + docs/assets/icon-48.png docs/assets/icon-512.png docs/assets/icon-64.png \ + docs/assets/logo-400.png docs/assets/logo-horizontal-1200.png \ + docs/assets/logo-horizontal-800.png docs/assets/logo-horizontal-full.png \ + docs/assets/logo-horizontal-master.png +git commit -m "chore(assets): update icon and logo set + +Co-Authored-By: Claude Sonnet 5 " +``` + +--- + +### Task 2: Add `CODE_OF_CONDUCT.md` + +**Files:** +- Create: `CODE_OF_CONDUCT.md` + +**Interfaces:** None. + +- [ ] **Step 1: Write the file** — copy Contributor Covenant v2.1 verbatim (per + `skills/project-incubation/references/project-structure.md`'s own guidance: "adopt ... verbatim rather + than drafting one"), with the enforcement contact set to the same channel `SECURITY.md` already uses. + Read `SECURITY.md`'s reporting section first so the contact matches rather than introducing a second, + inconsistent contact path. + +- [ ] **Step 2: Commit** + +```bash +git add CODE_OF_CONDUCT.md +git commit -m "docs: add CODE_OF_CONDUCT.md (Contributor Covenant v2.1) + +Co-Authored-By: Claude Sonnet 5 " +``` + +--- + +### Task 3: Add `.editorconfig` + +**Files:** +- Create: `.editorconfig` + +**Interfaces:** None. + +- [ ] **Step 1: Write the file** + +```ini +root = true + +[*] +indent_style = space +indent_size = 4 +charset = utf-8 +end_of_line = lf +trim_trailing_whitespace = true +insert_final_newline = true + +[*.py] +indent_size = 4 + +[*.md] +trim_trailing_whitespace = false + +[*.{yml,yaml,json,toml}] +indent_size = 2 + +[*.{bat,cmd}] +end_of_line = crlf +``` + +Note the deviation from `project-structure.md`'s generic example (`indent_size = 2` at the top level): MCPg +is a Python-first repo under `ruff format`'s default 4-space indent — matching the dominant language's +actual convention takes priority over the reference doc's generic default, with narrower overrides for +YAML/JSON/TOML and Windows batch files. + +- [ ] **Step 2: Commit** + +```bash +git add .editorconfig +git commit -m "chore: add .editorconfig + +Co-Authored-By: Claude Sonnet 5 " +``` + +--- + +### Task 4: `py.typed` marker + fix stale `license-files` entry + pin `hatchling` floor + +**Files:** +- Create: `src/mcpg/py.typed` (empty file) +- Modify: `pyproject.toml:12` (`license-files`), `pyproject.toml:106` (`[build-system] requires`) +- Test: `tests/unit/test_packaging.py` (create if it doesn't already exist — check first) + +**Interfaces:** None (packaging metadata only). + +- [ ] **Step 1: Check for an existing packaging test file** + +```bash +find tests -iname "*packaging*" -o -iname "*wheel*" +``` + +If one exists, add to it; if not, create `tests/unit/test_packaging.py`. + +- [ ] **Step 2: Write the failing test** — confirms the marker file exists and would ship in the wheel + (the packaging-correctness angle Standards Compliance flagged, not just "the file is somewhere in the + repo"): + +```python +"""Packaging-correctness checks: py.typed presence and wheel include rules.""" + +from __future__ import annotations + +from pathlib import Path + + +def test_py_typed_marker_present() -> None: + """PEP 561: the py.typed marker must exist in the package directory.""" + marker = Path(__file__).resolve().parents[2] / "src" / "mcpg" / "py.typed" + assert marker.is_file() + assert marker.read_text(encoding="utf-8") == "" +``` + +- [ ] **Step 3: Run test to verify it fails** + +Run: `uv run pytest tests/unit/test_packaging.py -v` +Expected: FAIL — `src/mcpg/py.typed` doesn't exist yet. + +- [ ] **Step 4: Create the marker file** + +```bash +: > src/mcpg/py.typed +``` + +(An empty file — PEP 561 requires only its presence, no content.) + +- [ ] **Step 5: Confirm hatchling ships it** — `[tool.hatch.build.targets.wheel] packages = ["src/mcpg"]` + already includes every file under `src/mcpg` by default (hatchling's default wheel include is the whole + package directory when `packages` names it), so no separate include-glob edit is needed. Verify directly + rather than assuming: + +```bash +uv run python -m build --wheel --outdir /tmp/mcpg-wheel-check +python -c "import zipfile; z = zipfile.ZipFile(next(iter(__import__('pathlib').Path('/tmp/mcpg-wheel-check').glob('*.whl')))); print('mcpg/py.typed' in z.namelist())" +``` + +Expected: `True`. If `False`, add an explicit include rule to `[tool.hatch.build.targets.wheel]` before +proceeding — don't ship this task assuming it worked. + +- [ ] **Step 6: Run test to verify it passes** + +Run: `uv run pytest tests/unit/test_packaging.py -v` +Expected: PASS + +- [ ] **Step 7: Fix the stale `license-files` entry** — `src/mcpg/_vendor/LICENSE` doesn't exist (the + vendored SQL kernel was de-vendored per ADR-0007; `_vendor/` now holds only first-party `sql/` + submodules). Edit `pyproject.toml:12`: + +```diff +- license-files = ["LICENSE", "src/mcpg/_vendor/LICENSE"] ++ license-files = ["LICENSE"] +``` + +- [ ] **Step 8: Pin the `hatchling` build-system floor** — `pyproject.toml:106`: + +```diff +- requires = ["hatchling"] ++ requires = ["hatchling>=1.26"] +``` + +(1.26 is PyPA's own documented minimum-version example for hatchling in `writing-pyproject-toml`, and is +older than anything this repo has run in CI — safe floor, not a forced upgrade.) + +- [ ] **Step 9: Run the full check + tests** + +```bash +uv run ruff check . && uv run ruff format --check . && uv run mypy src/mcpg && uv run pytest -q +``` + +Expected: all pass. + +- [ ] **Step 10: Update CHANGELOG.md** — add under `[Unreleased]` → `Fixed`: + +```markdown +- **`py.typed` marker was missing despite the `Typing :: Typed` classifier.** Added `src/mcpg/py.typed` + and verified it ships in the built wheel. +- **`license-files` in `pyproject.toml` pointed at `src/mcpg/_vendor/LICENSE`, which hasn't existed since + the SQL kernel was de-vendored (ADR-0007).** Removed the stale entry. +``` + +and under `Changed`: + +```markdown +- Pinned `hatchling>=1.26` as the build-system floor (previously unpinned). +``` + +- [ ] **Step 11: Commit** + +```bash +git add src/mcpg/py.typed pyproject.toml tests/unit/test_packaging.py CHANGELOG.md +git commit -m "fix(packaging): add py.typed marker, drop stale license-files entry, pin hatchling floor + +Co-Authored-By: Claude Sonnet 5 " +``` + +--- + +### Task 5: Shared `MCPgError` base for the ~20+ module-level exception classes + +**Files:** +- Create: `src/mcpg/errors.py` +- Modify: every module currently declaring `class Error(Exception)` — confirmed list from + `grep -rn "^class.*Error(Exception)" src/mcpg/*.py`: `aio.py`, `audit_nl2sql.py`, `audit_trail.py`, + `composite.py`, `config.py`, `config_advisor.py`, `cron.py`, `cursors.py`, `data_movement.py` (two: + `ExportError`, `ImportDataError`), `database.py`, `demo.py`, `diesel.py`, `drizzle.py`, `ecto.py`, + `ent.py`, `extensions.py`, `graph.py`, `graph_projection.py`, `headline_curator.py`, and any further + matches beyond that grep's default output limit — **re-run the grep at execution time and treat its + live output as the authoritative list**, not the names enumerated here, since this plan was written + against a point-in-time snapshot. +- Test: `tests/unit/test_errors.py` + +**Interfaces:** +- Produces: `mcpg.errors.MCPgError`, a plain `Exception` subclass with no added behavior — every existing + domain exception's public name, message format, and `raise`/`except` call sites are unchanged; only the + base class in each `class XError(Exception):` declaration changes to `class XError(MCPgError):`. + +- [ ] **Step 1: Write the failing test** + +```python +"""MCPgError is the common ancestor every domain-specific error subclasses.""" + +from __future__ import annotations + +import importlib +import inspect +import pkgutil + +import mcpg +from mcpg.errors import MCPgError + + +def _iter_mcpg_modules() -> list[str]: + return [ + name + for _, name, is_pkg in pkgutil.walk_packages(mcpg.__path__, prefix="mcpg.") + if not is_pkg and "_vendor" not in name + ] + + +def test_every_domain_error_class_subclasses_mcpg_error() -> None: + offenders: list[str] = [] + for module_name in _iter_mcpg_modules(): + module = importlib.import_module(module_name) + for obj_name, obj in vars(module).items(): + if ( + inspect.isclass(obj) + and obj_name.endswith("Error") + and obj.__module__ == module_name + and issubclass(obj, Exception) + and obj is not MCPgError + and not issubclass(obj, MCPgError) + ): + offenders.append(f"{module_name}.{obj_name}") + assert not offenders, f"Exception classes not subclassing MCPgError: {offenders}" + + +def test_mcpg_error_is_a_plain_exception_subclass() -> None: + assert issubclass(MCPgError, Exception) + assert MCPgError.__doc__ # documented, not a bare pass-through +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/unit/test_errors.py -v` +Expected: FAIL — `mcpg.errors` doesn't exist yet, and every existing `XError` still subclasses `Exception` +directly. + +- [ ] **Step 3: Create `src/mcpg/errors.py`** + +```python +"""The common ancestor for every MCPg-raised exception. + +Every domain-specific error class in this package (``ConfigError``, +``DatabaseError``, ``CursorError``, and the rest) subclasses this instead of +``Exception`` directly, so calling code that wants to catch "any error MCPg's +own logic raised" — as distinct from an unexpected bug surfacing from a +dependency — has one type to catch instead of an enumerated list kept in sync +by hand. + +This is a pure marker base: it adds no behavior, no new attributes, and no +change to any existing exception's message format or call sites. Catching a +specific subclass (``except ConfigError:``) behaves exactly as it did before; +``except MCPgError:`` is the new capability this adds. +""" + +from __future__ import annotations + + +class MCPgError(Exception): + """Base class for every exception MCPg's own logic raises. + + Not raised directly — always through one of its domain-specific + subclasses (``ConfigError``, ``DatabaseError``, etc.). Catch this + directly only when the intent is genuinely "any MCPg-internal error," + not a specific failure mode. + """ +``` + +- [ ] **Step 4: Update every domain exception class** — for each file in the confirmed list, change the + one-line class declaration and add the import. Worked example (`config.py`): + +```diff ++ from mcpg.errors import MCPgError ++ +- class ConfigError(Exception): ++ class ConfigError(MCPgError): + """Raised when the environment configuration is missing or invalid.""" +``` + +Apply the same two-line change (import + base-class swap) to every other file in the confirmed list. +Nothing else in any of these files changes — no `raise` call site, no `except` clause, no message string. + +- [ ] **Step 5: Run test to verify it passes** + +Run: `uv run pytest tests/unit/test_errors.py -v` +Expected: PASS + +- [ ] **Step 6: Run the full check + tests** + +```bash +uv run ruff check . && uv run ruff format --check . && uv run mypy src/mcpg && uv run pytest -q +``` + +- [ ] **Step 7: Update CHANGELOG.md** under `Added`: + +```markdown +- `mcpg.errors.MCPgError`, a common base class every domain-specific exception (`ConfigError`, + `DatabaseError`, `CursorError`, and ~20 others) now subclasses — lets calling code catch "any + MCPg-internal error" with one type instead of an enumerated list. No existing exception's name, message, + or call sites changed. +``` + +- [ ] **Step 8: Commit** + +```bash +git add src/mcpg/errors.py tests/unit/test_errors.py src/mcpg/*.py CHANGELOG.md +git commit -m "refactor: introduce MCPgError as the common base for domain exceptions + +Co-Authored-By: Claude Sonnet 5 " +``` + +--- + +### Task 6: Log the 7 silent `except Exception: pass` sites + +**Files:** +- Modify: `src/mcpg/advisors.py:797-798`, `src/mcpg/audit.py:248-249`, `src/mcpg/audit_trail.py:251-252`, + `src/mcpg/audit_trail.py:839-840`, `src/mcpg/audit_trail.py:852-853`, `src/mcpg/listen.py:251-252`, + `src/mcpg/migrations.py:349-350` — **re-check line numbers at execution time**; earlier tasks in this + plan may have shifted them by a few lines within each file. +- Test: extend each file's corresponding existing test (`tests/unit/test_advisors.py`, + `test_audit.py`, `test_audit_trail.py`, `test_listen.py`, `test_migrations.py`) — confirmed to exist via + `tests/unit/_fakes.py`'s companions; verify each file exists before writing to it. + +**Interfaces:** None — this changes zero control flow. Every site keeps its exact `except Exception:` +scope and continues doing exactly what it did before; only a log call is added. `migrations.py`'s site +still re-raises the outer exception after its cleanup step's own exception is logged, unchanged. + +- [ ] **Step 1: Fix `advisors.py`** — best-effort advisory-text generation: + +```diff + try: + if plan is not None and plan.sequential_scans: + rationale_parts.append( + "- Consider adding indexes on columns used in WHERE or JOIN clauses for tables with Seq Scan." + ) +- except Exception: +- pass ++ except Exception: ++ logger.debug("Skipping sequential-scan advisory line; plan inspection failed", exc_info=True) +``` + +Confirm `logger = logging.getLogger(__name__)` (or equivalent) already exists at module scope in +`advisors.py` before assuming `logger` is in scope — check the top of the file first. + +- [ ] **Step 2: Fix `audit.py`** — best-effort Postgres version-string detection with a documented fallback: + +```diff + except Exception: +- pass ++ logger.debug("Version/dbname query failed; falling back to 'PostgreSQL Unknown'", exc_info=True) + return "PostgreSQL Unknown", "unknown" +``` + +- [ ] **Step 3: Fix `audit_trail.py` (three sites, lines ~251, ~839, ~852)** — read each surrounding + block first; each is a different best-effort path (read the 15 lines above each `except Exception:` to + write an accurate one-line description, don't reuse the same message for all three): + +```python +except Exception: + logger.debug("", exc_info=True) +``` + +- [ ] **Step 4: Fix `listen.py`** — bounded socket-close during shutdown (already has an explanatory + comment above it — keep the comment, add the log line): + +```diff + try: + await asyncio.wait_for(conn.close(), timeout=2.0) +- except Exception: +- pass ++ except Exception: ++ logger.debug("Best-effort connection close during shutdown failed", exc_info=True) +``` + +- [ ] **Step 5: Fix `migrations.py`** — cleanup-of-a-half-built-shadow-schema, inside a block that already + re-raises the real error. **Only add the log line; do not touch the `raise` below it:** + +```diff + except Exception: + # The shadow is half-built; drop it so we don't accumulate + # orphaned schemas across failed prepares. + try: + await driver.execute_query(f'DROP SCHEMA IF EXISTS "{shadow_schema}" CASCADE') +- except Exception: +- pass ++ except Exception: ++ logger.debug("Failed to drop half-built shadow schema %r during cleanup", shadow_schema, exc_info=True) + raise +``` + +- [ ] **Step 6: For each modified file, add or extend one test asserting the debug log fires** — worked + example for `migrations.py` (adapt the fixture/mock setup to match each file's existing test patterns — + read the existing test file first rather than inventing a new fixture style): + +```python +def test_shadow_schema_cleanup_failure_is_logged(caplog: pytest.LogCaptureFixture) -> None: + """A failed DROP SCHEMA during shadow-schema cleanup logs at debug, not silently.""" + caplog.set_level(logging.DEBUG, logger="mcpg.migrations") + # ... existing fixture setup that makes the DROP SCHEMA cleanup itself fail ... + with pytest.raises(SomeExpectedOuterException): + await function_under_test(...) + assert any("shadow schema" in record.message for record in caplog.records) +``` + +Write the equivalent for the other 6 sites, matching each file's own existing test conventions. + +- [ ] **Step 7: Run the full check + tests** + +```bash +uv run ruff check . && uv run ruff format --check . && uv run mypy src/mcpg && uv run pytest -q +``` + +- [ ] **Step 8: Update CHANGELOG.md** under `Fixed`: + +```markdown +- **Seven `except Exception: pass` sites now log at `debug` level with `exc_info=True`** instead of + swallowing silently — `advisors.py`, `audit.py`, `audit_trail.py` (×3), `listen.py`, `migrations.py`. + No control flow changed; these were already best-effort/cleanup paths and remain so, now with + observability into how often they actually fire. +``` + +- [ ] **Step 9: Commit** + +```bash +git add src/mcpg/advisors.py src/mcpg/audit.py src/mcpg/audit_trail.py src/mcpg/listen.py \ + src/mcpg/migrations.py tests/unit/test_advisors.py tests/unit/test_audit.py \ + tests/unit/test_audit_trail.py tests/unit/test_listen.py tests/unit/test_migrations.py \ + CHANGELOG.md +git commit -m "fix: log the 7 silent except-Exception-pass sites at debug level + +Co-Authored-By: Claude Sonnet 5 " +``` + +--- + +### Task 7: Mount `/readyz` + +**Files:** +- Modify: `src/mcpg/http_runtime.py` (near line 580-581, where `/healthz` and `/metrics` are mounted; the + route handler pattern to follow is `_health_response_factory()` just above it) +- Test: `tests/unit/test_http_runtime.py` + +**Interfaces:** +- Produces: `_readiness_response_factory() -> Callable[[Request], Awaitable[Response]]`, same shape as the + existing `_health_response_factory()`. Mounted at `Route("/readyz", ..., methods=["GET"])`. +- Consumes: the app's DB pool handle (however `_health_response_factory` already accesses it — read that + function's body first and reuse the exact same access pattern, don't invent a new one) and, when + `settings.auth_mode == "oidc"`, whether the `OIDCVerifier`'s JWKS fetch has ever succeeded (read + `oidc.py`'s `OIDCVerifier`/`_ensure_jwks_client` to find the right signal — likely a cached-client-present + check, not a fresh fetch on every readiness poll). + +- [ ] **Step 1: Read `_health_response_factory` and `OIDCVerifier` first** + +```bash +grep -n "_health_response_factory" -A 15 src/mcpg/http_runtime.py +grep -n "class OIDCVerifier" -A 30 src/mcpg/oidc.py +``` + +Confirm the exact attribute/method names before writing Step 2 — do not guess them. + +- [ ] **Step 2: Write the failing test** + +```python +async def test_readyz_returns_200_when_pool_has_a_connection(http_app_with_live_pool) -> None: + """/readyz reports ready once the DB pool has at least one usable connection.""" + client = TestClient(http_app_with_live_pool) + response = client.get("/readyz") + assert response.status_code == 200 + + +async def test_readyz_returns_503_when_pool_unavailable(http_app_with_broken_pool) -> None: + """/readyz reports not-ready when the DB pool can't produce a connection.""" + client = TestClient(http_app_with_broken_pool) + response = client.get("/readyz") + assert response.status_code == 503 + + +def test_readyz_is_auth_exempt() -> None: + """/readyz stays reachable without a bearer token, same as /healthz.""" + assert "/readyz" in _AUTH_EXEMPT_PATHS +``` + +Adapt `http_app_with_live_pool` / `http_app_with_broken_pool` to whatever fixture pattern +`test_http_runtime.py` already uses for `/healthz` — read its existing health-check tests first (this file +already tests `/healthz`, so the pool-mocking fixture almost certainly already exists; reuse it rather than +building a new one). + +- [ ] **Step 3: Run test to verify it fails** + +Run: `uv run pytest tests/unit/test_http_runtime.py -k readyz -v` +Expected: FAIL — 404, no such route. + +- [ ] **Step 4: Implement `_readiness_response_factory` and mount the route** + +```python +def _readiness_response_factory() -> Callable[[Request], Awaitable[Response]]: + """Build the /readyz handler: 200 once the DB pool can serve a connection, 503 otherwise. + + Distinct from /healthz (liveness — "is the process alive") — this reports whether the + process can currently do useful work, so an orchestrator can pull a degraded instance out + of rotation without restarting it. + """ + + async def readyz(request: Request) -> Response: + try: + async with request.app.state.pool.connection(timeout=2.0): + pass + except Exception: + return JSONResponse({"status": "not ready"}, status_code=503) + return JSONResponse({"status": "ready"}) + + return readyz +``` + +(Adjust the pool-access expression — `request.app.state.pool`, or whatever `_health_response_factory` +actually uses — to match Step 1's findings exactly.) + +```diff + app.router.routes.append(Route("/healthz", _health_response_factory(), methods=["GET"])) ++ app.router.routes.append(Route("/readyz", _readiness_response_factory(), methods=["GET"])) +``` + +`/readyz` is already in `_AUTH_EXEMPT_PATHS` (line 54) — no change needed there. + +- [ ] **Step 5: Run test to verify it passes** + +Run: `uv run pytest tests/unit/test_http_runtime.py -k readyz -v` +Expected: PASS + +- [ ] **Step 6: Run the full check + tests** + +```bash +uv run ruff check . && uv run ruff format --check . && uv run mypy src/mcpg && uv run pytest -q +``` + +- [ ] **Step 7: Update CHANGELOG.md** under `Added`: + +```markdown +- `/readyz` readiness endpoint on the HTTP transport — reports 503 when the DB pool can't currently serve + a connection, distinct from `/healthz`'s liveness-only check. Previously reserved in the auth-exemption + set but never mounted. +``` + +- [ ] **Step 8: Commit** + +```bash +git add src/mcpg/http_runtime.py tests/unit/test_http_runtime.py CHANGELOG.md +git commit -m "feat(http): mount /readyz readiness probe + +Co-Authored-By: Claude Sonnet 5 " +``` + +--- + +### Task 8: Bound `run_select`'s fetch instead of materialize-then-truncate + +**Files:** +- Modify: `src/mcpg/query.py` (both `run_select`-shaped functions — the one at line ~103 and its sibling + around line ~200; re-locate both by name at execution time, don't assume line numbers held across earlier + tasks' edits) +- Test: `tests/unit/test_query.py` (check it exists; extend if so) + +**Interfaces:** +- Consumes: `SqlDriver.RowResult` (unchanged), `SafeSqlDriver.execute_query` (unchanged signature). +- Produces: `run_select(driver, sql, *, timeout=..., max_rows=...) -> QueryResult` — **same public + signature and same `QueryResult` shape** (`columns`, `rows`, `row_count`, `truncated`) as today; only the + internal fetch strategy changes from "fetch all, then slice" to "fetch at most `max_rows + 1`, stop + there." + +- [ ] **Step 1: Read `SqlDriver.execute_query` and `SafeSqlDriver.execute_query` fully** — confirm whether + the underlying psycopg cursor is exposed anywhere for a `fetchmany`-style bound, or whether + `execute_query` always does a full `fetchall()` internally with no bound parameter today: + +```bash +grep -n "fetchall\|fetchmany\|async def execute_query" src/mcpg/sql/driver.py src/mcpg/sql/safety.py +``` + +- [ ] **Step 2: Write the failing test** — asserts the fetch itself is bounded, not just the returned + slice (a test that only checks `len(result.rows) <= max_rows` would already pass today and wouldn't + catch this bug — the test has to observe how many rows were pulled from the driver): + +```python +async def test_run_select_does_not_fetch_beyond_max_rows_plus_one(monkeypatch) -> None: + """A query matching far more rows than max_rows only pulls max_rows+1 from the driver.""" + fetched_counts: list[int] = [] + + class _CountingFakeDriver(SqlDriver): + async def execute_query(self, query, params=None, force_readonly=True): + # Simulate a driver-level bound: a real fix passes a row cap through to the + # underlying fetch rather than materializing everything first. + requested = getattr(self, "_last_requested_max_rows", None) + fetched_counts.append(requested) + n = requested if requested is not None else 1_000_000 + return [SqlDriver.RowResult(cells={"n": i}) for i in range(min(n, 1_000_000))] + + result = await run_select(_CountingFakeDriver(), "SELECT * FROM huge_table", max_rows=5) + assert result.truncated is True + assert result.row_count == 5 + assert fetched_counts[-1] is not None and fetched_counts[-1] <= 6 # max_rows + 1, not 1,000,000 +``` + +(This test's exact shape depends on Step 1's findings — if `execute_query` has no row-cap parameter to +plumb through yet, the test should assert on whatever bounding mechanism Step 3 introduces; adjust the fake +driver accordingly, but keep the core assertion: the driver-observed fetch count must be bounded by +`max_rows + 1`, not the full result-set size.) + +- [ ] **Step 3: Run test to verify it fails** + +Run: `uv run pytest tests/unit/test_query.py -k does_not_fetch_beyond -v` +Expected: FAIL — today's `run_select` calls `execute_query` with no row cap at all. + +- [ ] **Step 4: Implement the bound** — the exact mechanism depends on Step 1's findings. If + `psycopg`'s cursor is reachable, prefer a server-side cursor with `fetchmany(max_rows + 1)`. If + `execute_query` only exposes a full-materialization API today, the minimally-invasive fix is adding an + optional `row_limit: int | None` parameter to `SqlDriver.execute_query`/`SafeSqlDriver.execute_query` + that, when set, stops iterating the cursor after `row_limit` rows instead of calling `fetchall()`: + +```python +# In SqlDriver.execute_query (sql/driver.py) — sketch, adapt to the file's real cursor-handling code: +async def execute_query( + self, + query: LiteralString, + params: list[Any] | None = None, + force_readonly: bool = True, + row_limit: int | None = None, +) -> list[RowResult] | None: + ... + if row_limit is not None: + rows = await cursor.fetchmany(row_limit) + else: + rows = await cursor.fetchall() + ... +``` + +Then in `query.py`'s `run_select`: + +```diff +- rows = await safe_driver.execute_query(sql) ++ rows = await safe_driver.execute_query(sql, row_limit=max_rows + 1) + all_rows = [dict(row.cells) for row in rows or []] + truncated = len(all_rows) > max_rows + result_rows = all_rows[:max_rows] +``` + +`SafeSqlDriver.execute_query` needs the same `row_limit` parameter threaded through to its wrapped +`self.sql_driver.execute_query` call. Apply the identical change to `query.py`'s second `run_select`-shaped +function (the one around line ~200 per the audit). + +- [ ] **Step 5: Run test to verify it passes** + +Run: `uv run pytest tests/unit/test_query.py -k does_not_fetch_beyond -v` +Expected: PASS + +- [ ] **Step 6: Run the full check + tests, including the SQL-kernel adversarial suite** + +```bash +uv run ruff check . && uv run ruff format --check . && uv run mypy src/mcpg +uv run pytest -q tests/unit/test_query.py tests/unit/test_sql_kernel_driver.py tests/unit/test_sql_kernel_safety.py +``` + +- [ ] **Step 7: Note the integration-test limit honestly** — this change cannot be verified against a real + Postgres instance in this environment (no `MCPG_TEST_DATABASE_URL` available locally). Flag this + explicitly in the PR description rather than claiming full verification; CI's integration matrix + (`tests/integration/`) will exercise it against real Postgres on push. + +- [ ] **Step 8: Update CHANGELOG.md** under `Fixed`: + +```markdown +- **`run_select` fully materialized a query's entire result set before truncating to `max_rows`,** rather + than bounding the fetch itself — a query without its own `LIMIT` against a large table could pull + millions of rows into memory before the truncation ever ran. The fetch is now bounded to `max_rows + 1` + at the driver level. +``` + +- [ ] **Step 9: Commit** + +```bash +git add src/mcpg/query.py src/mcpg/sql/driver.py src/mcpg/sql/safety.py tests/unit/test_query.py CHANGELOG.md +git commit -m "fix(query): bound result-set fetch instead of materialize-then-truncate + +Co-Authored-By: Claude Sonnet 5 " +``` + +--- + +### Task 9: Reuse a shared `httpx.AsyncClient` in `nl2sql.py` and `oidc.py` + +**Files:** +- Modify: `src/mcpg/nl2sql.py` (lines ~389, ~440, ~489 — the three per-call `httpx.AsyncClient(...)` + constructions), `src/mcpg/oidc.py:159` +- Test: `tests/unit/test_nl2sql.py`, `tests/unit/test_oidc.py` + +**Interfaces:** +- Each of the three NL→SQL provider functions currently opens its own `async with httpx.AsyncClient(...)`. + Replace with a module- or class-level client constructed once and passed in / held on the calling + object, matching whichever of those two shapes the existing provider-class structure already uses (read + `AnthropicProvider`/`OpenAIProvider`/`GeminiProvider`'s `__init__` first — if they're already + instantiated once per server lifetime by `build_provider`, add the client there; if they're constructed + fresh per call today, that's the actual root cause and the fix is making provider construction + lifetime-scoped, not just the client). +- `OIDCVerifier` similarly should hold one `httpx.AsyncClient` for its lifetime rather than opening one per + discovery-document fetch. + +- [ ] **Step 1: Read the provider class constructors and `build_provider`** + +```bash +grep -n "class AnthropicProvider\|class OpenAIProvider\|class GeminiProvider\|def build_provider\|def __init__" src/mcpg/nl2sql.py | head -20 +grep -n "class OIDCVerifier\|def __init__\|_ensure_jwks_client" src/mcpg/oidc.py | head -10 +``` + +- [ ] **Step 2: Write the failing test** — asserts a single client instance is reused across two calls, + not recreated: + +```python +async def test_provider_reuses_one_httpx_client_across_calls(monkeypatch) -> None: + """Two translate calls through the same provider instance share one AsyncClient.""" + seen_clients: list[object] = [] + real_init = httpx.AsyncClient.__init__ + + def _tracking_init(self, *args, **kwargs): + seen_clients.append(self) + return real_init(self, *args, **kwargs) + + monkeypatch.setattr(httpx.AsyncClient, "__init__", _tracking_init) + provider = build_provider("anthropic", api_key="test-key") + # ... call the provider's translate method twice with a mocked transport ... + assert len(set(id(c) for c in seen_clients)) == 1 +``` + +Adapt the mocked-transport plumbing to whatever `test_nl2sql.py` already uses for provider tests (it +almost certainly already mocks the HTTP layer somehow to test translation without a real API call — reuse +that fixture). + +- [ ] **Step 3: Run test to verify it fails** + +Run: `uv run pytest tests/unit/test_nl2sql.py -k reuses_one_httpx_client -v` +Expected: FAIL — today's code constructs a new client per call. + +- [ ] **Step 4: Implement the shared client** — worked shape (adapt to the real class structure found in + Step 1): + +```python +class AnthropicProvider: + def __init__(self, api_key: str, *, base_url: str | None = None, timeout: float = ...) -> None: + self._client = httpx.AsyncClient(timeout=timeout) + ... + + async def translate(self, ...) -> ...: + response = await self._client.post(..., ...) + ... + + async def aclose(self) -> None: + """Close the underlying HTTP client. Call once when the provider is no longer needed.""" + await self._client.aclose() +``` + +Apply the equivalent for `OpenAIProvider`, `GeminiProvider`, and `OIDCVerifier`. Check whether anything +already calls a provider-lifecycle teardown hook (server shutdown, lifespan context) to wire `aclose()` +into — if MCPg's `server.py` has an existing lifespan/shutdown hook, add the new `aclose()` call there +rather than leaving the client to be garbage-collected unclosed. + +- [ ] **Step 5: Run test to verify it passes** + +Run: `uv run pytest tests/unit/test_nl2sql.py -k reuses_one_httpx_client -v` +Expected: PASS + +- [ ] **Step 6: Run the full check + tests** + +```bash +uv run ruff check . && uv run ruff format --check . && uv run mypy src/mcpg +uv run pytest -q tests/unit/test_nl2sql.py tests/unit/test_oidc.py +``` + +- [ ] **Step 7: Update CHANGELOG.md** under `Changed`: + +```markdown +- NL→SQL providers and the OIDC verifier now reuse one `httpx.AsyncClient` for their lifetime instead of + constructing a new client (and paying a fresh TCP/TLS handshake) per call. +``` + +- [ ] **Step 8: Commit** + +```bash +git add src/mcpg/nl2sql.py src/mcpg/oidc.py tests/unit/test_nl2sql.py tests/unit/test_oidc.py CHANGELOG.md +git commit -m "perf: reuse a shared httpx.AsyncClient in NL2SQL providers and OIDCVerifier + +Co-Authored-By: Claude Sonnet 5 " +``` + +--- + +### Task 10: HTTP transport fails closed without auth configured (BREAKING — maintainer-approved) + +**Files:** +- Modify: `src/mcpg/config.py` (add `http_allow_unauthenticated: bool = False` field + its + `MCPG_HTTP_ALLOW_UNAUTHENTICATED` env parse, next to the other `http_*` settings), `src/mcpg/http_runtime.py` + (lines ~605-613, the `else: logger.warning(...)` branch) +- Test: `tests/unit/test_config.py`, `tests/unit/test_http_runtime.py` + +**Interfaces:** +- `Settings.http_allow_unauthenticated: bool` (new field, default `False`). +- `build_http_app` raises `ConfigError` (not a warning) when `settings.auth_mode != "oidc"` and + `settings.http_auth_token is None` and `settings.http_allow_unauthenticated is False`. + +- [ ] **Step 1: Write the failing config test** + +```python +def test_http_allow_unauthenticated_defaults_false() -> None: + settings = load_settings({"MCPG_DATABASE_URL": "postgresql://u:p@localhost/db"}) + assert settings.http_allow_unauthenticated is False + + +def test_http_allow_unauthenticated_env_var_parses() -> None: + settings = load_settings({ + "MCPG_DATABASE_URL": "postgresql://u:p@localhost/db", + "MCPG_HTTP_ALLOW_UNAUTHENTICATED": "true", + }) + assert settings.http_allow_unauthenticated is True +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `uv run pytest tests/unit/test_config.py -k http_allow_unauthenticated -v` +Expected: FAIL — field doesn't exist. + +- [ ] **Step 3: Add the field to `Settings` and its loader** — follow the exact pattern the neighboring + `http_auth_token` field already uses in both the dataclass definition and `load_settings`'s parsing + block (read that pattern first, mirror it exactly rather than inventing a new style): + +```python +# In the Settings dataclass, near http_auth_token: +http_allow_unauthenticated: bool = False +``` + +```python +# In load_settings, near where http_auth_token is parsed: +http_allow_unauthenticated = False +if (raw := secrets.get("MCPG_HTTP_ALLOW_UNAUTHENTICATED")) is not None: + http_allow_unauthenticated = _parse_bool("MCPG_HTTP_ALLOW_UNAUTHENTICATED", raw) +``` + +(And add `http_allow_unauthenticated=http_allow_unauthenticated` to the final `Settings(...)` construction +call, matching every other field's wiring.) + +- [ ] **Step 4: Run to verify it passes** + +Run: `uv run pytest tests/unit/test_config.py -k http_allow_unauthenticated -v` +Expected: PASS + +- [ ] **Step 5: Write the failing `http_runtime` test** + +```python +def test_build_http_app_raises_without_auth_or_opt_out() -> None: + """HTTP transport refuses to start unauthenticated unless explicitly opted out.""" + settings = _settings_factory(auth_mode="none", http_auth_token=None, http_allow_unauthenticated=False) + with pytest.raises(ConfigError, match="unauthenticated"): + build_http_app(server=object(), settings=settings, kind="streamable-http") + + +def test_build_http_app_starts_with_explicit_opt_out() -> None: + """The MCPG_HTTP_ALLOW_UNAUTHENTICATED escape hatch still works, loudly logged.""" + settings = _settings_factory(auth_mode="none", http_auth_token=None, http_allow_unauthenticated=True) + app = build_http_app(server=object(), settings=settings, kind="streamable-http") + assert app is not None +``` + +Adapt `_settings_factory` to whatever `test_http_runtime.py` already uses to build a `Settings` instance +for these tests (it already has fixtures for the OIDC and bearer-token cases per the existing `assert +inner_invoked` tests found during the code review — reuse that pattern). + +- [ ] **Step 6: Run to verify it fails** + +Run: `uv run pytest tests/unit/test_http_runtime.py -k raises_without_auth -v` +Expected: FAIL — today's code only warns. + +- [ ] **Step 7: Implement the fail-closed check** + +```diff + if settings.http_auth_token is not None: + app.add_middleware(_BearerAuthMiddleware, token=settings.http_auth_token) ++ elif settings.http_allow_unauthenticated: ++ logger.warning( ++ "MCPg HTTP transport %s is running WITHOUT AUTH — MCPG_HTTP_ALLOW_UNAUTHENTICATED=true " ++ "was set explicitly. This is your deliberate choice; if it wasn't, unset that variable " ++ "and set MCPG_HTTP_AUTH_TOKEN or MCPG_AUTH_MODE=oidc instead.", ++ kind, ++ ) + else: +- logger.warning( +- "MCPg HTTP transport %s is running without auth. " +- "Set MCPG_HTTP_AUTH_TOKEN or MCPG_AUTH_MODE=oidc to require " +- "bearer tokens on every request.", +- kind, +- ) ++ raise ConfigError( ++ f"MCPg HTTP transport ({kind}) refuses to start unauthenticated. Set " ++ "MCPG_HTTP_AUTH_TOKEN, set MCPG_AUTH_MODE=oidc, or set " ++ "MCPG_HTTP_ALLOW_UNAUTHENTICATED=true to explicitly opt out (not recommended)." ++ ) +``` + +Import `ConfigError` from `mcpg.config` at the top of `http_runtime.py` if not already imported. + +- [ ] **Step 8: Run to verify it passes** + +Run: `uv run pytest tests/unit/test_http_runtime.py -k "raises_without_auth or starts_with_explicit_opt_out" -v` +Expected: PASS + +- [ ] **Step 9: Run the full check + tests** + +```bash +uv run ruff check . && uv run ruff format --check . && uv run mypy src/mcpg && uv run pytest -q +``` + +- [ ] **Step 10: Update `docs/user-guide.md` and/or `docs/installation.md`** — grep for existing HTTP- + transport setup instructions and add a line documenting the new required `MCPG_HTTP_AUTH_TOKEN` / + `MCPG_AUTH_MODE=oidc` / `MCPG_HTTP_ALLOW_UNAUTHENTICATED=true` choice: + +```bash +grep -rln "MCPG_HTTP_AUTH_TOKEN\|http.*transport" docs/*.md +``` + +Add a short paragraph to whichever file(s) that finds, next to the existing auth documentation. + +- [ ] **Step 11: Update CHANGELOG.md** under `Security` (this is the breaking one — flag it clearly): + +```markdown +### Security + +- **BREAKING: the HTTP transport now refuses to start unauthenticated by default.** Previously it started + anyway and only logged a warning if neither `MCPG_HTTP_AUTH_TOKEN` nor `MCPG_AUTH_MODE=oidc` was set. + Deployments that relied on the unauthenticated default must now either configure auth or set + `MCPG_HTTP_ALLOW_UNAUTHENTICATED=true` to explicitly opt back in (loudly logged when set). The default + `stdio` transport is unaffected. +``` + +- [ ] **Step 12: Commit** + +```bash +git add src/mcpg/config.py src/mcpg/http_runtime.py tests/unit/test_config.py tests/unit/test_http_runtime.py \ + docs/user-guide.md CHANGELOG.md +git commit -m "security!: HTTP transport fails closed without auth configured + +BREAKING CHANGE: the HTTP transport now raises ConfigError at startup instead of starting +unauthenticated-with-a-warning when neither MCPG_HTTP_AUTH_TOKEN nor MCPG_AUTH_MODE=oidc is +set. Set MCPG_HTTP_ALLOW_UNAUTHENTICATED=true to explicitly opt out. + +Co-Authored-By: Claude Sonnet 5 " +``` + +(Adjust the second `git add` path to whichever doc file Step 10 actually touched.) + +--- + +### Task 11: Rate limiting defaults to enabled (BREAKING — maintainer-approved) + +**Files:** +- Modify: `src/mcpg/config.py:192` (`rate_limit_enabled: bool = False`) and `:998` + (`rate_limit_enabled = False`, the `load_settings` default) +- Test: `tests/unit/test_config.py` + +**Interfaces:** No signature change — only the default value. + +- [ ] **Step 1: Write the failing test** + +```python +def test_rate_limit_enabled_defaults_true() -> None: + settings = load_settings({"MCPG_DATABASE_URL": "postgresql://u:p@localhost/db"}) + assert settings.rate_limit_enabled is True + + +def test_rate_limit_enabled_can_still_be_disabled_explicitly() -> None: + settings = load_settings({ + "MCPG_DATABASE_URL": "postgresql://u:p@localhost/db", + "MCPG_RATE_LIMIT_ENABLED": "false", + }) + assert settings.rate_limit_enabled is False +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `uv run pytest tests/unit/test_config.py -k rate_limit_enabled_defaults -v` +Expected: FAIL — current default is `False`. + +- [ ] **Step 3: Flip both defaults** + +```diff + # Settings dataclass, line 192: +- rate_limit_enabled: bool = False ++ rate_limit_enabled: bool = True +``` + +```diff + # load_settings, line 998: +- rate_limit_enabled = False ++ rate_limit_enabled = True +``` + +- [ ] **Step 4: Run to verify it passes** + +Run: `uv run pytest tests/unit/test_config.py -k rate_limit_enabled -v` +Expected: PASS + +- [ ] **Step 5: Check for tests that assumed the old default** — a global default flip like this can break + existing tests that constructed a `Settings`/loaded config without explicitly setting + `MCPG_RATE_LIMIT_ENABLED` and implicitly relied on it being off: + +```bash +uv run pytest -q 2>&1 | tail -40 +``` + +If any existing test fails because it now hits rate limiting unexpectedly, fix that test by having it set +`MCPG_RATE_LIMIT_ENABLED=false` explicitly (the test's actual intent was "rate limiting isn't the thing +under test here," which is still achievable — it just needs to say so now instead of relying on a default +that no longer holds) rather than reverting the default. + +- [ ] **Step 6: Run the full check + tests** + +```bash +uv run ruff check . && uv run ruff format --check . && uv run mypy src/mcpg && uv run pytest -q +``` + +- [ ] **Step 7: Update CHANGELOG.md** under `Security`: + +```markdown +- **BREAKING: rate limiting (`MCPG_RATE_LIMIT_ENABLED`) now defaults to `true`** (previously `false`). + Set it to `false` explicitly to restore the previous unlimited behavior. +``` + +- [ ] **Step 8: Commit** + +```bash +git add src/mcpg/config.py tests/unit/test_config.py CHANGELOG.md +git commit -m "security!: rate limiting enabled by default + +BREAKING CHANGE: MCPG_RATE_LIMIT_ENABLED now defaults to true. Set it to false explicitly to +restore the previous unlimited default. + +Co-Authored-By: Claude Sonnet 5 " +``` + +--- + +### Task 12: NL→SQL provenance — record the schema context a translation actually saw + +**Files:** +- Modify: `src/mcpg/nl2sql.py` (`TranslationResult` dataclass around line 298, and + `translate_nl_to_sql` around line 1084 where schema context is gathered and the result is constructed) +- Test: `tests/unit/test_nl2sql.py` + +**Interfaces:** +- `TranslationResult` gains one new field: `schema_context: str` (or `list[str]` if the existing internal + representation is already structured — match whatever `translate_nl_to_sql` already builds internally + rather than introducing a second representation). This is additive to the dataclass — every existing + field stays, in place, so nothing that constructs or reads a `TranslationResult` today breaks except + code that builds one positionally without the new field (check for that pattern specifically). + +- [ ] **Step 1: Read how schema context is currently gathered** + +```bash +grep -n "schema.*context\|gather.*schema\|def translate_nl_to_sql" src/mcpg/nl2sql.py | head -10 +``` + +Confirm the exact variable holding the schema context before writing Step 2. + +- [ ] **Step 2: Write the failing test** + +```python +async def test_translation_result_records_the_schema_context_it_saw(monkeypatch) -> None: + """A caller can trace generated SQL back to the schema evidence the model was given.""" + # ... mock the provider call so the model's raw response is controlled ... + result = await translate_nl_to_sql(driver=fake_driver, question="how many users?", settings=settings) + assert result.schema_context # non-empty + assert "users" in result.schema_context # the table the question is actually about was included +``` + +Adapt the fake-driver/provider mocking to whatever `test_nl2sql.py`'s existing `translate_nl_to_sql` tests +already use. + +- [ ] **Step 3: Run to verify it fails** + +Run: `uv run pytest tests/unit/test_nl2sql.py -k schema_context_it_saw -v` +Expected: FAIL — `TranslationResult` has no `schema_context` field. + +- [ ] **Step 4: Add the field and populate it** + +```diff + @dataclass(frozen=True, slots=True) + class TranslationResult: + sql: str + explanation: str + model: str + provider: str ++ schema_context: str + executed: bool + ... +``` + +In `translate_nl_to_sql`, thread the already-gathered schema-context string (found in Step 1) into the +`TranslationResult(...)` construction call — it's already computed and sent to the model as part of the +prompt; this task only makes it visible on the return value instead of discarding it after the model call. + +- [ ] **Step 5: Run to verify it passes** + +Run: `uv run pytest tests/unit/test_nl2sql.py -k schema_context_it_saw -v` +Expected: PASS + +- [ ] **Step 6: Check every other `TranslationResult(...)` construction site** — any early-return path + (parse failure, safety-check rejection) also constructs a `TranslationResult`; give each an accurate + `schema_context` value (the context that *was* gathered before the failure, or an empty string only if + gathering itself never happened on that path — verify per-site, don't default all of them to `""` + without checking). + +- [ ] **Step 7: Run the full check + tests** + +```bash +uv run ruff check . && uv run ruff format --check . && uv run mypy src/mcpg +uv run pytest -q tests/unit/test_nl2sql.py +``` + +- [ ] **Step 8: Update CHANGELOG.md** under `Added`: + +```markdown +- `TranslationResult` (NL→SQL) now records `schema_context` — the schema evidence actually sent to the + model for that translation — so a generated query's provenance is traceable, not just which + model/provider produced it. +``` + +- [ ] **Step 9: Commit** + +```bash +git add src/mcpg/nl2sql.py tests/unit/test_nl2sql.py CHANGELOG.md +git commit -m "feat(nl2sql): record schema-context provenance on TranslationResult + +Co-Authored-By: Claude Sonnet 5 " +``` + +--- + +### Task 13: Centralized log-redaction filter + +**Files:** +- Modify: `src/mcpg/obs_logging.py` (add a `logging.Filter` subclass, attach it in `setup_logging`) +- Test: `tests/unit/test_obs_logging.py` (check it exists; create if not) + +**Interfaces:** +- Produces: `RedactionFilter(logging.Filter)` — its `filter(record)` method rewrites `record.msg` / + `record.args` (or the JSON-formatted output, depending on where `JSONFormatter` does its work — read + `JSONFormatter.format` first) to pass any already-obfuscated string through unchanged, and additionally + runs `mcpg.sql.obfuscate_password` (the existing, already-tested redaction function — reuse it, don't + reimplement) over the rendered message as a backstop for the case a call site forgot to call it directly. + +- [ ] **Step 1: Read `JSONFormatter.format` and `obfuscate_password` fully** + +```bash +grep -n "def obfuscate_password" -A 15 src/mcpg/sql/driver.py +``` + +- [ ] **Step 2: Write the failing test** + +```python +def test_redaction_filter_scrubs_a_connection_string_even_when_a_call_site_forgot(caplog) -> None: + """The centralized filter catches a password-bearing log line even without obfuscate_password.""" + logger = logging.getLogger("mcpg.test_redaction") + logger.addFilter(RedactionFilter()) + with caplog.at_level(logging.INFO, logger="mcpg.test_redaction"): + logger.info("connecting to postgresql://user:hunter2@host/db") + assert "hunter2" not in caplog.text +``` + +- [ ] **Step 3: Run to verify it fails** + +Run: `uv run pytest tests/unit/test_obs_logging.py -k redaction_filter -v` +Expected: FAIL — `RedactionFilter` doesn't exist. + +- [ ] **Step 4: Implement** + +```python +import logging + +from mcpg.sql import obfuscate_password + + +class RedactionFilter(logging.Filter): + """Backstop redaction: scrubs any password-bearing connection string that reaches a log + call without having been passed through obfuscate_password() at the call site. + + Not a replacement for calling obfuscate_password() explicitly where a value is known to + carry credentials — that per-call-site discipline still matters for accuracy (this filter + only recognizes the same connection-string shapes obfuscate_password() already does). This + is the centralized enforcement layer for the case a future call site forgets. + """ + + def filter(self, record: logging.LogRecord) -> bool: + record.msg = obfuscate_password(record.getMessage()) + record.args = () + return True +``` + +Attach it in `setup_logging`: + +```diff + def setup_logging(settings: Settings) -> None: + ... ++ for handler in logging.getLogger("mcpg").handlers: ++ handler.addFilter(RedactionFilter()) +``` + +(Match the exact handler-attachment shape `setup_logging` already uses — read the rest of the function +before adding this, since it may already loop over handlers in a specific way this should follow.) + +- [ ] **Step 5: Run to verify it passes** + +Run: `uv run pytest tests/unit/test_obs_logging.py -k redaction_filter -v` +Expected: PASS + +- [ ] **Step 6: Run the full check + tests** + +```bash +uv run ruff check . && uv run ruff format --check . && uv run mypy src/mcpg +uv run pytest -q tests/unit/test_obs_logging.py +``` + +- [ ] **Step 7: Update CHANGELOG.md** under `Security`: + +```markdown +- Added a centralized log-redaction filter (`RedactionFilter`) as a backstop for any log call that + reaches a handler without having explicitly redacted a connection string first — complements, doesn't + replace, the existing per-call-site `obfuscate_password()` discipline. +``` + +- [ ] **Step 8: Commit** + +```bash +git add src/mcpg/obs_logging.py tests/unit/test_obs_logging.py CHANGELOG.md +git commit -m "security: add centralized log-redaction filter as a backstop + +Co-Authored-By: Claude Sonnet 5 " +``` + +--- + +### Task 14: Audit `logger.error`/`logger.warning` calls inside `except` blocks for lost tracebacks + +**Files:** +- Modify: every site found by the audit below (confirmed starting point: `tenancy.py:245`; re-run the + search at execution time for the full current list) + +**Interfaces:** None — same as Task 6, this only adds `exc_info=True` (or switches to `logger.exception`), +never changes control flow. + +- [ ] **Step 1: Enumerate every candidate site** + +```bash +grep -rn "logger\.\(error\|warning\)(" src/mcpg/*.py +``` + +For each hit, read 10 lines of context above it. Classify each as: (a) inside an `except` block, logging +about the exception that was just caught, with no `exc_info=True` and not `logger.exception` → **fix**; +(b) inside an `except` block but logging about something unrelated to the exception itself (e.g., a +retry-attempt counter) → leave as-is; (c) not inside an `except` block at all → leave as-is. Build the +concrete list from this run, not from the 2-hit sample in the earlier code-review report — that report's +own coverage note said its grep-based estimate wasn't exhaustive. + +- [ ] **Step 2: For each site classified "fix," change it** — worked example (`tenancy.py:245`): + +```diff +- logger.error("Error rolling back transaction during role-wrapped execute: %s", rollback_error) ++ logger.error("Error rolling back transaction during role-wrapped execute: %s", rollback_error, exc_info=True) +``` + +Prefer `logger.exception(...)` (which implies `exc_info=True` and must be called from inside the `except` +block) over manually adding `exc_info=True` when the call site is already positioned to use it — check +each site for which form fits its existing structure with the smaller diff. + +- [ ] **Step 3: Run the full check + tests after every ~5 sites fixed** (not all at once — this touches + many files; verify incrementally): + +```bash +uv run ruff check . && uv run mypy src/mcpg && uv run pytest -q +``` + +- [ ] **Step 4: Update CHANGELOG.md** under `Fixed`, once the final count is known: + +```markdown +- Error-logging call sites inside `except` blocks now preserve tracebacks (`exc_info=True` or + `logger.exception`) where they previously logged only the exception's string form — audited across + `src/mcpg`, N sites fixed (see PR diff for the full list). +``` + +- [ ] **Step 5: Commit** (one commit for this whole task, listing every touched file): + +```bash +git add src/mcpg/*.py CHANGELOG.md +git commit -m "fix: preserve tracebacks in error-logging call sites inside except blocks + +Co-Authored-By: Claude Sonnet 5 " +``` + +--- + +### Task 15: Circuit breaker on external calls (LLM providers, OIDC JWKS fetch) + +**Files:** +- Modify: `pyproject.toml` (add `circuitbreaker` to `[project.dependencies]` or a suitable extra — check + whether NL2SQL/OIDC are already gated behind an extra like `otel` is, or are core deps; match that + shape), `src/mcpg/nl2sql.py` (each provider's HTTP call), `src/mcpg/oidc.py` (`_ensure_jwks_client`) +- Test: `tests/unit/test_nl2sql.py`, `tests/unit/test_oidc.py` + +**Interfaces:** +- Each provider's translate call and the JWKS-client-resolution call get wrapped with + `circuitbreaker.circuit` (sync-compatible; confirm its async support directly against the installed + package's own docs/tests before assuming — the audit report noted it as "sync+async support" from a + secondary characterization, verify against the actual library once it's installed). + +- [ ] **Step 1: Add the dependency** + +```bash +uv add circuitbreaker +``` + +- [ ] **Step 2: Confirm async support directly** + +```bash +uv run python -c "import circuitbreaker; help(circuitbreaker.circuit)" +``` + +Read the actual signature/docstring rather than assuming from the audit report's secondary source. If it +turns out not to support `async def` cleanly, fall back to a small hand-rolled closed/open/half-open +wrapper instead of forcing a mismatched library onto async code — note that deviation in the commit message +if it happens. + +- [ ] **Step 3: Write the failing test** (worked for the OIDC JWKS fetch — adapt the same shape for each + NL2SQL provider): + +```python +async def test_jwks_fetch_opens_circuit_after_repeated_failures(monkeypatch) -> None: + """After enough consecutive JWKS-fetch failures, the breaker opens and fails fast.""" + verifier = OIDCVerifier(issuer="https://idp.example", audience="mcpg", jwks_url="https://idp.example/jwks") + call_count = 0 + + async def _always_fails(*args, **kwargs): + nonlocal call_count + call_count += 1 + raise httpx.ConnectError("simulated outage") + + monkeypatch.setattr(verifier, "_fetch_jwks", _always_fails) # or whatever the real internal call is named + for _ in range(10): + with pytest.raises(Exception): + await verifier.verify("some-token") + # After the breaker opens, further calls should fail fast without re-invoking _fetch_jwks + calls_before_open = call_count + with pytest.raises(Exception): + await verifier.verify("another-token") + assert call_count == calls_before_open # breaker short-circuited, didn't call _fetch_jwks again +``` + +- [ ] **Step 4: Run to verify it fails** + +Run: `uv run pytest tests/unit/test_oidc.py -k opens_circuit -v` +Expected: FAIL — no breaker exists yet. + +- [ ] **Step 5: Wrap the calls** + +```python +from circuitbreaker import circuit + +class OIDCVerifier: + @circuit(failure_threshold=5, recovery_timeout=30) + async def _fetch_jwks_client(self) -> PyJWKClient: + ... +``` + +(Confirm `@circuit`'s exact parameter names against Step 2's findings — `failure_threshold`/ +`recovery_timeout` are illustrative, not verified.) Apply the equivalent decorator to each NL2SQL +provider's HTTP call method. + +- [ ] **Step 6: Run to verify it passes** + +Run: `uv run pytest tests/unit/test_oidc.py -k opens_circuit -v` +Expected: PASS + +- [ ] **Step 7: Run the full check + tests** + +```bash +uv run ruff check . && uv run ruff format --check . && uv run mypy src/mcpg +uv run pytest -q tests/unit/test_nl2sql.py tests/unit/test_oidc.py +``` + +- [ ] **Step 8: Update CHANGELOG.md** under `Added`: + +```markdown +- Circuit breaker (`circuitbreaker`) around NL→SQL provider calls and the OIDC JWKS fetch — repeated + failures now fail fast instead of each request separately paying the full timeout cost against a + degraded dependency. +``` + +- [ ] **Step 9: Commit** + +```bash +git add pyproject.toml uv.lock src/mcpg/nl2sql.py src/mcpg/oidc.py tests/unit/test_nl2sql.py \ + tests/unit/test_oidc.py CHANGELOG.md +git commit -m "feat: add circuit breaker around external LLM provider and OIDC JWKS calls + +Co-Authored-By: Claude Sonnet 5 " +``` + +--- + +### Task 16: Retry with backoff on external calls + +**Files:** +- Modify: `pyproject.toml` (add `tenacity`), `src/mcpg/nl2sql.py`, `src/mcpg/oidc.py` +- Test: `tests/unit/test_nl2sql.py`, `tests/unit/test_oidc.py` + +**Interfaces:** +- The same call sites Task 15 wrapped with `@circuit` also get `tenacity.retry` — order matters: retry + should sit *inside* the circuit breaker (retry a few times quickly, and only count the whole retried + attempt as one failure toward the breaker's threshold), not outside it (which would let retries alone + exhaust the breaker's threshold in one logical call). Confirm this ordering explicitly in code, don't + leave it to decorator-application order being accidentally correct. + +- [ ] **Step 1: Add the dependency** + +```bash +uv add tenacity +``` + +- [ ] **Step 2: Write the failing test** + +```python +async def test_jwks_fetch_retries_transient_failures_before_giving_up(monkeypatch) -> None: + """A JWKS fetch that fails twice then succeeds is retried, not immediately surfaced as an error.""" + attempts = 0 + + async def _fails_twice_then_succeeds(*args, **kwargs): + nonlocal attempts + attempts += 1 + if attempts < 3: + raise httpx.ConnectError("transient") + return _fake_jwks_client() + + verifier = OIDCVerifier(issuer="https://idp.example", audience="mcpg", jwks_url="https://idp.example/jwks") + monkeypatch.setattr(verifier, "_fetch_jwks_client", _fails_twice_then_succeeds) + await verifier._ensure_jwks_client() + assert attempts == 3 +``` + +- [ ] **Step 3: Run to verify it fails** + +Run: `uv run pytest tests/unit/test_oidc.py -k retries_transient -v` +Expected: FAIL — no retry exists yet. + +- [ ] **Step 4: Implement** + +```python +from tenacity import retry, stop_after_attempt, wait_exponential_jitter + +class OIDCVerifier: + @retry(stop=stop_after_attempt(3), wait=wait_exponential_jitter(initial=0.5, max=5)) + @circuit(failure_threshold=5, recovery_timeout=30) + async def _fetch_jwks_client(self) -> PyJWKClient: + ... +``` + +(`retry` outermost so a single logical call's retries count as one attempt toward the breaker — verify +this reads correctly against `circuitbreaker`'s actual failure-counting semantics from Task 15's Step 2 +findings, adjust ordering if the library counts differently than assumed here.) + +- [ ] **Step 5: Run to verify it passes** + +Run: `uv run pytest tests/unit/test_oidc.py -k retries_transient -v` +Expected: PASS + +- [ ] **Step 6: Run the full check + tests** + +```bash +uv run ruff check . && uv run ruff format --check . && uv run mypy src/mcpg +uv run pytest -q tests/unit/test_nl2sql.py tests/unit/test_oidc.py +``` + +- [ ] **Step 7: Update CHANGELOG.md** under `Added`: + +```markdown +- Retry with exponential backoff + jitter (`tenacity`) around NL→SQL provider calls and the OIDC JWKS + fetch, layered inside the circuit breaker added above. +``` + +- [ ] **Step 8: Commit** + +```bash +git add pyproject.toml uv.lock src/mcpg/nl2sql.py src/mcpg/oidc.py tests/unit/test_nl2sql.py \ + tests/unit/test_oidc.py CHANGELOG.md +git commit -m "feat: add retry-with-backoff around external LLM provider and OIDC JWKS calls + +Co-Authored-By: Claude Sonnet 5 " +``` + +--- + +### Task 17: Scope `bandit`'s `B608` skip instead of a repo-wide exemption + +**Files:** +- Modify: `pyproject.toml:[tool.bandit] skips`, `.github/workflows/ci.yml` (mirrors the same skip list — + check it matches after this change), plus inline `# nosec B608` at each site bandit would otherwise flag + once the global skip is removed +- Test: none (CI-tooling config; verified by running bandit itself, not pytest) + +**Interfaces:** None. + +- [ ] **Step 1: Remove the global skip and see what bandit actually flags** + +```bash +uv run bandit -r src/mcpg --skip B101,B110 -ll +``` + +(Dropped `B608` from the skip list for this diagnostic run only — don't edit the config yet.) + +- [ ] **Step 2: For each B608 hit, add a scoped `# nosec B608` with a justification comment** — worked + example (adapt to each real hit's actual line, don't apply this verbatim without checking): + +```python +query = f"SELECT * FROM {table_name}" # nosec B608 — table_name is validated against + # _SECONDARY_DB_NAME/an identifier allowlist above, + # never raw user input; see sql/allowlist.py for the + # broader query-construction safety model. +``` + +Every `# nosec B608` must carry an accurate, specific justification — if a hit turns out NOT to be a false +positive (i.e., it's an actual place user input could reach unescaped SQL construction), stop and treat it +as a real Critical security finding instead of suppressing it. Re-verify each site's actual safety, don't +assume the earlier audit's characterization was exhaustive. + +- [ ] **Step 3: Update `pyproject.toml` and `ci.yml` to drop `B608` from the global skip list** + +```diff + [tool.bandit] + exclude_dirs = ["tests"] +- skips = ["B101", "B608", "B110"] ++ skips = ["B101", "B110"] +``` + +```diff + # .github/workflows/ci.yml, the bandit step: +- run: uv run bandit -r src/mcpg --skip B101,B608,B110 -ll ++ run: uv run bandit -r src/mcpg --skip B101,B110 -ll +``` + +- [ ] **Step 4: Confirm bandit passes clean with the narrower skip** + +```bash +uv run bandit -r src/mcpg --skip B101,B110 -ll +``` + +Expected: no findings (every real hit now carries an inline `# nosec B608` from Step 2). + +- [ ] **Step 5: Update CHANGELOG.md** under `Changed`: + +```markdown +- Scoped `bandit`'s `B608` (hardcoded-SQL) suppression from a repo-wide skip to per-site `# nosec B608` + annotations with justification comments, so the check stays load-bearing for any future module that + builds a query string unsafely. +``` + +- [ ] **Step 6: Commit** + +```bash +git add pyproject.toml .github/workflows/ci.yml src/mcpg/*.py CHANGELOG.md +git commit -m "chore(security): scope bandit B608 suppression to justified per-site annotations + +Co-Authored-By: Claude Sonnet 5 " +``` + +--- + +### Task 18: License enumeration in CI (`pip-licenses`) + +**Files:** +- Modify: `pyproject.toml` (dev dependency), `.github/workflows/ci.yml` (new step in the `security` job) + +**Interfaces:** None — CI-only. + +- [ ] **Step 1: Add the dependency** + +```bash +uv add --dev pip-licenses +``` + +- [ ] **Step 2: Add a CI step** (in the existing `security` job, alongside `pip-audit`/`bandit`): + +```yaml + - name: License enumeration (pip-licenses) + run: uv run pip-licenses --format=markdown --with-urls --order=license +``` + +Non-blocking report for now (per `project-structure.md`'s own "warn by default, promote to block once +stable" CI-gate guidance) — don't add `--fail-on` yet without the maintainer first reviewing what the +current dependency tree's license mix actually looks like. + +- [ ] **Step 3: Run locally to confirm it doesn't error** + +```bash +uv run pip-licenses --format=markdown --with-urls --order=license | head -20 +``` + +- [ ] **Step 4: Update CHANGELOG.md** under `Added`: + +```markdown +- License enumeration (`pip-licenses`) added to CI as a non-blocking report step. +``` + +- [ ] **Step 5: Commit** + +```bash +git add pyproject.toml uv.lock .github/workflows/ci.yml CHANGELOG.md +git commit -m "ci: add license enumeration (pip-licenses) as a non-blocking report + +Co-Authored-By: Claude Sonnet 5 " +``` + +--- + +### Task 19: HSTS default bump + `TrustedHostMiddleware` + +**Files:** +- Modify: `src/mcpg/http_runtime.py` (the `hsts_max_age: int = 31536000` default; add + `TrustedHostMiddleware` wiring), `src/mcpg/config.py` (`http_hsts_max_age` default, if it's config-driven + rather than hardcoded — check both) +- Test: `tests/unit/test_http_runtime.py` + +**Interfaces:** +- `Settings.http_trusted_hosts: tuple[str, ...] = ()` — new field, empty tuple meaning "no host-header + validation configured" (matches the existing `http_allowed_origins` empty-tuple-means-off convention for + CORS — follow that exact pattern). + +- [ ] **Step 1: Bump the HSTS default** + +```bash +grep -rn "31536000\|hsts_max_age" src/mcpg/config.py src/mcpg/http_runtime.py +``` + +Change every occurrence of the `31536000` (1-year) default to `63072000` (2-year, OWASP's current +recommendation) — both the dataclass field default and the `load_settings` parse default, matching +whichever pattern every other `http_*` numeric setting already uses. + +- [ ] **Step 2: Write the failing test for `TrustedHostMiddleware`** + +```python +def test_trusted_host_middleware_added_when_configured() -> None: + settings = _settings_factory(http_trusted_hosts=("api.example.com",)) + app = build_http_app(server=object(), settings=settings, kind="streamable-http") + middleware_classes = [m.cls for m in app.user_middleware] + assert TrustedHostMiddleware in middleware_classes + + +def test_trusted_host_middleware_absent_when_not_configured() -> None: + settings = _settings_factory(http_trusted_hosts=()) + app = build_http_app(server=object(), settings=settings, kind="streamable-http") + middleware_classes = [m.cls for m in app.user_middleware] + assert TrustedHostMiddleware not in middleware_classes +``` + +- [ ] **Step 3: Run to verify it fails** + +Run: `uv run pytest tests/unit/test_http_runtime.py -k trusted_host -v` +Expected: FAIL — no such field/wiring yet. + +- [ ] **Step 4: Implement** — add `http_trusted_hosts` to `Settings` and its loader (same pattern as + `http_allowed_origins`, comma-split env var), then wire the middleware conditionally, mirroring the + existing CORS conditional: + +```diff + if settings.http_allowed_origins: + from starlette.middleware.cors import CORSMiddleware + ... ++ if settings.http_trusted_hosts: ++ from starlette.middleware.trustedhost import TrustedHostMiddleware ++ app.add_middleware(TrustedHostMiddleware, allowed_hosts=list(settings.http_trusted_hosts)) +``` + +- [ ] **Step 5: Run to verify it passes** + +Run: `uv run pytest tests/unit/test_http_runtime.py -k trusted_host -v` +Expected: PASS + +- [ ] **Step 6: Run the full check + tests** + +```bash +uv run ruff check . && uv run ruff format --check . && uv run mypy src/mcpg && uv run pytest -q +``` + +- [ ] **Step 7: Update CHANGELOG.md** under `Changed` and `Added`: + +```markdown +- HSTS `max-age` default bumped from 31536000 (1 year) to 63072000 (2 years), OWASP's current + recommendation — the old value remains the `hstspreload.org` minimum-eligibility floor, not the target. +- Optional `TrustedHostMiddleware` support via `MCPG_HTTP_TRUSTED_HOSTS` (comma-separated), off by + default, matching the existing `MCPG_HTTP_ALLOWED_ORIGINS` convention. +``` + +- [ ] **Step 8: Commit** + +```bash +git add src/mcpg/config.py src/mcpg/http_runtime.py tests/unit/test_http_runtime.py CHANGELOG.md +git commit -m "security: bump HSTS default to 2 years, add optional TrustedHostMiddleware + +Co-Authored-By: Claude Sonnet 5 " +``` + +--- + +### Task 20: `.env.example` + +**Files:** +- Create: `.env.example` + +**Interfaces:** None. + +- [ ] **Step 1: Enumerate every `MCPG_*` env var** — the authoritative source is `config.py`'s + `load_settings`, not the docs (docs can drift; `config.py` cannot): + +```bash +grep -on "MCPG_[A-Z_]*" src/mcpg/config.py | sort -u -t: -k2 +``` + +- [ ] **Step 2: Write the file** — one line per variable, commented, no real values, grouped by the same + sections `config.py`'s own `Settings` dataclass uses (connection, pool, transport, auth, rate-limit, + observability, secrets-backend, nl2sql): + +```bash +# MCPg configuration template — copy to .env and fill in real values. +# Every MCPG_* variable this server reads; see docs/user-guide.md for full detail on each. + +# --- Database (required) --- +MCPG_DATABASE_URL=postgresql://user:password@localhost:5432/dbname + +# --- Access mode --- +# MCPG_ACCESS_MODE=restricted # read-only | restricted | unrestricted + +# ... (continue for every variable Step 1 found, grouped logically, each with a one-line comment) +``` + +- [ ] **Step 3: Cross-check against `docs/user-guide.md`** — confirm every variable documented there also + appears in `.env.example` (and vice versa); if the two disagree, `config.py`'s actual behavior is the + tiebreaker, and the doc (not `.env.example`) is what's wrong — flag any doc drift found, don't silently + paper over it by matching the doc instead of the code. + +- [ ] **Step 4: Commit** + +```bash +git add .env.example +git commit -m "docs: add .env.example documenting every MCPG_* variable + +Co-Authored-By: Claude Sonnet 5 " +``` + +--- + +### Task 21: Add remaining dev dependencies (`pytest-mock`, `pytest-randomly`, `pytest-socket`, `time-machine`, `pytest-rerunfailures`) + +**Files:** +- Modify: `pyproject.toml` (`[dependency-groups] dev`) + +**Interfaces:** None yet — this task only adds the dependencies and confirms the suite still passes under +`pytest-randomly`'s order randomization. It does **not** globally enable `pytest-socket`'s network-blocking +mode (that's a separate, riskier change — see Step 4) or rewrite existing `unittest.mock` usage to +`pytest-mock` (that's cosmetic churn across 11 files with no correctness benefit — out of scope here). + +- [ ] **Step 1: Add the dependencies** + +```bash +uv add --dev pytest-mock pytest-randomly pytest-socket time-machine pytest-rerunfailures +``` + +- [ ] **Step 2: Run the full unit suite** — `pytest-randomly` activates automatically once installed + (no config needed) and will genuinely reorder tests: + +```bash +uv run pytest -q tests/unit +``` + +- [ ] **Step 3: If anything fails under randomized order, that's a real order-dependence bug, not a tool + problem** — fix the actual test isolation issue (shared mutable module state, a fixture scoped wider + than its real reuse, leftover state from a prior test) rather than pinning a fixed seed to hide it. If a + fix isn't tractable in this task's scope, document the specific failing test and file it as a known + issue rather than silently reverting the dependency addition. + +- [ ] **Step 4: Leave `pytest-socket` unwired for now** — adding the dependency doesn't activate blocking + by default; that requires an explicit `--disable-socket` flag or `pytest_socket.disable_socket()` in + `conftest.py`. Flipping that on repo-wide risks breaking any currently-passing test that makes a real + call and was never audited for it — out of scope for this task. Note in the CHANGELOG that the dependency + is available for opt-in per-test use (`@pytest.mark.disable_socket` or equivalent) but not globally + enabled yet. + +- [ ] **Step 5: Update CHANGELOG.md** under `Added`: + +```markdown +- Dev dependencies: `pytest-mock`, `pytest-randomly` (test-order randomization — active by default once + installed), `pytest-socket` (available for opt-in per-test network blocking, not globally enabled), + `time-machine`, `pytest-rerunfailures`. +``` + +- [ ] **Step 6: Commit** + +```bash +git add pyproject.toml uv.lock CHANGELOG.md +git commit -m "chore(deps): add pytest-mock, pytest-randomly, pytest-socket, time-machine, pytest-rerunfailures + +Co-Authored-By: Claude Sonnet 5 " +``` + +--- + +### Task 22: Fix the 5 weak `assert ` test assertions + +**Files:** +- Modify: `tests/unit/test_demo.py:48`, `tests/unit/test_http_runtime.py:125,146,380`, + `tests/unit/test_warehousepg_reads.py:301` + +**Interfaces:** None — test-only. + +- [ ] **Step 1: Read each site's full context** (the 10 lines above each assertion) to determine what the + flag/value actually represents, then strengthen each to assert the specific expected content, not just + truthiness. Worked example — read `test_http_runtime.py` around line 125 first, then, illustratively: + +```diff +- assert inner_invoked ++ assert inner_invoked is True # or, if inner_invoked is meant to carry the call's argument: ++ assert inner_invoked == expected_call_args +``` + +Do this per-site based on what each test actually sets `inner_invoked`/`text`/`ao_calls` to — don't apply +a single mechanical transform to all five without reading what each one means. + +- [ ] **Step 2: Run the full test suite** + +```bash +uv run pytest -q +``` + +- [ ] **Step 3: Commit** + +```bash +git add tests/unit/test_demo.py tests/unit/test_http_runtime.py tests/unit/test_warehousepg_reads.py +git commit -m "test: strengthen 5 bare-truthy assertions to check actual expected values + +Co-Authored-By: Claude Sonnet 5 " +``` + +(No CHANGELOG entry — test-quality-only, no shipped behavior change.) + +--- + +### Task 23: Enable and fix the opt-in ruff categories that pass an assess-first bar + +**Rescoped after direct measurement** (2026-08-25) — the objective is to assess each finding and fix what's +actually worth fixing, not mechanically drive every ruff category to zero regardless of value. Measured +before deciding: + +- `mypy --strict` passes with **zero issues** across all 108 source files today. `ANN` (475 violations) + checks annotation presence via lint pattern-matching on top of a codebase mypy strict mode already + independently guarantees is fully typed — low marginal value. **Not fixed, not enabled this pass.** +- `D` (2,284 violations, the largest category by far) is **100% outside `tools.py`** — the actual public + MCP tool surface (`uv run ruff check --select D103 src/mcpg/tools.py` → 0 hits) already has docstrings + everywhere it matters for the LLM agent consuming these tools. The 2,284 hits are internal + implementation-module documentation debt. Real, but not worth manufacturing ~2,300 docstrings + (`CLAUDE.md`'s "verify before you write" rule means each would need genuine per-function reading, not + templated filler) for a category with no functional consumer. **Not fixed, not enabled this pass** — + reported as a measured baseline for a possible dedicated future documentation pass, not silently dropped. +- `TC` (165) and `PT` (75, test-only) are mechanical style/hygiene with no functional-bug or + contract-safety story behind them. **Deferred**, not fixed this pass. +- `FBT` (196 total) is the opposite case: **103 of 196 (52%) are in `tools.py`** — real value, since + boolean-trap clarity on a tool signature genuinely affects whether an LLM agent calls it correctly. + **Fixed, prioritizing the `tools.py` subset first**, via the tool-snapshot-regeneration process below. +- `C90` (67, one confirmed case already read: `sql/safety.py`'s `_validate_node`, complexity 22 — the + fuzz-tested, adversarially-pinned AST walker) is assessed **per function**: refactored where it's a cheap, + safe win; justified-suppressed (citing the module's own existing correctness rationale, not a new excuse) + where the complexity is inherent to a security-critical algorithm and a refactor would be pure risk for + no safety benefit. +- `PYI` (16), `ASYNC` (18), `PTH` (8), `C4`, `SIM` (~40 combined) are small enough that "fix everything in + this category" and "assess each one" converge on the same amount of work — fixed in full. + +**Categories enabled in `pyproject.toml` by the end of this task:** `C90`, `ASYNC`, `C4`, `SIM`, `PTH`, +`PYI`, `FBT`. **Categories measured, reported, and deliberately left unenabled:** `D`, `ANN`, `TC`, `PT` — +see the CHANGELOG entry in Step 13 below for the exact baseline counts to hand off if a future pass wants +to pick these up. + +**Files:** +- Modify: `pyproject.toml:205` (`[tool.ruff.lint] select`), plus every file with a violation — this is a + repo-wide sweep, executed and verified category-by-category, not file-by-file. + +**Interfaces:** None — pure lint/style/documentation remediation, no behavior change. Any diff that looks +like it would change behavior (an `SIM`/`C4` autofix that alters control flow, an `FBT` fix that changes a +function's calling convention) must be double-checked against the test suite before being accepted as +"just a lint fix." + +**Methodology** (this task is executed as repeated apply-and-verify loops per category, not as +individually pre-authored diffs — the categories are ordered smallest/safest to largest/riskiest): + +- [ ] **Step 1: `PTH` (flake8-use-pathlib) — 8 violations, autofix + review** + +```bash +uv run ruff check --select PTH --fix . +uv run ruff check --select PTH . # confirm 0 remaining +uv run git diff --stat # sanity-check the diff shape is what's expected (os.path -> Path calls) +``` + +Review the diff by hand (8 hits is small enough to read every one) — confirm no `Path` conversion changed +actual runtime behavior around symlinks or relative-path resolution before accepting. + +- [ ] **Step 2: `C4` (flake8-comprehensions) — check current count, autofix** + +```bash +uv run ruff check --select C4 --statistics . +uv run ruff check --select C4 --fix . +uv run ruff check --select C4 . # confirm 0 remaining +``` + +- [ ] **Step 3: `SIM` (flake8-simplify) — autofixable subset first, then manual for the rest** + +```bash +uv run ruff check --select SIM --fix . # picks up the [*]-marked autofixable rules +uv run ruff check --select SIM --statistics . # see what's left (non-autofixable: SIM105, SIM108, SIM117, SIM102, etc.) +``` + +For each remaining manual violation, read the specific rule's rationale (`ruff rule ` or +`docs.astral.sh/ruff/rules/`) and apply the suggested pattern — e.g. `SIM105` +(`suppressible-exception`) becomes `contextlib.suppress(...)`, `SIM117` collapses nested `with` statements. +Run `uv run ruff check --select SIM .` after each batch of ~10 files until it reports 0. + +- [ ] **Step 4: `TC` and `PT` — skipped, per the rescoping note above.** Mechanical style/hygiene + categories with no functional-bug or contract-safety story; not enabled, not fixed this pass. Baseline + counts (165 and 75 respectively, measured 2026-08-25) go in the Step 8 CHANGELOG entry as a reported + follow-up candidate. + +- [ ] **Step 6: `ASYNC` (flake8-async) — 18 violations, review each individually (correctness-adjacent, + not just style)** + +```bash +uv run ruff check --select ASYNC --statistics . +``` + +For each `ASYNC109` (14, `async-function-with-timeout`) hit: read the flagged function and confirm whether +it's genuinely reimplementing what `asyncio.timeout()` already provides — if so, refactor to use +`asyncio.timeout()` directly; if the existing pattern is intentionally different for a reason (e.g., needs +to distinguish timeout from cancellation), add a scoped `# noqa: ASYNC109` with a comment explaining why. +For the 3 `ASYNC240` (`blocking-path-method-in-async-function`) and 1 `ASYNC221` +(`run-process-in-async-function`) hits: these are the correctness-relevant ones — confirm whether the +flagged call actually blocks the event loop in practice (a `Path.exists()` on a local, fast filesystem path +during startup is a different risk than one on a hot per-request path) and fix by moving the call to +`asyncio.to_thread(...)` where it's a genuine concern, or suppress with a justification comment where it +isn't. + +```bash +uv run ruff check --select ASYNC . # confirm 0 remaining (fixed or justified-suppressed) +uv run pytest -q +``` + +- [ ] **Step 7: `C90` (mccabe complexity) — 67 violations, review each, refactor or justify** + +```bash +uv run ruff check --select C901 --statistics . +``` + +For each of the 67 flagged functions: read it, and either (a) refactor to reduce branching — extract a +helper, replace a long if/elif chain with a lookup table/dispatch dict, early-return to flatten nesting — +or (b) if the complexity is inherent to the domain (the `pglast` AST walker in `sql/safety.py` is a +plausible candidate — a recursive node-type dispatcher is naturally branchy) add a scoped +`# noqa: C901` with a one-sentence justification. **Do not blanket-suppress this category** — each of the +67 needs an individual decision, logged in the commit message as a summary (how many refactored vs. +justified-suppressed). + +```bash +uv run ruff check --select C901 . # confirm 0 remaining +uv run mypy src/mcpg && uv run pytest -q +``` + +- [ ] **Step 8: `ANN` — skipped, per the rescoping note above.** `mypy --strict` already passes at 0 issues + across all 108 source files — this category's 475 violations check annotation presence redundantly with + what strict mode already independently guarantees. Not enabled, not fixed this pass; baseline count goes + in the Step 8-equivalent CHANGELOG entry (numbered Step 13 below) as a reported follow-up candidate. + +- [ ] **Step 9: `FBT` (flake8-boolean-trap) — 196 violations (103 in `tools.py`, 93 elsewhere) — fixed, + `tools.py` first** + +**Read this step's risk note before starting**: MCPg's tool-return dataclasses and tool-function signatures +are covered by `tests/contract/tool_surface.snapshot.json` — a frozen contract of all 254 exposed MCP +tools. Converting a boolean positional parameter to keyword-only on a function that's directly exposed as +an MCP tool changes that tool's generated JSON schema (the `mcp` SDK derives the schema from the function +signature). **Before fixing any `FBT001`/`FBT002` hit, check whether the flagged function is a registered +MCP tool** (search `tools.py` for where it's registered) — if it is, this is not a safe mechanical +lint fix, it's a tool-contract change requiring `MCPG_REGENERATE_TOOL_SNAPSHOT=1` regeneration and explicit +review of the resulting schema diff, per `CLAUDE.md`'s own source-of-truth map. + +```bash +uv run ruff check --select FBT --statistics . +``` + +For each hit: +1. Check if the function is a registered MCP tool (`grep -n "" src/mcpg/tools.py`). +2. **If yes**: treat as its own reviewed change — convert the boolean parameter to keyword-only + (`*, flag: bool = False`), then regenerate and diff the tool snapshot: + ```bash + MCPG_REGENERATE_TOOL_SNAPSHOT=1 uv run pytest tests/contract/test_tool_surface.py + git diff tests/contract/tool_surface.snapshot.json + ``` + Confirm the resulting schema diff is the expected, intentional shape (a parameter moving from + positional to keyword-only in the JSON schema) before committing it — do not regenerate and accept + blindly. +3. **If no** (an internal helper, not exposed as a tool): convert to keyword-only directly — + ```diff + - def helper(data, verbose): + + def helper(data, *, verbose): + ``` + and update every call site in the same commit (ruff's `--fix` does not do this one automatically for + `FBT002`; grep for every call site of the changed function and update each). + +Given the volume (196, split 103/93 as noted above) and the per-site tool-contract check required for the +`tools.py` subset, budget this as the largest single sub-task among the categories actually being fixed — +work through it function-by-function, `tools.py` first, committing in batches of ~15-20 fixed functions +rather than one giant commit, running the full check + test suite after each batch: + +```bash +uv run ruff check . && uv run ruff format --check . && uv run mypy src/mcpg && uv run pytest -q +``` + +- [ ] **Step 10: `D` — skipped, per the rescoping note above.** `src/mcpg/tools.py` (the actual public MCP + tool surface) has 0 `D103` violations already — the 2,284 hits are entirely internal-module documentation + debt with no functional consumer. Not enabled, not fixed this pass; the baseline count is reported in the + CHANGELOG (Step 13) as a candidate for a dedicated future documentation pass, not silently dropped. + +- [ ] **Step 11: Enable the assessed category list in `pyproject.toml`** + +```diff +- select = ["E", "F", "I", "B", "W", "N", "UP", "RUF"] ++ select = ["E", "F", "I", "B", "W", "N", "UP", "RUF", "C90", "ASYNC", "C4", "SIM", "PTH", "PYI", "FBT"] +``` + +`D`, `ANN`, `TC`, `PT` are deliberately **not** added — see the rescoping note at the top of this task. +`PYI` (16 pre-existing violations, `PYI034`/`PYI041`) gets fixed alongside whichever of Steps 1-9 touches +the same files, since the volume is small — confirm `uv run ruff check --select PYI .` reports 0 before +this step. + +- [ ] **Step 12: Final full-repo verification** + +```bash +uv run ruff check . +uv run ruff format --check . +uv run mypy src/mcpg +uv run pytest -q --cov +``` + +Expected: `ruff check .` reports zero violations across every category now in `select`; coverage still +clears `fail_under = 90`. + +- [ ] **Step 13: Update CHANGELOG.md** under `Changed`: + +```markdown +- Enabled Ruff's `C90`, `ASYNC`, `C4`, `SIM`, `PTH`, `PYI`, and `FBT` categories (previously only the + default-on categories were selected) and fixed every existing violation in them (~340 total, of which + 196 were `FBT` — 103 on the public `tools.py` MCP surface, prioritized first, with tool-snapshot-contract + review where applicable). `C901` complexity hotspots were individually refactored where cheap and safe, + or justified-suppressed where complexity is inherent to a security-critical algorithm (the SQL-safety + AST walker) — not blanket-suppressed. +- **Assessed but deliberately not enabled**, given no functional-bug or contract-safety benefit found to + justify the cost: `D` (2,284 pre-existing violations, 100% outside the public tool surface — `tools.py` + itself is already fully documented), `ANN` (475 — redundant with `mypy --strict`, which already passes + clean), `TC` (165), `PT` (75, test-only). Baseline counts recorded here as measured 2026-08-25, for + whoever picks up a documentation- or type-hygiene-focused pass later. +``` + +- [ ] **Step 14: Final commit for the select-list change itself** + +```bash +git add pyproject.toml CHANGELOG.md +git commit -m "lint: enable assessed ruff opt-in categories (C90, ASYNC, C4, SIM, PTH, PYI, FBT) + +Co-Authored-By: Claude Sonnet 5 " +``` + +--- + +### Task 24: `auto-merge-bot-prs.yml` actor-check review + +**Files:** +- Modify: `.github/workflows/auto-merge-bot-prs.yml` (comment only, unless investigation finds a real + issue) + +**Interfaces:** None. + +- [ ] **Step 1: Verify the `[bot]`-suffix namespace claim directly** rather than leaving it as an + unverified assumption in the shipped PR: + +```bash +gh api users/dependabot%5Bbot%5D 2>&1 | head -20 +``` + +Confirm the `type` field reads `"Bot"` — this is GitHub's own signal that `[bot]`-suffixed usernames are +reserved for App-created identities, not freely registrable by a human account. If this can't be confirmed +directly (no `gh` API access, or the field doesn't say what's expected), escalate to the maintainer rather +than shipping an unverified assumption either way. + +- [ ] **Step 2: Add a comment documenting the verified assumption** (not a logic change, since Step 1 is + expected to confirm the existing check is safe): + +```diff + if: | + github.actor == 'dependabot[bot]' || + github.actor == 'renovate[bot]' || + github.actor == 'github-actions[bot]' || ++ # The `[bot]`-suffix username namespace is reserved for GitHub App identities ++ # (confirmed via `gh api users/dependabot%5Bbot%5D` — `type: Bot`); a human account ++ # cannot register one, so this clause doesn't broaden trust beyond the three named ++ # bots above in practice. Re-verify this assumption if GitHub's account-namespace ++ # rules ever change. + endsWith(github.actor, '[bot]') +``` + +- [ ] **Step 3: Commit** + +```bash +git add .github/workflows/auto-merge-bot-prs.yml +git commit -m "docs(ci): document the verified [bot]-namespace assumption in auto-merge check + +Co-Authored-By: Claude Sonnet 5 " +``` + +--- + +### Deferred — consciously out of scope for this plan, with reasoning + +Self-review against both audit reports found five findings with no task above. Each is a real finding, not +dropped silently — reasoning for deferring each: + +- **Secrets rotation requires a process restart** (Security, Important #8, `secrets.py`). Fixing this + properly means a TTL-bounded cache refresh per cloud backend (vault/aws/gcp), each with different + SDK-specific semantics — a genuinely separate, focused piece of work, not a same-shape fix to fold into + an already-large plan. Flag as a follow-up plan of its own. +- **No structured-logging library detected** (Standards Compliance, Important #3). Superseded by this same + audit's Observability finding: `obs_logging.py` already implements a hand-rolled JSON formatter on stdlib + `logging`, which this skill's own reference doc treats as an acceptable production-tier alternative to + `structlog` — no code change indicated. +- **`bandit` has no baseline/diff-mode config** (Security, Minor). Low value to add before there's an + actual accepted-risk suppression to baseline against — revisit once one exists. +- **`pylock.toml` not exported** (Dependency & Supply Chain, Minor). Interop nicety for non-`uv` + installers; no current consumer needs it. +- **No correlation/request-ID threading through log lines** (Observability, Minor). Lower value given + MCPg's actual shape (synchronous, single-request-scoped tool calls, not a multi-hop request graph) — + this domain's own report already noted the same caveat. + +--- + +### Task 25: Final PR assembly + +- [ ] **Step 1: Re-run the full verification suite one last time** + +```bash +uv run ruff check . && uv run ruff format --check . && uv run mypy src/mcpg && uv run pytest -q --cov +``` + +- [ ] **Step 2: Review the full commit log for this branch** + +```bash +git log --oneline main..HEAD +``` + +- [ ] **Step 3: Write the PR description**, covering: summary of what changed and why (link back to the + two audit runs), the two breaking-change call-outs (auth fail-closed, rate-limit default) with migration + instructions, roadmap linkage (`N/A — internal audit remediation`), and the checklist items from + `.github/PULL_REQUEST_TEMPLATE.md`. + +- [ ] **Step 4: STOP — do not push or open the PR yet.** Confirm with the maintainer before this + outward-facing step, per this session's own working agreement (push/PR-open is a hard-to-reverse, + outward-facing action distinct from local commits). diff --git a/docs/user-guide.md b/docs/user-guide.md index b92d303f..7362b661 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -205,6 +205,13 @@ static ceiling. ### Streamable HTTP (Cursor, Continue, custom web clients, etc.) +The HTTP transport refuses to start unless it's authenticated — set +either `MCPG_HTTP_AUTH_TOKEN` (static bearer, below) or +`MCPG_AUTH_MODE=oidc` (next). If you deliberately need it unauthenticated +(e.g. behind a proxy that already enforces auth), set +`MCPG_HTTP_ALLOW_UNAUTHENTICATED=true` — this is loudly logged on every +startup and not recommended. + ```bash export MCPG_DATABASE_URL=postgresql://user:pass@localhost:5432/mydb export MCPG_TRANSPORT=streamable-http @@ -839,9 +846,13 @@ MCPg ships with defence-in-depth defaults: - **Per-session timeouts** (`MCPG_STATEMENT_TIMEOUT_MS`, default 30 s; `MCPG_LOCK_TIMEOUT_MS`, default 5 s) self-terminate runaway queries and lock waits. -- **HTTP authn.** Static bearer (`MCPG_HTTP_AUTH_TOKEN`) with - constant-time compare, or OIDC JWT validation against the - issuer's JWKS (asymmetric algorithms only). +- **HTTP authn enforced on startup.** The HTTP transport refuses to + start unless authenticated: static bearer (`MCPG_HTTP_AUTH_TOKEN`) + with constant-time compare, or OIDC JWT validation against the + issuer's JWKS (asymmetric algorithms only). Use + `MCPG_HTTP_ALLOW_UNAUTHENTICATED=true` to explicitly opt out + (loudly logged; not recommended). The default `stdio` transport is + unaffected. - **Audit redaction** as documented above. - **Graceful shutdown draining** (controlled by `MCPG_SHUTDOWN_DRAIN_SECONDS`, default 30 s) ensures the server lifespan exit drains all in-flight tool calls before releasing database connection pools. - **Multi-tenancy via `SET ROLE`.** One process can serve many diff --git a/pyproject.toml b/pyproject.toml index ecceb375..2ff5acd5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ description = "A production-grade PostgreSQL Model Context Protocol (MCP) server readme = "README.md" requires-python = ">=3.12" license = "MIT" -license-files = ["LICENSE", "src/mcpg/_vendor/LICENSE"] +license-files = ["LICENSE"] authors = [ {name = "Devopam Mittra", email = "devopam@gmail.com"}, ] @@ -64,6 +64,8 @@ dependencies = [ # JWT validation for the OIDC auth path. ``[crypto]`` pulls in # ``cryptography`` so RS256 / ES256 signature checks work. "pyjwt[crypto]>=2.8", + "circuitbreaker>=2.1.3", + "tenacity>=9.1.4", ] # Optional extras the runtime picks up only on demand. ``otel`` adds @@ -103,7 +105,7 @@ Security = "https://github.com/devopam/MCPg/blob/main/SECURITY.md" mcpg = "mcpg.__main__:main" [build-system] -requires = ["hatchling"] +requires = ["hatchling>=1.26"] build-backend = "hatchling.build" # Single source of truth for the version: hatchling reads ``__version__`` @@ -115,6 +117,7 @@ path = "src/mcpg/__init__.py" [tool.hatch.build.targets.wheel] packages = ["src/mcpg"] +include = ["src/mcpg/py.typed"] # Keep the sdist tight — ship the source tree + the prose a user # actually wants when they unpack from PyPI, and leave dev-only @@ -155,6 +158,12 @@ dev = [ "opentelemetry-api>=1.27", "opentelemetry-sdk>=1.27", "opentelemetry-exporter-otlp-proto-http>=1.27", + "pip-licenses>=5.5.5", + "pytest-mock>=3.15.1", + "pytest-randomly>=4.1.0", + "pytest-socket>=0.8.1", + "time-machine>=3.5.0", + "pytest-rerunfailures>=16.6", ] # Benchmark suite (roadmap 19) — dev-only. Keeps the shipped package lean; # `duckdb` generates the TPC-H dataset for benchmarks/datasets/load_tpch.py, @@ -202,7 +211,7 @@ extend-exclude = ["scratch", "demo.py"] force-exclude = true [tool.ruff.lint] -select = ["E", "F", "I", "B", "W", "N", "UP", "RUF"] +select = ["E", "F", "I", "B", "W", "N", "UP", "RUF", "C90", "ASYNC", "C4", "SIM", "PTH", "PYI", "FBT"] [tool.ruff.lint.isort] known-first-party = ["mcpg"] @@ -218,6 +227,16 @@ exclude = ["scratch/"] module = ["pglast", "pglast.*"] ignore_missing_imports = true +[[tool.mypy.overrides]] +# circuitbreaker (audit remediation Task 15) ships no py.typed marker and no +# stub package — `@circuit` therefore types as `Any`, which on its own would +# only need ignore_missing_imports. The decorated methods still need a +# `# type: ignore[untyped-decorator]` at each call site (see nl2sql.py / +# oidc.py) because --strict's disallow_untyped_decorators fires independently +# of import resolution. +module = ["circuitbreaker"] +ignore_missing_imports = true + [[tool.mypy.overrides]] # Optional cloud-secrets SDKs ship without typed stubs we want to lean # on — the providers in mcpg.secrets import them lazily and convert diff --git a/src/mcpg/advisors.py b/src/mcpg/advisors.py index fc517f85..d71124c4 100644 --- a/src/mcpg/advisors.py +++ b/src/mcpg/advisors.py @@ -23,6 +23,7 @@ from __future__ import annotations +import logging import re from dataclasses import dataclass from typing import Any @@ -30,6 +31,8 @@ from mcpg.query import QueryError, analyze_query_plan from mcpg.sql import SqlDriver +logger = logging.getLogger(__name__) + # Stable rule identifiers — agents may filter by these. RULE_MISSING_PRIMARY_KEY = "missing_primary_key" RULE_UNINDEXED_FOREIGN_KEY = "unindexed_foreign_key" @@ -238,7 +241,11 @@ async def _recommend_graph_indices(driver: SqlDriver, schema: str) -> list[Findi ] -async def _redundant_indexes(driver: SqlDriver, schema: str) -> list[Finding]: +# C901 rationale: pairwise index-prefix comparison algorithm (O(n^2) over a +# table's indexes, checking column-vector prefixes + partial-index predicate +# / expression matches) -- the branching is the redundancy-detection logic +# itself; splitting it up risks subtly changing which indexes get flagged. +async def _redundant_indexes(driver: SqlDriver, schema: str) -> list[Finding]: # noqa: C901 """Identify B-Tree indexes whose columns are a leading prefix of another index. Operators can drop prefix-redundant indexes to reclaim disk space and reduce @@ -710,7 +717,15 @@ class OptimizationResult: rationale: str -async def optimize_query(driver: SqlDriver, sql: str) -> OptimizationResult: +# C901 rationale: several independent regex-based anti-pattern detectors +# (SELECT *, missing LIMIT, IN-subquery, leading wildcard) whose boolean +# flags are computed once and then reused across both the findings list and +# the rewritten-SQL/rationale construction -- splitting the checks apart +# would still need to thread the same shared flags through, and this +# function's return shape is contract-pinned +# (tests/contract/tool_return_shapes.snapshot.json), so a restructuring +# carries real regression risk for a lint-only benefit. +async def optimize_query(driver: SqlDriver, sql: str) -> OptimizationResult: # noqa: C901 """Analyze a SQL query for anti-patterns and performance issues, returning an optimized version.""" findings = [] ex_summary = "" @@ -795,7 +810,7 @@ async def optimize_query(driver: SqlDriver, sql: str) -> OptimizationResult: "- Consider adding indexes on columns used in WHERE or JOIN clauses for tables with Seq Scan." ) except Exception: - pass + logger.debug("Skipping sequential-scan advisory line; plan inspection failed", exc_info=True) rationale = ( "\n".join(rationale_parts) diff --git a/src/mcpg/aio.py b/src/mcpg/aio.py index fa4a1d96..2afb2f4e 100644 --- a/src/mcpg/aio.py +++ b/src/mcpg/aio.py @@ -43,6 +43,7 @@ from dataclasses import dataclass, field +from mcpg.errors import MCPgError from mcpg.sql import SqlDriver # PG 19 ships the AIO subsystem. The version-num probe is the boundary @@ -73,7 +74,7 @@ _MIN_STATS_WINDOW_SECONDS = 300.0 -class AioError(Exception): +class AioError(MCPgError): """Raised when an AIO operation cannot complete.""" diff --git a/src/mcpg/audit.py b/src/mcpg/audit.py index 04bc563c..35820bbe 100644 --- a/src/mcpg/audit.py +++ b/src/mcpg/audit.py @@ -246,7 +246,7 @@ async def _get_version_and_db(driver: SqlDriver) -> tuple[str, str]: short_ver = ver.split(",")[0] if "," in ver else ver return short_ver, str(rows[0].cells["dbname"]) except Exception: - pass + audit_logger.debug("Version/dbname query failed; falling back to 'PostgreSQL Unknown'", exc_info=True) return "PostgreSQL Unknown", "unknown" @@ -449,7 +449,15 @@ async def audit_memory_io(driver: SqlDriver, health_score: dict[str, int]) -> Ca ) -async def audit_transactions_connections(driver: SqlDriver) -> CategoryResult: +# C901 rationale: 4 independent metric checks (rollback rate, connection +# saturation, XID wraparound age, prepared transactions), each its own +# query + try/except + threshold-based status classification, accumulating +# into a shared `category_score`. Extracting each into a helper is possible +# but this shape repeats across every `audit_*` function in this module +# (see audit_database, audit_sequences, audit_settings below) -- a proper +# fix is a module-wide "run checks, accumulate score" helper, out of scope +# for a per-function lint pass. +async def audit_transactions_connections(driver: SqlDriver) -> CategoryResult: # noqa: C901 """Analyze transaction health: rollback rate, wraparound, prepared transactions, and connection saturation.""" metrics: list[MetricResult] = [] category_score = 100 @@ -1469,19 +1477,9 @@ async def audit_database(driver: SqlDriver, schema: str, log_table: str | None = cat_vector = await audit_vector_indexes(driver) cat_rag_pipeline = await audit_rag_pipeline(driver) + optional_categories = [cat_sequences, cat_settings, cat_turboquant, cat_pg_search, cat_vector, cat_rag_pipeline] categories = [cat_mem, cat_tx, cat_lock, cat_bloat, cat_slow, cat_auth] - if cat_sequences is not None: - categories.append(cat_sequences) - if cat_settings is not None: - categories.append(cat_settings) - if cat_turboquant is not None: - categories.append(cat_turboquant) - if cat_pg_search is not None: - categories.append(cat_pg_search) - if cat_vector is not None: - categories.append(cat_vector) - if cat_rag_pipeline is not None: - categories.append(cat_rag_pipeline) + categories.extend(cat for cat in optional_categories if cat is not None) # 2. Dynamic scoring overall_score = round(sum(cat.score for cat in categories) / len(categories)) diff --git a/src/mcpg/audit_integrity.py b/src/mcpg/audit_integrity.py index a0479046..fdb1782c 100644 --- a/src/mcpg/audit_integrity.py +++ b/src/mcpg/audit_integrity.py @@ -29,7 +29,12 @@ _VERIFY_BATCH_SIZE = 1_000 -async def verify_audit_chain(driver: SqlDriver) -> dict[str, Any]: +# C901 rationale: security-sensitive HMAC signature-chain verification +# (tamper detection over keyset-paginated audit rows, including the +# tail-truncation check against a separately-read chain_tip). The branching +# is the verification logic itself; restructuring it is pure risk to a +# tamper-evidence guarantee for no lint benefit. +async def verify_audit_chain(driver: SqlDriver) -> dict[str, Any]: # noqa: C901 """Verify the integrity of the audit events signature chain. Reads the audit events sequentially (ordered by id), computes the HMAC @@ -40,10 +45,7 @@ async def verify_audit_chain(driver: SqlDriver) -> dict[str, Any]: A dict with 'status' (either 'ok' or 'tampered'), and details if tampered. """ settings = getattr(driver, "settings", None) - if settings is not None: - key_str = settings.audit_hmac_key or "" - else: - key_str = environ.get("MCPG_AUDIT_HMAC_KEY", "").strip() + key_str = settings.audit_hmac_key or "" if settings is not None else environ.get("MCPG_AUDIT_HMAC_KEY", "").strip() if not key_str: return { diff --git a/src/mcpg/audit_nl2sql.py b/src/mcpg/audit_nl2sql.py index 032171ba..48f39c1c 100644 --- a/src/mcpg/audit_nl2sql.py +++ b/src/mcpg/audit_nl2sql.py @@ -47,6 +47,7 @@ from os import environ from typing import Any, Literal +from mcpg.errors import MCPgError from mcpg.extensions import extension_installed from mcpg.sql import SqlDriver, obfuscate_password @@ -84,7 +85,7 @@ _NATIVE_WINDOW_DAYS = 7 -class NL2SQLAuditError(Exception): +class NL2SQLAuditError(MCPgError): """Raised when the NL→SQL audit subsystem can't satisfy a request.""" @@ -225,14 +226,19 @@ def _resolve_settings( compress_after = (source.get("MCPG_NL2SQL_AUDIT_COMPRESS_AFTER") or "").strip() or "7 days" _check_interval(compress_after, kind="MCPG_NL2SQL_AUDIT_COMPRESS_AFTER") rls_raw = (source.get("MCPG_NL2SQL_AUDIT_RLS") or "").strip().lower() - rls = True if rls_raw in ("", "true", "1", "yes", "on") else False + rls = rls_raw in ("", "true", "1", "yes", "on") reader_role = (source.get("MCPG_NL2SQL_AUDIT_READER_ROLE") or "").strip() or None if reader_role is not None: _check_identifier(reader_role, kind="reader role") return backend_raw, retention_days, chunk_interval, compress_after, rls, reader_role -async def ensure_nl2sql_audit_table( +# C901 rationale: idempotent DDL provisioning (double-checked-locking cache +# check, then schema/table/compression/retention/RLS setup branched by which +# of native-partitioned / pg_partman / TimescaleDB backend was detected) -- +# the branching is the backend-selection matrix itself, and the double-check +# lock is load-bearing concurrency-safety, not incidental complexity. +async def ensure_nl2sql_audit_table( # noqa: C901 driver: SqlDriver, *, env: Mapping[str, str] | None = None, @@ -357,7 +363,7 @@ async def ensure_nl2sql_audit_table( statements.append(sql_compress_policy) compression_enabled = True except Exception as exc: # pragma: no cover - depends on TSDB version - logger.warning("add_compression_policy raised, continuing: %s", exc) + logger.warning("add_compression_policy raised, continuing: %s", exc, exc_info=True) sql_retention = ( f"SELECT add_retention_policy('{_QUALIFIED}', INTERVAL '{retention_days} days', if_not_exists => TRUE)" @@ -366,7 +372,7 @@ async def ensure_nl2sql_audit_table( await driver.execute_query(sql_retention, force_readonly=False) statements.append(sql_retention) except Exception as exc: # pragma: no cover - logger.warning("add_retention_policy raised, continuing: %s", exc) + logger.warning("add_retention_policy raised, continuing: %s", exc, exc_info=True) elif backend == "pg_partman": # pg_partman is the partition manager; LZ4 TOAST compression # is the storage-level codec. Both are independent layers. diff --git a/src/mcpg/audit_trail.py b/src/mcpg/audit_trail.py index 1282b112..c73e032e 100644 --- a/src/mcpg/audit_trail.py +++ b/src/mcpg/audit_trail.py @@ -21,6 +21,7 @@ import hashlib import hmac import json +import logging from dataclasses import dataclass from datetime import UTC, datetime, timedelta from os import environ @@ -33,9 +34,12 @@ from mcpg.audit_nl2sql import _check_interval as _shared_check_interval from mcpg.audit_nl2sql import detect_backend as _detect_backend from mcpg.config import _parse_bool +from mcpg.errors import MCPgError from mcpg.extensions import extension_installed from mcpg.sql import SqlDriver, obfuscate_password +logger = logging.getLogger(__name__) + _AUDIT_LOCK: asyncio.Lock | None = None @@ -56,7 +60,7 @@ def _get_audit_lock() -> asyncio.Lock: _MASK = "****" -class AuditTrailError(Exception): +class AuditTrailError(MCPgError): """Raised when an audit-trail maintenance operation is rejected.""" @@ -249,7 +253,10 @@ async def record_audit( try: audit_integrity = _parse_bool("MCPG_AUDIT_INTEGRITY", raw) except Exception: - pass + logger.debug( + "Malformed MCPG_AUDIT_INTEGRITY env value; defaulting audit_integrity to False", + exc_info=True, + ) key_str = environ.get("MCPG_AUDIT_HMAC_KEY", "").strip() audit_hmac_key = key_str if key_str else None @@ -603,10 +610,7 @@ def _events_native_partition_sql(month_start: datetime) -> str: # Normalise to the first of the month. start = month_start.replace(day=1, hour=0, minute=0, second=0, microsecond=0) # Next month's first day. Avoid relativedelta to keep no deps. - if start.month == 12: - end = start.replace(year=start.year + 1, month=1) - else: - end = start.replace(month=start.month + 1) + end = start.replace(year=start.year + 1, month=1) if start.month == 12 else start.replace(month=start.month + 1) suffix = start.strftime("%Y%m") return ( f"CREATE TABLE IF NOT EXISTS {AUDIT_SCHEMA}.{AUDIT_TABLE}_p{suffix} " @@ -837,7 +841,7 @@ async def _events_migrate_timescaledb( statements.append(sql_compress_pol) compression_enabled = True except Exception: - pass + logger.debug("TimescaleDB add_compression_policy failed; continuing without compression", exc_info=True) if retention_days is not None: # HMAC chain anchors on the oldest event. Operator opt-in is @@ -850,7 +854,7 @@ async def _events_migrate_timescaledb( await driver.execute_query(sql_retention, force_readonly=False) statements.append(sql_retention) except Exception: - pass + logger.debug("TimescaleDB add_retention_policy failed; continuing without retention", exc_info=True) return row_count, compression_enabled, statements diff --git a/src/mcpg/cache.py b/src/mcpg/cache.py index 46986c32..aeb7d1aa 100644 --- a/src/mcpg/cache.py +++ b/src/mcpg/cache.py @@ -122,7 +122,8 @@ async def _init_client(self) -> None: except ImportError: logger.error( "Redis caching is configured (MCPG_REDIS_URL), but the 'redis' package is not installed. " - "Please run 'pip install redis' to enable Redis caching. Falling back to InMemoryCache." + "Please run 'pip install redis' to enable Redis caching. Falling back to InMemoryCache.", + exc_info=True, ) raise @@ -136,7 +137,7 @@ async def get(self, key: str) -> Any | None: return None return json.loads(raw) except Exception as e: - logger.warning(f"Error fetching from Redis cache for key {key!r}: {e}") + logger.warning(f"Error fetching from Redis cache for key {key!r}: {e}", exc_info=True) return None async def set(self, key: str, value: Any, ttl: int) -> None: @@ -147,7 +148,7 @@ async def set(self, key: str, value: Any, ttl: int) -> None: raw = json.dumps(value) await self._client.set(f"{self._prefix}{key}", raw, ex=ttl) except Exception as e: - logger.warning(f"Error setting Redis cache for key {key!r}: {e}") + logger.warning(f"Error setting Redis cache for key {key!r}: {e}", exc_info=True) async def clear(self) -> None: await self._init_client() @@ -160,14 +161,14 @@ async def clear(self) -> None: async for key in self._client.scan_iter(f"{self._prefix}*"): await self._client.delete(key) except Exception as e: - logger.warning(f"Error clearing Redis cache: {e}") + logger.warning(f"Error clearing Redis cache: {e}", exc_info=True) async def close(self) -> None: if self._client: try: await self._client.close() except Exception as e: - logger.warning(f"Error closing Redis client: {e}") + logger.warning(f"Error closing Redis client: {e}", exc_info=True) finally: self._client = None self._initialized = False @@ -178,6 +179,7 @@ class CacheManager: def __init__( self, + *, enabled: bool = True, ttl_seconds: int = 300, maxsize: int = 1024, @@ -205,7 +207,7 @@ async def start(self) -> None: except ImportError: pass except Exception as e: - logger.warning(f"Redis cache initialization failed: {e}. Falling back to InMemoryCache.") + logger.warning(f"Redis cache initialization failed: {e}. Falling back to InMemoryCache.", exc_info=True) self._driver = InMemoryCache(maxsize=self._maxsize) logger.info("InMemoryCache backend initialized successfully.") diff --git a/src/mcpg/composite.py b/src/mcpg/composite.py index 9f8bdf53..53ae8dfd 100644 --- a/src/mcpg/composite.py +++ b/src/mcpg/composite.py @@ -24,6 +24,7 @@ from dataclasses import dataclass, field from typing import Any +from mcpg.errors import MCPgError from mcpg.introspection import ( ColumnInfo, ConstraintInfo, @@ -41,7 +42,7 @@ _IDENTIFIER = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\Z") -class CompositeError(Exception): +class CompositeError(MCPgError): """Raised when a composite tool's inputs are invalid.""" diff --git a/src/mcpg/config.py b/src/mcpg/config.py index a22f11af..a84f27f8 100644 --- a/src/mcpg/config.py +++ b/src/mcpg/config.py @@ -12,9 +12,10 @@ from dataclasses import dataclass from enum import StrEnum from os import environ -from os.path import isabs +from pathlib import Path from urllib.parse import urlparse +from mcpg.errors import MCPgError from mcpg.nl2sql import AUTO_PICK_ORDER, VENDOR_ENV_VAR_HINT, VENDOR_KEY_ENV_VARS from mcpg.secrets import SecretsError, build_secrets_provider from mcpg.sql import obfuscate_password @@ -35,7 +36,7 @@ _FALSE_VALUES = frozenset({"false", "0", "no", "off"}) -class ConfigError(Exception): +class ConfigError(MCPgError): """Raised when the environment configuration is missing or invalid.""" @@ -112,6 +113,12 @@ class Settings: # When unset, the HTTP transport runs without auth (current # behaviour). stdio is never gated. http_auth_token: str | None = None + # When True, explicitly opts out of the fail-closed startup check in + # ``build_http_app`` that otherwise raises ``ConfigError`` when the + # HTTP transport would start with neither ``http_auth_token`` nor + # ``auth_mode == "oidc"`` configured. Setting this is a deliberate, + # loudly-logged choice — not the default. + http_allow_unauthenticated: bool = False # HTTP transport authentication mode. ``static`` (the default) does # constant-time comparison against ``http_auth_token``. ``oidc`` # validates the bearer JWT against the configured OIDC provider's @@ -189,7 +196,7 @@ class Settings: nl2sql_audit_compress_after: str = "7 days" nl2sql_audit_rls: bool = True nl2sql_audit_reader_role: str | None = None - rate_limit_enabled: bool = False + rate_limit_enabled: bool = True rate_limit_max_requests: int = 60 rate_limit_window_seconds: int = 60 rate_limit_heavy_max: int = 5 @@ -254,7 +261,15 @@ class Settings: dynamic_session_intent: bool = False http_max_body_bytes: int = 1048576 http_allowed_origins: tuple[str, ...] = () - http_hsts_max_age: int = 31536000 + http_hsts_max_age: int = 63072000 + # Host-header validation (Starlette's TrustedHostMiddleware) for the + # HTTP transports. Empty (default) = no host-header gate (current + # behaviour) — matches the http_allowed_origins/CORS convention. + # When set, each entry is an allowed host (wildcards like + # ``*.example.com`` are supported by TrustedHostMiddleware itself); + # requests with a Host header that doesn't match get a 400. + # ``MCPG_HTTP_TRUSTED_HOSTS=`` comma-separated. + http_trusted_hosts: tuple[str, ...] = () # IP allowlist for the HTTP transports. Empty (default) = no # network-level gate (current behaviour). When set, each entry is # an IP address or CIDR range that the client's connecting IP @@ -365,6 +380,7 @@ def __repr__(self) -> str: f"http_allowed_origins={self.http_allowed_origins!r}, " f"http_ip_allowlist={self.http_ip_allowlist!r}, " f"http_hsts_max_age={self.http_hsts_max_age}, " + f"http_trusted_hosts={self.http_trusted_hosts!r}, " f"http_tls_certfile={self.http_tls_certfile!r}, " f"http_tls_keyfile={self.http_tls_keyfile!r}, " f"http_tls_ca_certs={self.http_tls_ca_certs!r}, " @@ -475,7 +491,13 @@ def _parse_positive_int(var: str, raw: str) -> int: return value -def load_settings(env: Mapping[str, str] | None = None) -> Settings: +# C901 rationale: reads every `MCPG_*` environment variable into `Settings` +# (complexity 178 -- by far the largest in the repo). Every branch is one +# independent `if env.get("MCPG_X")` for one config knob; this is the +# process-wide config-loading entrypoint every module reads (see CLAUDE.md's +# source-of-truth map), so restructuring it is a rewrite of the thing the +# whole app depends on for a lint-only benefit, not a cheap win. +def load_settings(env: Mapping[str, str] | None = None) -> Settings: # noqa: C901 """Build :class:`Settings` from environment variables. Args: @@ -545,7 +567,7 @@ def load_settings(env: Mapping[str, str] | None = None) -> Settings: if (raw := env.get("MCPG_SUBPROCESS_BIN_ALLOWLIST")) is not None: dirs = tuple(d.strip() for d in raw.split(",") if d.strip()) for d in dirs: - if not isabs(d): + if not Path(d).is_absolute(): raise ConfigError(f"MCPG_SUBPROCESS_BIN_ALLOWLIST entries must be absolute paths (got {d!r})") subprocess_bin_allowlist = dirs @@ -679,6 +701,10 @@ def load_settings(env: Mapping[str, str] | None = None) -> Settings: raise ConfigError("MCPG_HTTP_AUTH_TOKEN must not be blank when set") http_auth_token = stripped + http_allow_unauthenticated = False + if (raw := secrets.get("MCPG_HTTP_ALLOW_UNAUTHENTICATED")) is not None: + http_allow_unauthenticated = _parse_bool("MCPG_HTTP_ALLOW_UNAUTHENTICATED", raw) + auth_mode = "static" if (raw := env.get("MCPG_AUTH_MODE")) is not None: candidate = raw.strip().lower() @@ -995,7 +1021,7 @@ def load_settings(env: Mapping[str, str] | None = None) -> Settings: raise ConfigError("MCPG_NL2SQL_AUDIT_READER_ROLE must not be blank when set") nl2sql_audit_reader_role = stripped - rate_limit_enabled = False + rate_limit_enabled = True if (raw := env.get("MCPG_RATE_LIMIT_ENABLED")) is not None: rate_limit_enabled = _parse_bool("MCPG_RATE_LIMIT_ENABLED", raw) @@ -1104,7 +1130,7 @@ def _positive_int(var: str, default: int) -> int: parts = [piece.strip() for piece in raw.split(pathsep) if piece.strip()] for part in parts: - if not isabs(part): + if not Path(part).is_absolute(): raise ConfigError(f"MCPG_MIGRATION_SCRIPTS_ROOTS entries must be absolute paths (got {part!r})") migration_scripts_roots = tuple(parts) @@ -1158,7 +1184,7 @@ def _positive_int(var: str, default: int) -> int: raise ConfigError(f"MCPG_HTTP_IP_ALLOWLIST entry is not a valid IP / CIDR (got {part!r})") from exc http_ip_allowlist = tuple(parts) - http_hsts_max_age = 31536000 + http_hsts_max_age = 63072000 if (raw := env.get("MCPG_HTTP_HSTS_MAX_AGE")) is not None: try: val = int(raw) @@ -1168,6 +1194,10 @@ def _positive_int(var: str, default: int) -> int: except ValueError: raise ConfigError(f"MCPG_HTTP_HSTS_MAX_AGE must be a non-negative integer (got {raw!r})") from None + http_trusted_hosts: tuple[str, ...] = () + if (raw := env.get("MCPG_HTTP_TRUSTED_HOSTS")) is not None: + http_trusted_hosts = tuple(h.strip() for h in raw.split(",") if h.strip()) + # HTTP TLS / mTLS. Parsed together so we can enforce the # invariant "if either cert or key is set, the other must be too" # at boot — uvicorn doesn't validate this until startup, by which @@ -1193,11 +1223,8 @@ def _positive_int(var: str, default: int) -> int: ("MCPG_HTTP_TLS_KEYFILE", http_tls_keyfile), ("MCPG_HTTP_TLS_CA_CERTS", http_tls_ca_certs), ): - if path is not None: - from os.path import isfile - - if not isfile(path): - raise ConfigError(f"{env_var} points to a non-existent file: {path!r}") + if path is not None and not Path(path).is_file(): + raise ConfigError(f"{env_var} points to a non-existent file: {path!r}") http_request_timeout_seconds = 0 if (raw := env.get("MCPG_HTTP_REQUEST_TIMEOUT_SECONDS")) is not None: @@ -1273,6 +1300,7 @@ def _positive_int(var: str, default: int) -> int: pool_min_size=pool_min_size, pool_max_size=pool_max_size, http_auth_token=http_auth_token, + http_allow_unauthenticated=http_allow_unauthenticated, auth_mode=auth_mode, oidc_issuer=oidc_issuer, oidc_audience=oidc_audience, @@ -1320,6 +1348,7 @@ def _positive_int(var: str, default: int) -> int: http_allowed_origins=http_allowed_origins, http_ip_allowlist=http_ip_allowlist, http_hsts_max_age=http_hsts_max_age, + http_trusted_hosts=http_trusted_hosts, http_tls_certfile=http_tls_certfile, http_tls_keyfile=http_tls_keyfile, http_tls_ca_certs=http_tls_ca_certs, diff --git a/src/mcpg/config_advisor.py b/src/mcpg/config_advisor.py index 6e58bd76..8336b71f 100644 --- a/src/mcpg/config_advisor.py +++ b/src/mcpg/config_advisor.py @@ -42,6 +42,7 @@ from dataclasses import dataclass, field +from mcpg.errors import MCPgError from mcpg.sql import SqlDriver # Status codes shared across the audit tools — same vocabulary as @@ -55,7 +56,7 @@ _VALID_STORAGE = frozenset({"ssd", "hdd", "san"}) -class ConfigAdvisorError(Exception): +class ConfigAdvisorError(MCPgError): """Raised when a config-advisor argument fails validation.""" @@ -112,7 +113,12 @@ async def _has_pg_sequences(driver: SqlDriver) -> bool: return bool(rows and rows[0].cells.get("present")) -async def audit_sequences( +# C901 rationale: direction-aware sequence-exhaustion math (ascending vs. +# descending sequences consume toward opposite bounds -- see the inline +# "gemini review on #181" note) plus threshold classification and a +# multi-branch summary message; the branching is the correctness-sensitive +# exhaustion-detection logic itself. +async def audit_sequences( # noqa: C901 driver: SqlDriver, *, warning_pct: float = 80.0, @@ -306,7 +312,11 @@ class SettingsAuditResult: _MB = 1024 * 1024 -async def audit_settings( +# C901 rationale: a long series of independent `pg_settings` sanity checks +# (fsync, full_page_writes, autovacuum, RAM-relative ratios, ...), each its +# own `if` appending a `SettingFinding` -- the branching is one rule per +# check, same shape as the `audit_*` functions in mcpg/audit.py. +async def audit_settings( # noqa: C901 driver: SqlDriver, *, total_ram_mb: int | None = None, diff --git a/src/mcpg/cron.py b/src/mcpg/cron.py index 15be8776..f09004f4 100644 --- a/src/mcpg/cron.py +++ b/src/mcpg/cron.py @@ -12,11 +12,12 @@ import re from dataclasses import dataclass +from mcpg.errors import MCPgError from mcpg.extensions import extension_installed from mcpg.sql import SqlDriver -class CronError(Exception): +class CronError(MCPgError): """Raised when a pg_cron operation cannot complete.""" diff --git a/src/mcpg/cursors.py b/src/mcpg/cursors.py index 8f281a01..0baaa39b 100644 --- a/src/mcpg/cursors.py +++ b/src/mcpg/cursors.py @@ -23,11 +23,12 @@ import time from collections.abc import AsyncIterator from dataclasses import dataclass, field -from typing import Any +from typing import Any, Self import psycopg from psycopg.rows import dict_row +from mcpg.errors import MCPgError from mcpg.sql import SafeSqlDriver, SqlDriver logger = logging.getLogger(__name__) @@ -38,7 +39,7 @@ HARD_FETCH_BATCH = 10_000 -class CursorError(Exception): +class CursorError(MCPgError): """Raised when a cursor operation fails or is rejected.""" @@ -280,7 +281,7 @@ async def _close_internal(self, cursor_id: str) -> bool: try: await active.connection.close() except Exception as exc: - logger.warning("Error closing cursor %s connection: %s", cursor_id, exc) + logger.warning("Error closing cursor %s connection: %s", cursor_id, exc, exc_info=True) return True async def _sweep_expired(self) -> None: @@ -290,7 +291,7 @@ async def _sweep_expired(self) -> None: for cid in expired: await self._close_internal(cid) - async def __aenter__(self) -> CursorManager: + async def __aenter__(self) -> Self: return self async def __aexit__(self, *exc_info: object) -> None: diff --git a/src/mcpg/data_movement.py b/src/mcpg/data_movement.py index ced6fb8a..3d864a68 100644 --- a/src/mcpg/data_movement.py +++ b/src/mcpg/data_movement.py @@ -29,6 +29,7 @@ from urllib.parse import unquote, urlparse from mcpg.database import Database +from mcpg.errors import MCPgError from mcpg.query import QueryError, run_select from mcpg.shell import ShellError, SubprocessLimits, run_pg_binary from mcpg.sql import SqlDriver @@ -44,7 +45,7 @@ DEFAULT_EXPORT_LIMIT = 10_000 -class ExportError(Exception): +class ExportError(MCPgError): """Raised when an export call is rejected or fails.""" @@ -210,7 +211,13 @@ def _libpq_env_from_url(database_url: str) -> dict[str, str]: _PG_DUMP_FORMATS = frozenset({"plain", "custom", "directory", "tar"}) -async def dump_database( +# C901 rationale: pg_dump argv construction (format validation, schema-name +# identifier validation, credential-via-env-not-argv handling) plus +# format-dependent output decoding -- the validation branches are exactly +# what keeps this shell-out safe (identifiers checked before reaching argv, +# unsupported formats rejected before spawn); splitting them apart doesn't +# reduce risk, just relocates it. +async def dump_database( # noqa: C901 database_url: str, *, timeout_sec: int, @@ -578,7 +585,7 @@ def _tail(buf: bytes, *, max_bytes: int = 4096) -> str: # --- bulk imports (in-process; gated behind WRITE) ----------------------- -class ImportDataError(Exception): +class ImportDataError(MCPgError): """Raised when an import call is rejected or fails. Named ``ImportDataError`` so it does not shadow the builtin diff --git a/src/mcpg/database.py b/src/mcpg/database.py index de3b49a4..6b2141ba 100644 --- a/src/mcpg/database.py +++ b/src/mcpg/database.py @@ -10,9 +10,10 @@ import logging from collections.abc import Sequence from types import TracebackType -from typing import Any +from typing import Any, Self from mcpg.config import Settings +from mcpg.errors import MCPgError from mcpg.multidb import PRIMARY_DATABASE_ID, make_read_only_driver, resolve_primary_id from mcpg.replicas import ReplicaPool, RoutedSqlDriver, _make_driver_for_pool from mcpg.sql import DbConnPool, SqlDriver, obfuscate_password @@ -20,7 +21,7 @@ logger = logging.getLogger(__name__) -class DatabaseError(Exception): +class DatabaseError(MCPgError): """Raised when the database cannot be connected to or used.""" @@ -130,7 +131,7 @@ async def close(self) -> None: try: await pool.close() except Exception as exc: - logger.warning("Error closing secondary database %r pool: %s", name, exc) + logger.warning("Error closing secondary database %r pool: %s", name, exc, exc_info=True) await self._pool.close() self._connected = False @@ -315,7 +316,7 @@ async def run_unmanaged(self, sql: str) -> None: finally: await connection.set_autocommit(False) - async def __aenter__(self) -> Database: + async def __aenter__(self) -> Self: await self.connect() return self diff --git a/src/mcpg/ddl_dryrun.py b/src/mcpg/ddl_dryrun.py index 68175e2f..1963b975 100644 --- a/src/mcpg/ddl_dryrun.py +++ b/src/mcpg/ddl_dryrun.py @@ -110,7 +110,13 @@ async def _default_acquire(database: Database) -> AsyncIterator[Any]: yield conn -async def dry_run_ddl( +# C901 rationale: always-rolled-back DDL dry-run with SQLSTATE-specific +# lock-timeout detection (55P03), WAL-delta measurement, and pg_locks +# introspection -- the branching is the correctness contract itself ("this +# never raises out", always rolls back); splitting it up risks a code path +# that forgets to roll back or misclassifies a lock-timeout as a generic +# error. +async def dry_run_ddl( # noqa: C901 database: Database, ddl_sql: str, *, diff --git a/src/mcpg/demo.py b/src/mcpg/demo.py index f8412553..07393f40 100644 --- a/src/mcpg/demo.py +++ b/src/mcpg/demo.py @@ -40,6 +40,8 @@ from psycopg import sql from psycopg.rows import TupleRow +from mcpg.errors import MCPgError + DEMO_SCHEMA = "mcpg_demo" # Stamped as the schema comment on seed; ``drop_demo`` refuses to drop a @@ -60,7 +62,7 @@ _REVIEW_COUNT = 900 -class DemoError(Exception): +class DemoError(MCPgError): """Raised when seeding or dropping the demo schema cannot proceed.""" diff --git a/src/mcpg/diagrams.py b/src/mcpg/diagrams.py index e935e58b..18f83f7d 100644 --- a/src/mcpg/diagrams.py +++ b/src/mcpg/diagrams.py @@ -11,6 +11,8 @@ import re from mcpg.introspection import ( + ForeignKeyInfo, + TableInfo, describe_table, list_constraints, list_foreign_keys, @@ -41,6 +43,45 @@ def _parse_pk_columns(definition: str) -> set[str]: return {column.strip().strip('"') for column in match.group(1).split(",")} +async def _render_entity_block( + driver: SqlDriver, schema: str, table: TableInfo, fk_columns_by_table: dict[str, set[str]] +) -> list[str]: + """Render one table's ``erDiagram`` entity block (columns, PK/FK tags).""" + columns = await describe_table(driver, schema, table.name) + constraints = await list_constraints(driver, schema, table.name) + pk_columns: set[str] = set() + for constraint in constraints: + if constraint.type == "primary_key": + pk_columns |= _parse_pk_columns(constraint.definition) + + fk_columns = fk_columns_by_table.get(table.name, set()) + + lines = [f" {_sanitize(table.name)} {{"] + for column in columns: + attrs: list[str] = [] + if column.name in pk_columns: + attrs.append("PK") + if column.name in fk_columns: + attrs.append("FK") + suffix = f" {' '.join(attrs)}" if attrs else "" + lines.append(f" {_sanitize(column.data_type)} {_sanitize(column.name)}{suffix}") + lines.append(" }") + return lines + + +def _render_relationship_lines(foreign_keys: list[ForeignKeyInfo], entity_names: set[str]) -> list[str]: + """Render the ``erDiagram`` relationship line for each intra-schema FK.""" + lines: list[str] = [] + for fk in foreign_keys: + if fk.from_table not in entity_names or fk.to_table not in entity_names: + # Cross-schema FK or pointing to a filtered-out table — skip the + # edge rather than emit a dangling reference. + continue + label = ",".join(fk.from_columns) + lines.append(f' {_sanitize(fk.to_table)} ||--o{{ {_sanitize(fk.from_table)} : "{label}"') + return lines + + async def generate_schema_diagram(driver: SqlDriver, schema: str, *, include_partitions: bool = False) -> str: """Render a Mermaid ER diagram for the tables in ``schema``. @@ -60,33 +101,9 @@ async def generate_schema_diagram(driver: SqlDriver, schema: str, *, include_par lines = ["erDiagram"] for table in tables: - columns = await describe_table(driver, schema, table.name) - constraints = await list_constraints(driver, schema, table.name) - pk_columns: set[str] = set() - for constraint in constraints: - if constraint.type == "primary_key": - pk_columns |= _parse_pk_columns(constraint.definition) - - fk_columns = fk_columns_by_table.get(table.name, set()) - - lines.append(f" {_sanitize(table.name)} {{") - for column in columns: - attrs: list[str] = [] - if column.name in pk_columns: - attrs.append("PK") - if column.name in fk_columns: - attrs.append("FK") - suffix = f" {' '.join(attrs)}" if attrs else "" - lines.append(f" {_sanitize(column.data_type)} {_sanitize(column.name)}{suffix}") - lines.append(" }") + lines.extend(await _render_entity_block(driver, schema, table, fk_columns_by_table)) - for fk in foreign_keys: - if fk.from_table not in entity_names or fk.to_table not in entity_names: - # Cross-schema FK or pointing to a filtered-out table — skip the - # edge rather than emit a dangling reference. - continue - label = ",".join(fk.from_columns) - lines.append(f' {_sanitize(fk.to_table)} ||--o{{ {_sanitize(fk.from_table)} : "{label}"') + lines.extend(_render_relationship_lines(foreign_keys, entity_names)) return "\n".join(lines) + "\n" diff --git a/src/mcpg/diesel.py b/src/mcpg/diesel.py index 079ed573..d0330b2b 100644 --- a/src/mcpg/diesel.py +++ b/src/mcpg/diesel.py @@ -21,8 +21,11 @@ import re +from mcpg.errors import MCPgError from mcpg.introspection import ( ColumnInfo, + ForeignKeyInfo, + TableInfo, describe_table, list_constraints, list_enums, @@ -34,7 +37,7 @@ _IDENTIFIER = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\Z") -class DieselExportError(Exception): +class DieselExportError(MCPgError): """Raised when a Diesel export call is rejected or fails.""" @@ -186,6 +189,37 @@ def _pascal(name: str) -> str: return "".join(p[:1].upper() + p[1:] for p in parts if p) or name +async def _build_table_blocks( + driver: SqlDriver, schema: str, tables: list[TableInfo], enum_names: set[str] +) -> list[str]: + """Render one ``table! { ... }`` block per table (step 2 of the export).""" + blocks: list[str] = [] + for table in tables: + columns = await describe_table(driver, schema, table.name) + for col in columns: + _check_identifier(col.name, "column") + constraints = await list_constraints(driver, schema, table.name) + pk_columns: list[str] = [] + for con in constraints: + if con.type == "primary_key": + pk_columns = _parse_pk_columns(con.definition) + break + blocks.append(_render_table_block(table.name, columns, pk_columns, enum_names)) + return blocks + + +def _build_joinable_lines(fks_all: list[ForeignKeyInfo], entity_names: set[str]) -> list[str]: + """Emit ``joinable!`` for every single-column intra-schema FK (step 3).""" + joinable_lines: list[str] = [] + for fk in fks_all: + if fk.to_table not in entity_names: + continue # cross-schema FK — Diesel's joinable! can't span schemas cleanly + if len(fk.from_columns) != 1: + continue # composite FKs are a documented v1 gap + joinable_lines.append(_render_joinable(fk.from_table, fk.to_table, fk.from_columns[0])) + return joinable_lines + + async def generate_diesel_schema(driver: SqlDriver, schema: str) -> str: """Emit a Diesel ORM ``schema.rs`` for ``schema``. @@ -217,26 +251,10 @@ async def generate_diesel_schema(driver: SqlDriver, schema: str) -> str: blocks.append(enum_module) # 2. One table! macro per table. - for table in tables: - columns = await describe_table(driver, schema, table.name) - for col in columns: - _check_identifier(col.name, "column") - constraints = await list_constraints(driver, schema, table.name) - pk_columns: list[str] = [] - for con in constraints: - if con.type == "primary_key": - pk_columns = _parse_pk_columns(con.definition) - break - blocks.append(_render_table_block(table.name, columns, pk_columns, enum_names)) + blocks.extend(await _build_table_blocks(driver, schema, tables, enum_names)) # 3. joinable! for every single-column intra-schema FK. - joinable_lines: list[str] = [] - for fk in fks_all: - if fk.to_table not in entity_names: - continue # cross-schema FK — Diesel's joinable! can't span schemas cleanly - if len(fk.from_columns) != 1: - continue # composite FKs are a documented v1 gap - joinable_lines.append(_render_joinable(fk.from_table, fk.to_table, fk.from_columns[0])) + joinable_lines = _build_joinable_lines(fks_all, entity_names) if joinable_lines: blocks.append("\n".join(joinable_lines)) diff --git a/src/mcpg/drizzle.py b/src/mcpg/drizzle.py index 9b5999c3..05f3e579 100644 --- a/src/mcpg/drizzle.py +++ b/src/mcpg/drizzle.py @@ -17,6 +17,7 @@ import re from collections.abc import Iterable +from mcpg.errors import MCPgError from mcpg.introspection import ( ColumnInfo, ForeignKeyInfo, @@ -33,7 +34,7 @@ _IDENTIFIER = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\Z") -class DrizzleError(Exception): +class DrizzleError(MCPgError): """Raised when a Drizzle export call is rejected or fails.""" @@ -288,7 +289,13 @@ def _collect_used_helpers(body: str) -> set[str]: } -async def generate_drizzle_schema(driver: SqlDriver, schema: str) -> str: +# C901 rationale: schema-generator in the same family as diesel.py / +# sqlc.py (per-table constraint/index/type mapping to Drizzle's TS DSL), but +# at complexity 21 -- constraint classification (PK / single-unique / +# composite-unique) feeds both the per-column rendering and the +# __table_args__-equivalent output, so extracting sections cleanly needs +# more restructuring than the cheap wins already taken in diesel.py/sqlc.py. +async def generate_drizzle_schema(driver: SqlDriver, schema: str) -> str: # noqa: C901 """Emit a Drizzle ORM TypeScript schema for ``schema``. Returns a string with the import line, every ``pgEnum`` declaration, diff --git a/src/mcpg/dynamic_session_intent.py b/src/mcpg/dynamic_session_intent.py index 79d7dc7f..bedda760 100644 --- a/src/mcpg/dynamic_session_intent.py +++ b/src/mcpg/dynamic_session_intent.py @@ -51,10 +51,11 @@ from mcp.server.context import CallNext, ServerRequestContext from mcpg.about import BUCKET_IDS +from mcpg.errors import MCPgError from mcpg.session_intent import _TOOL_NAME_PRESETS, INTENT_PRESETS, resolve_intent, resolved_tool_names -class DynamicIntentError(ValueError): +class DynamicIntentError(MCPgError, ValueError): """Raised when :func:`enable_intent` is given an unrecognized name.""" diff --git a/src/mcpg/ecto.py b/src/mcpg/ecto.py index 013f0513..f1d3dd00 100644 --- a/src/mcpg/ecto.py +++ b/src/mcpg/ecto.py @@ -20,6 +20,7 @@ import re +from mcpg.errors import MCPgError from mcpg.introspection import ( ColumnInfo, ForeignKeyInfo, @@ -34,7 +35,7 @@ _IDENTIFIER = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\Z") -class EctoExportError(Exception): +class EctoExportError(MCPgError): """Raised when an Ecto export call is rejected or fails.""" diff --git a/src/mcpg/ent.py b/src/mcpg/ent.py index 8459a25f..2b8592f0 100644 --- a/src/mcpg/ent.py +++ b/src/mcpg/ent.py @@ -17,6 +17,7 @@ import re +from mcpg.errors import MCPgError from mcpg.introspection import ( ColumnInfo, ForeignKeyInfo, @@ -30,7 +31,7 @@ _IDENTIFIER = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\Z") -class EntExportError(Exception): +class EntExportError(MCPgError): """Raised when an Ent export call is rejected or fails.""" diff --git a/src/mcpg/errors.py b/src/mcpg/errors.py new file mode 100644 index 00000000..aba63e2f --- /dev/null +++ b/src/mcpg/errors.py @@ -0,0 +1,26 @@ +"""The common ancestor for every MCPg-raised exception. + +Every domain-specific error class in this package (``ConfigError``, +``DatabaseError``, ``CursorError``, and the rest) subclasses this instead of +``Exception`` directly, so calling code that wants to catch "any error MCPg's +own logic raised" — as distinct from an unexpected bug surfacing from a +dependency — has one type to catch instead of an enumerated list kept in sync +by hand. + +This is a pure marker base: it adds no behavior, no new attributes, and no +change to any existing exception's message format or call sites. Catching a +specific subclass (``except ConfigError:``) behaves exactly as it did before; +``except MCPgError:`` is the new capability this adds. +""" + +from __future__ import annotations + + +class MCPgError(Exception): + """Base class for every exception MCPg's own logic raises. + + Not raised directly — always through one of its domain-specific + subclasses (``ConfigError``, ``DatabaseError``, etc.). Catch this + directly only when the intent is genuinely "any MCPg-internal error," + not a specific failure mode. + """ diff --git a/src/mcpg/extensions.py b/src/mcpg/extensions.py index 5f864bfc..764cd9ac 100644 --- a/src/mcpg/extensions.py +++ b/src/mcpg/extensions.py @@ -10,6 +10,7 @@ from dataclasses import dataclass +from mcpg.errors import MCPgError from mcpg.sql import SqlDriver # Extensions MCPg will enable on request: well-known, widely-used extensions. @@ -48,7 +49,7 @@ ) -class ExtensionError(Exception): +class ExtensionError(MCPgError): """Raised when an extension cannot be enabled.""" diff --git a/src/mcpg/graph.py b/src/mcpg/graph.py index 95d44775..55ae1786 100644 --- a/src/mcpg/graph.py +++ b/src/mcpg/graph.py @@ -13,9 +13,11 @@ from mcpg.context import AppContext from mcpg.database import DatabaseError +from mcpg.errors import MCPgError +from mcpg.policy import Capability, check_permission -class GraphError(Exception): +class GraphError(MCPgError): """Raised on invalid input or invariant violations across the Apache AGE graph integration. @@ -29,6 +31,29 @@ class GraphError(Exception): """ +# Catalog-derived identifiers (label/table names read back from +# ``ag_catalog.ag_label``) are NOT validated by Postgres/AGE — a quoted +# identifier can contain arbitrary characters, including embedded double +# quotes. Every such name is checked against this pattern immediately +# before it is interpolated into a SQL string, mirroring +# ``graph_projection._check_identifier``. +_LABEL_IDENTIFIER = re.compile(r"\A[A-Za-z_][A-Za-z0-9_]*\Z") + + +def _check_label_identifier(name: str) -> None: + """Raise :class:`GraphError` if ``name`` is not a plain SQL identifier. + + Guards every catalog-derived label/table name (from ``ag_label``) + before it is interpolated into an f-string SQL query. Aborts the + whole call rather than silently skipping the offending label — same + precedent as ``graph_projection.generate_graph_projection``, which + raises on the first invalid catalog-derived table name instead of + dropping it and continuing. + """ + if not _LABEL_IDENTIFIER.match(name): + raise GraphError(f"invalid label name {name!r}; must match [A-Za-z_][A-Za-z0-9_]*") + + class GraphInfo(TypedDict): """Structured information about an Apache AGE graph.""" @@ -65,18 +90,17 @@ def parse_agtype(val: Any) -> Any: This helper strips those type suffixes recursively and parses the clean payload using standard ``json.loads``. """ - if isinstance(val, str): - if val.endswith("::vertex") or val.endswith("::edge") or val.endswith("::path"): - # Split by unescaped double quotes to safely strip suffixes only outside string literals - parts = re.split(r'(? GraphDescripti if not graph_name.replace("_", "").isalnum() or graph_name[0].isdigit(): raise GraphError(f"invalid graph name: {graph_name!r}") + # Read-only introspection — same capability gate as the other graph + # read tools (cypher.run_cypher's read path, graph_diagram.generate_graph_diagram). + check_permission(Capability.READ, context.settings.access_mode) + driver = context.database.driver() # 1. Fetch graph metadata to ensure it exists @@ -157,11 +185,17 @@ async def describe_graph(context: AppContext, graph_name: str) -> GraphDescripti # but we must avoid internal '_ag_label_vertex' and '_ag_label_edge' tables. if name.startswith("_ag_label"): continue + # Catalog-derived name — validated before it reaches the f-string + # below. Raises (aborting describe_graph) rather than silently + # skipping, so a corrupted/malicious label surfaces as an error + # instead of a quietly wrong report. + _check_label_identifier(name) try: # Query row counts of the backing label table under the graph's schema count_rows = await driver.execute_query( - f'SELECT COALESCE(COUNT(*), 0) as cnt FROM "{graph_name}"."{name}";' + f'SELECT COALESCE(COUNT(*), 0) as cnt FROM "{graph_name}"."{name}";', + force_readonly=True, ) cnt = int(count_rows[0].cells["cnt"]) if count_rows else 0 except Exception: diff --git a/src/mcpg/graph_diagram.py b/src/mcpg/graph_diagram.py index a7c9fb64..bc4e6e46 100644 --- a/src/mcpg/graph_diagram.py +++ b/src/mcpg/graph_diagram.py @@ -12,7 +12,7 @@ from mcpg.context import AppContext from mcpg.database import DatabaseError -from mcpg.graph import GraphError +from mcpg.graph import GraphError, _check_label_identifier from mcpg.policy import Capability, check_permission logger = logging.getLogger(__name__) @@ -25,7 +25,12 @@ class DiagramResult(TypedDict): mermaid: str -async def generate_graph_diagram( +# C901 rationale: a numbered validate/permission-check/graph-exists/render +# pipeline (complexity 22) where each numbered step's failure mode (invalid +# name, permission denial, missing graph) needs its own distinct error +# before the AGE-catalog rendering proceeds -- collapsing the checks would +# blur which precondition failed. +async def generate_graph_diagram( # noqa: C901 context: AppContext, graph_name: str, limit: int = 50, @@ -78,6 +83,12 @@ async def generate_graph_diagram( kind = str(row.cells["kind"]) if name.startswith("_ag_label"): continue + # Catalog-derived name — validated up front, before it can reach + # either the interpolated SQL below or the generated Mermaid text + # (subgraph/edge labels). Raises (aborting the whole diagram), + # matching graph.describe_graph and graph_projection's precedent + # of aborting rather than silently dropping the offending label. + _check_label_identifier(name) if kind == "v": vertex_tables.append(name) elif kind == "e": @@ -92,6 +103,7 @@ async def generate_graph_diagram( v_rows = await driver.execute_query( f'SELECT id, properties::text as props FROM "{graph_name}"."{tbl}" LIMIT %s;', [limit - len(nodes)], + force_readonly=True, ) for vr in v_rows or []: raw_props = vr.cells.get("props") or "{}" @@ -104,7 +116,7 @@ async def generate_graph_diagram( } ) except Exception as exc: - logger.warning("failed to fetch vertices from label %s: %s", tbl, exc) + logger.warning("failed to fetch vertices from label %s: %s", tbl, exc, exc_info=True) # 6. Fetch edges edges: list[dict[str, Any]] = [] @@ -115,6 +127,7 @@ async def generate_graph_diagram( e_rows = await driver.execute_query( f'SELECT start_id, end_id, properties::text as props FROM "{graph_name}"."{tbl}" LIMIT %s;', [limit - len(edges)], + force_readonly=True, ) for er in e_rows or []: edges.append( @@ -125,7 +138,7 @@ async def generate_graph_diagram( } ) except Exception as exc: - logger.warning("failed to fetch edges from label %s: %s", tbl, exc) + logger.warning("failed to fetch edges from label %s: %s", tbl, exc, exc_info=True) # 7. Render Mermaid Flowchart lines = ["flowchart TD"] diff --git a/src/mcpg/graph_mgmt.py b/src/mcpg/graph_mgmt.py index 733bce3c..0a4b9633 100644 --- a/src/mcpg/graph_mgmt.py +++ b/src/mcpg/graph_mgmt.py @@ -70,7 +70,7 @@ async def create_graph(context: AppContext, graph_name: str) -> GraphMgmtResult: ) -async def drop_graph(context: AppContext, graph_name: str, cascade: bool = True) -> GraphMgmtResult: +async def drop_graph(context: AppContext, graph_name: str, *, cascade: bool = True) -> GraphMgmtResult: """Delete a property graph space, dropping all its nodes, edges, and schemas. Args: diff --git a/src/mcpg/graph_projection.py b/src/mcpg/graph_projection.py index 0204facc..ade1c73f 100644 --- a/src/mcpg/graph_projection.py +++ b/src/mcpg/graph_projection.py @@ -39,6 +39,7 @@ import re from dataclasses import dataclass +from mcpg.errors import MCPgError from mcpg.introspection import ColumnInfo, describe_table, list_foreign_keys from mcpg.sql import SqlDriver @@ -93,7 +94,7 @@ _BOOLEAN_TYPES = frozenset({"boolean", "bool"}) -class GraphProjectionError(Exception): +class GraphProjectionError(MCPgError): """Raised when a graph-projection request is rejected.""" @@ -286,7 +287,12 @@ def _edge_type_name(fk_name: str, from_table: str, to_table: str) -> str: return f"{from_table}_{to_table}" -async def generate_graph_projection( +# C901 rationale: relational-to-openCypher projection (complexity 23) -- +# validation, template-plan generation, and the optional row_limit>0 +# concrete-statement path are all read-only-guaranteed generation logic +# ("NEVER executed here"); the branching is the schema/table/row-limit +# validation matrix plus the two generation modes, not incidental nesting. +async def generate_graph_projection( # noqa: C901 driver: SqlDriver, schema: str, *, diff --git a/src/mcpg/headline_curator.py b/src/mcpg/headline_curator.py index 464837b8..092b1bb3 100644 --- a/src/mcpg/headline_curator.py +++ b/src/mcpg/headline_curator.py @@ -31,10 +31,11 @@ from dataclasses import dataclass, field from mcpg.about import CAPABILITIES, classify_tool +from mcpg.errors import MCPgError from mcpg.sql import SqlDriver -class HeadlineCuratorError(Exception): +class HeadlineCuratorError(MCPgError): """Raised when the recommender's arguments fail validation.""" diff --git a/src/mcpg/http_runtime.py b/src/mcpg/http_runtime.py index 33b11013..bd65fdbf 100644 --- a/src/mcpg/http_runtime.py +++ b/src/mcpg/http_runtime.py @@ -27,20 +27,25 @@ import logging import sys from collections.abc import Iterable +from contextlib import asynccontextmanager, suppress from typing import TYPE_CHECKING, Any, cast -from mcpg.config import Settings +from mcpg.config import ConfigError, Settings +from mcpg.errors import MCPgError from mcpg.observability import render_prometheus from mcpg.oidc import OIDCError, OIDCVerifier from mcpg.tenancy import _ROLE_SCOPE_KEY, TenancyError, current_role, validate_role if TYPE_CHECKING: - from collections.abc import Awaitable, Callable + from collections.abc import AsyncIterator, Awaitable, Callable + from contextlib import AbstractAsyncContextManager from starlette.applications import Starlette from starlette.requests import Request from starlette.responses import Response + from mcpg.database import Database + logger = logging.getLogger(__name__) # Standard Prometheus text-format content type. Compatible with both @@ -73,6 +78,38 @@ async def healthz(_request: Request) -> Response: return healthz +def _readiness_response_factory(database: Database | None) -> Callable[[Request], Awaitable[Response]]: + """Build the /readyz handler: ready once the DB pool has served a live connection. + + Distinct from /healthz (liveness — "is the process alive"): this reports + whether the process can currently do useful work, so an orchestrator can + pull a degraded instance out of rotation without restarting it. Reads + ``Database.is_connected`` (``self._connected and self._pool.is_valid`` — + see ``mcpg.database.Database``), the same side-effect-free signal + ``tools.py``'s ``server_info`` already uses to decide whether to attach a + live driver. No fresh connection is attempted on every poll. + + ``database`` is ``None`` when the wrapped app wasn't built by + ``create_server`` (e.g. a test stub that only implements + ``streamable_http_app``/``sse_app``) — there is nothing to assess, so we + report ready rather than failing a check that was never wired up. + + Note: ``is_connected`` only reflects the pool's state as of the last + ``pool_connect()``/``close()`` call — a database that answered at startup + and then dies mid-flight isn't detected until something next exercises + the pool. This still distinguishes "never came up" from "process alive", + which is /readyz's job here. + """ + from starlette.responses import PlainTextResponse + + async def readyz(_request: Request) -> Response: + if database is not None and not database.is_connected: + return PlainTextResponse("not ready\n", status_code=503) + return PlainTextResponse("ready\n") + + return readyz + + class _BearerAuthMiddleware: """ASGI middleware that enforces ``Authorization: Bearer ``. @@ -165,7 +202,7 @@ async def __call__(self, scope: dict[str, object], receive: object, send: object try: verified = await self._verifier.verify(token) except OIDCError as exc: - logger.warning("OIDC verification failed: %s", exc) + logger.warning("OIDC verification failed: %s", exc, exc_info=True) await _send_401(send, "invalid bearer token") return @@ -179,7 +216,7 @@ async def __call__(self, scope: dict[str, object], receive: object, send: object try: validate_role(verified.role) except TenancyError: - logger.warning("OIDC role claim has unsafe identifier: %r", verified.role) + logger.warning("OIDC role claim has unsafe identifier: %r", verified.role, exc_info=True) await _send_401(send, "role claim contains an invalid identifier") return @@ -406,7 +443,7 @@ def _client_is_allowed(self, scope: dict[str, object]) -> bool: class _SecurityHeadersMiddleware: """ASGI middleware that enforces standard security headers.""" - def __init__(self, app: object, *, hsts_max_age: int = 31536000) -> None: + def __init__(self, app: object, *, hsts_max_age: int = 63072000) -> None: self._app = app self._hsts_max_age = hsts_max_age @@ -440,7 +477,7 @@ async def send_wrapper(message: dict[str, object]) -> None: await self._app(scope, receive, send_wrapper) # type: ignore[operator] -class _RequestTooLargeError(Exception): +class _RequestTooLargeError(MCPgError): """Raised when the request body exceeds the configured maximum size.""" pass @@ -462,10 +499,8 @@ async def __call__(self, scope: dict[str, object], receive: object, send: object content_length = -1 for key, value in headers: if key.lower() == b"content-length": - try: + with suppress(ValueError): content_length = int(value) - except ValueError: - pass break if content_length > self._max_bytes: @@ -539,7 +574,39 @@ async def send_wrapper(message: dict[str, object]) -> None: await _send_504(send, f"request exceeded {self._timeout}s") -def build_http_app(server: object, settings: Settings, *, kind: str) -> Starlette: +def _close_oidc_verifier_on_lifespan_shutdown(app: Starlette, verifier: OIDCVerifier) -> None: + """Make the ASGI lifespan close ``verifier``'s HTTP client on shutdown. + + ``streamable_http_app()``/``sse_app()`` already install their own + lifespan (the MCP SDK's session-manager ``run()``, which itself + enters MCPg's ``make_lifespan`` closure — see ``mcpg.server``). That + closure is built by ``create_server`` *before* this function (and + the verifier) exist, so there's no way to reach it from here to add + a teardown call. Wrapping ``app.router.lifespan_context`` directly + is the one real hook available at this point: it runs inside the + same ASGI lifespan scope, alongside (not instead of) the SDK's own + teardown, and fires for both the streamable-http and sse transports + since both go through this function. + """ + inner_lifespan: Callable[[Starlette], AbstractAsyncContextManager[Any]] = app.router.lifespan_context + + @asynccontextmanager + async def _lifespan_with_oidc_close(started_app: Starlette) -> AsyncIterator[Any]: + try: + async with inner_lifespan(started_app) as state: + yield state + finally: + await verifier.aclose() + + app.router.lifespan_context = _lifespan_with_oidc_close + + +# C901 rationale: transport-kind dispatch (streamable-http/sse) plus mounting +# several independent routes (metrics/healthz/readyz) and conditional bearer +# auth wiring -- the SDK-quirk comment above (host-only kwarg, DNS-rebinding +# defaults) shows this is fragile-by-necessity glue code where consolidating +# branches risks reintroducing the transport-security regression it fixes. +def build_http_app(server: object, settings: Settings, *, kind: str) -> Starlette: # noqa: C901 """Wrap an MCPServer HTTP app with metrics + optional auth. Args: @@ -579,6 +646,13 @@ def build_http_app(server: object, settings: Settings, *, kind: str) -> Starlett # subclassing. app.router.routes.append(Route("/metrics", _metrics_response_factory(), methods=["GET"])) app.router.routes.append(Route("/healthz", _health_response_factory(), methods=["GET"])) + # ``server`` is typed ``object`` (tests pass bare stubs implementing only + # streamable_http_app/sse_app), so mcpg_database is read defensively — + # getattr rather than an isinstance/Protocol check, matching this + # function's existing style of type: ignore'd attribute access on + # ``server`` (see the streamable_http_app/sse_app calls just above). + database: Database | None = getattr(server, "mcpg_database", None) + app.router.routes.append(Route("/readyz", _readiness_response_factory(database), methods=["GET"])) # Middleware stack ordering: # In OIDC mode, the OIDC middleware verifies the JWT AND stashes @@ -600,18 +674,25 @@ def build_http_app(server: object, settings: Settings, *, kind: str) -> Starlett allowed_roles=settings.allowed_roles, ) app.add_middleware(_OIDCAuthMiddleware, verifier=verifier) + _close_oidc_verifier_on_lifespan_shutdown(app, verifier) else: if settings.default_role is not None or settings.allowed_roles: app.add_middleware(_TenantRoleMiddleware, allowed_roles=settings.allowed_roles) if settings.http_auth_token is not None: app.add_middleware(_BearerAuthMiddleware, token=settings.http_auth_token) - else: + elif settings.http_allow_unauthenticated: logger.warning( - "MCPg HTTP transport %s is running without auth. " - "Set MCPG_HTTP_AUTH_TOKEN or MCPG_AUTH_MODE=oidc to require " - "bearer tokens on every request.", + "MCPg HTTP transport %s is running WITHOUT AUTH — MCPG_HTTP_ALLOW_UNAUTHENTICATED=true " + "was set explicitly. This is your deliberate choice; if it wasn't, unset that variable " + "and set MCPG_HTTP_AUTH_TOKEN or MCPG_AUTH_MODE=oidc instead.", kind, ) + else: + raise ConfigError( + f"MCPg HTTP transport ({kind}) refuses to start unauthenticated. Set " + "MCPG_HTTP_AUTH_TOKEN, set MCPG_AUTH_MODE=oidc, or set " + "MCPG_HTTP_ALLOW_UNAUTHENTICATED=true to explicitly opt out (not recommended)." + ) # Outer middlewares (processed first on request) app.add_middleware(_SecurityHeadersMiddleware, hsts_max_age=settings.http_hsts_max_age) @@ -630,6 +711,10 @@ def build_http_app(server: object, settings: Settings, *, kind: str) -> Starlett allow_methods=["*"], allow_headers=["*"], ) + if settings.http_trusted_hosts: + from starlette.middleware.trustedhost import TrustedHostMiddleware + + app.add_middleware(TrustedHostMiddleware, allowed_hosts=list(settings.http_trusted_hosts)) # IP allowlist sits at the OUTERMOST layer (added last → processed # first per Starlette's middleware stacking) so denied clients diff --git a/src/mcpg/indexing.py b/src/mcpg/indexing.py index 4a94efde..d4f536da 100644 --- a/src/mcpg/indexing.py +++ b/src/mcpg/indexing.py @@ -75,15 +75,15 @@ class _TableAgg: suggestions: list[IndexSuggestion] = field(default_factory=list) _seen_columns: set[str] = field(default_factory=set, repr=False) - def add_stats(self, seq_scan: int, live_tup: int, is_partition: bool) -> None: + def add_stats(self, seq_scan: int, live_tup: int, *, is_partition: bool) -> None: self.seq_scans += seq_scan self.live_tuples += live_tup self.partitioned = self.partitioned or is_partition - def add_suggestion(self, column: str, data_type: str, is_unindexed_fk: bool = False) -> None: + def add_suggestion(self, column: str, data_type: str, *, is_unindexed_fk: bool = False) -> None: if column in self._seen_columns: return - suggestion = _suggest(column, data_type, is_unindexed_fk) + suggestion = _suggest(column, data_type, is_unindexed_fk=is_unindexed_fk) if suggestion is None: return self._seen_columns.add(column) @@ -101,7 +101,7 @@ def add_suggestion(self, column: str, data_type: str, is_unindexed_fk: bool = Fa ) -def _suggest(column: str, data_type: str, is_unindexed_fk: bool = False) -> IndexSuggestion | None: +def _suggest(column: str, data_type: str, *, is_unindexed_fk: bool = False) -> IndexSuggestion | None: """Suggest an index for a column based on FK status, then data type.""" # An unindexed foreign key wins: a plain btree on the FK column is the # highest-value fix, independent of the column's data type. @@ -184,8 +184,12 @@ async def recommend_indexes( physical = (row.cells["schemaname"], row.cells["relname"]) if physical not in counted: counted.add(physical) - agg.add_stats(row.cells["seq_scan"], row.cells["n_live_tup"], is_partition) - agg.add_suggestion(row.cells["column_name"], row.cells["data_type"], bool(row.cells.get("is_unindexed_fk"))) + agg.add_stats(row.cells["seq_scan"], row.cells["n_live_tup"], is_partition=is_partition) + agg.add_suggestion( + row.cells["column_name"], + row.cells["data_type"], + is_unindexed_fk=bool(row.cells.get("is_unindexed_fk")), + ) return [ IndexRecommendation( diff --git a/src/mcpg/introspection.py b/src/mcpg/introspection.py index 3516419f..4e955b90 100644 --- a/src/mcpg/introspection.py +++ b/src/mcpg/introspection.py @@ -1162,7 +1162,14 @@ async def list_generated_columns(driver: SqlDriver, schema: str) -> list[Generat ] -async def get_compact_schema(driver: SqlDriver, schema: str) -> str: +# C901 rationale: merges 3 separate catalog queries (columns, PKs, FKs) into +# nested per-table dicts before rendering the compact notation -- the +# `pg_attribute`/`information_schema` result shapes differ enough (vector +# dimension detection, cross-schema FK qualification) that each merge step +# is its own small state machine; deliberately 3 queries total per the +# docstring's own token-efficiency contract, not a candidate for further +# query splitting. +async def get_compact_schema(driver: SqlDriver, schema: str) -> str: # noqa: C901 """Return a highly condensed, token-efficient text summary of a schema. Collects all tables, columns, primary keys, and foreign keys in the schema diff --git a/src/mcpg/jooq.py b/src/mcpg/jooq.py index 512418d5..3e23b037 100644 --- a/src/mcpg/jooq.py +++ b/src/mcpg/jooq.py @@ -34,6 +34,7 @@ import re from xml.sax.saxutils import escape +from mcpg.errors import MCPgError from mcpg.introspection import ( ColumnInfo, describe_table, @@ -47,7 +48,7 @@ _EXCLUDED_TABLE_PATTERN = "mcpg_audit\\..*|mcpg_migrations\\..*" -class JooqExportError(Exception): +class JooqExportError(MCPgError): """Raised when a jOOQ export call is rejected or fails.""" @@ -127,13 +128,11 @@ async def generate_jooq_config( _check_identifier(t.name, "table") # Build the includes regex out of explicit table names — anchored - # so a future ``widget2`` table won't accidentally get generated. - if tables: - # Each name is already a plain identifier (validated above), so - # no regex-meta-character escaping is needed here. - includes_expr = "|".join(f"{schema}\\.{t.name}" for t in tables) - else: - includes_expr = "" # nothing to generate + # so a future ``widget2`` table won't accidentally get generated. Each + # name is already a plain identifier (validated above), so no + # regex-meta-character escaping is needed here. Empty when there are + # no tables to generate. + includes_expr = "|".join(f"{schema}\\.{t.name}" for t in tables) if tables else "" # Collect forced-type entries for JSON / JSONB columns across every table. forced_types: list[str] = [] diff --git a/src/mcpg/listen.py b/src/mcpg/listen.py index 14b3a5ca..ef4027e8 100644 --- a/src/mcpg/listen.py +++ b/src/mcpg/listen.py @@ -20,6 +20,7 @@ from __future__ import annotations import asyncio +import contextlib import logging import re import time @@ -27,7 +28,9 @@ from collections.abc import Awaitable, Callable from dataclasses import dataclass, field, replace from types import TracebackType -from typing import Any, Protocol +from typing import Any, Protocol, Self + +from mcpg.errors import MCPgError logger = logging.getLogger(__name__) @@ -39,7 +42,7 @@ _CHANNEL_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\Z") -class ListenError(Exception): +class ListenError(MCPgError): """Raised when a LISTEN/NOTIFY tool call is rejected or fails.""" @@ -249,13 +252,11 @@ async def close(self) -> None: try: await asyncio.wait_for(conn.close(), timeout=2.0) except Exception: - pass + logger.debug("Best-effort connection close during shutdown failed", exc_info=True) if task is not None: task.cancel() - try: + with contextlib.suppress(TimeoutError, asyncio.CancelledError, Exception): await asyncio.wait_for(task, timeout=2.0) - except (TimeoutError, asyncio.CancelledError, Exception): - pass # --- internals -------------------------------------------------- @@ -361,7 +362,7 @@ async def _reader_loop(self) -> None: self._task = None self._needs_resubscribe = True - async def __aenter__(self) -> ListenManager: + async def __aenter__(self) -> Self: return self async def __aexit__( diff --git a/src/mcpg/locks.py b/src/mcpg/locks.py index 22679d02..6a598443 100644 --- a/src/mcpg/locks.py +++ b/src/mcpg/locks.py @@ -17,10 +17,11 @@ from dataclasses import dataclass from typing import Any +from mcpg.errors import MCPgError from mcpg.sql import SqlDriver -class LocksError(Exception): +class LocksError(MCPgError): """Raised on invalid input to a lock-inspection function. Matches the ``*Error``-per-module convention every other surface @@ -285,7 +286,7 @@ def _normalize_cycle(cycle: list[int]) -> tuple[int, ...]: min_val = min(cycle) min_idx = cycle.index(min_val) normalized = cycle[min_idx:] + cycle[:min_idx] - return tuple([*normalized, min_val]) + return (*normalized, min_val) def _find_cycles(adj: dict[int, list[int]], all_pids: set[int]) -> list[list[int]]: diff --git a/src/mcpg/logical_replication.py b/src/mcpg/logical_replication.py index 0d5c4281..98ff92a0 100644 --- a/src/mcpg/logical_replication.py +++ b/src/mcpg/logical_replication.py @@ -52,6 +52,7 @@ from dataclasses import dataclass from mcpg.database import Database +from mcpg.errors import MCPgError # Unquoted PostgreSQL identifier: starts with letter / underscore, # then letters / digits / underscores. Same shape as @@ -60,7 +61,7 @@ _IDENTIFIER = re.compile(r"\A[A-Za-z_][A-Za-z0-9_]*\Z") -class LogicalReplicationError(Exception): +class LogicalReplicationError(MCPgError): """Raised when a logical-replication write is rejected or fails.""" @@ -272,7 +273,12 @@ async def drop_publication( # --------------------------------------------------------------------------- -async def create_subscription( +# C901 rationale: per-argument identifier validation (name, each publication, +# slot name) plus quoting before DDL construction (no parameter-bind slot is +# available for CREATE SUBSCRIPTION) -- each check is a distinct +# injection-defense gate; consolidating them doesn't reduce the number of +# things that must be individually correct. +async def create_subscription( # noqa: C901 database: Database, *, name: str, diff --git a/src/mcpg/maintenance.py b/src/mcpg/maintenance.py index 25aa53ba..a4d42ba2 100644 --- a/src/mcpg/maintenance.py +++ b/src/mcpg/maintenance.py @@ -12,6 +12,7 @@ from dataclasses import dataclass from mcpg.database import Database +from mcpg.errors import MCPgError # Accepted operation -> the SQL command it maps to. _OPERATIONS = { @@ -21,7 +22,7 @@ } -class MaintenanceError(Exception): +class MaintenanceError(MCPgError): """Raised when a maintenance request is rejected or fails.""" diff --git a/src/mcpg/migration_history.py b/src/mcpg/migration_history.py index 85179a14..30909a7b 100644 --- a/src/mcpg/migration_history.py +++ b/src/mcpg/migration_history.py @@ -121,7 +121,15 @@ def _quote_ident(name: str) -> str: return f'"{name.replace(chr(34), chr(34) * 2)}"' -async def read_migration_history( +# C901 rationale: dispatches on 8 known migration-framework bookkeeping +# table names (Alembic/Flyway/Diesel/Django/Prisma/golang-migrate/Goose/ +# Sequelize), each its own query + framework-specific row-parsing + +# try/except-and-skip-on-failure block feeding a distinct named result +# field. A dispatch-table-of-handlers refactor is plausible but is a +# larger restructuring than this pass's cheap-win budget; each framework's +# parsing failure must independently degrade without aborting the others, +# which the current straight-line branches make easy to audit. +async def read_migration_history( # noqa: C901 driver: SqlDriver, schema: str | None = None, ) -> MigrationHistoryReport: diff --git a/src/mcpg/migration_ingestion.py b/src/mcpg/migration_ingestion.py index 524bb7bf..a295865e 100644 --- a/src/mcpg/migration_ingestion.py +++ b/src/mcpg/migration_ingestion.py @@ -42,6 +42,7 @@ from dataclasses import dataclass, field from pathlib import Path +from mcpg.errors import MCPgError from mcpg.sql import SqlDriver _FRAMEWORKS = frozenset({"alembic", "flyway", "liquibase"}) @@ -76,7 +77,7 @@ _FIRST_COMMENT_CAP = 200 -class MigrationIngestionError(Exception): +class MigrationIngestionError(MCPgError): """Raised when a migration-ingestion request is rejected.""" diff --git a/src/mcpg/migrations.py b/src/mcpg/migrations.py index 31fb05fe..000a2809 100644 --- a/src/mcpg/migrations.py +++ b/src/mcpg/migrations.py @@ -18,6 +18,8 @@ from __future__ import annotations +import contextlib +import logging import re import time import uuid @@ -25,16 +27,19 @@ from datetime import UTC, datetime, timedelta from typing import Any +from mcpg.errors import MCPgError from mcpg.introspection import describe_table, list_constraints, list_indexes, list_tables from mcpg.schema_diff import SchemaDiff, compare_schemas from mcpg.sql import SqlDriver +logger = logging.getLogger(__name__) + _IDENTIFIER = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\Z") _SHADOW_PREFIX = "mcpg_shadow_" _NAME_SUFFIX_RE = re.compile(r"[^A-Za-z0-9_]") -class MigrationError(Exception): +class MigrationError(MCPgError): """Raised when a migration tool call is rejected or fails.""" @@ -347,7 +352,7 @@ async def prepare_migration( try: await driver.execute_query(f'DROP SCHEMA IF EXISTS "{shadow_schema}" CASCADE') except Exception: - pass + logger.debug("Failed to drop half-built shadow schema %r during cleanup", shadow_schema, exc_info=True) raise # Persist the staged row. INSERT through a parametrised statement so @@ -502,10 +507,9 @@ async def _execute_in_schema(driver: SqlDriver, schema: str, sql: str) -> None: await cur.execute(sql) return pool_obj = await driver.conn.pool_connect() - async with pool_obj.connection() as conn: - async with conn.cursor() as cur: - await cur.execute(f'SET LOCAL search_path TO "{schema}", public') - await cur.execute(sql) + async with pool_obj.connection() as conn, conn.cursor() as cur: + await cur.execute(f'SET LOCAL search_path TO "{schema}", public') + await cur.execute(sql) async def _load_record(driver: SqlDriver, migration_id: str) -> MigrationRecord | None: @@ -625,17 +629,15 @@ async def validate_migration( # INSERT INTO shadow.table SELECT * FROM target.table LIMIT N. # We catch FK-violation errors per-table so one bad table # doesn't abort the whole validation — the agent gets a - # complete picture instead. - try: + # complete picture instead. Sampling failure (FK violation, + # etc.) is recorded as zero rows for that table; the + # validation continues. + with contextlib.suppress(Exception): await driver.execute_query( f'INSERT INTO "{shadow_schema}"."{table.name}" ' f'SELECT * FROM "{target_schema}"."{table.name}" LIMIT %s', params=[sample_rows_per_table], ) - except Exception: - # Sampling failure (FK violation, etc.) is recorded as - # zero rows for that table; the validation continues. - pass # Snapshot counts before the candidate runs. before_counts: dict[str, int] = {} @@ -666,10 +668,8 @@ async def validate_migration( ) ) finally: - try: + with contextlib.suppress(Exception): await driver.execute_query(f'DROP SCHEMA IF EXISTS "{shadow_schema}" CASCADE') - except Exception: - pass return ValidationResult( target_schema=target_schema, @@ -746,10 +746,8 @@ async def validate_migration_schema( if applied: diff = await compare_schemas(driver, reference_schema, shadow_schema) finally: - try: + with contextlib.suppress(Exception): await driver.execute_query(f'DROP SCHEMA IF EXISTS "{shadow_schema}" CASCADE') - except Exception: - pass return MigrationSchemaValidationResult( target_schema=target_schema, diff --git a/src/mcpg/multidb.py b/src/mcpg/multidb.py index 0e7cc0b7..28c87a39 100644 --- a/src/mcpg/multidb.py +++ b/src/mcpg/multidb.py @@ -24,6 +24,7 @@ from __future__ import annotations +import contextlib import logging from collections.abc import Iterable from dataclasses import dataclass @@ -99,29 +100,33 @@ async def execute_query( self, query: Any, params: list[Any] | None = None, + *, force_readonly: bool = False, + row_limit: int | None = None, ) -> list[SqlDriver.RowResult] | None: # Ignore the caller's flag — a secondary is read-only, full stop. del force_readonly - return await super().execute_query(query, params, force_readonly=True) + return await super().execute_query(query, params, force_readonly=True, row_limit=row_limit) async def _execute_with_connection( # type: ignore[no-untyped-def] self, connection, query, params, + *, force_readonly, + row_limit=None, ): if not getattr(connection, "_timeouts_configured", False): async with connection.cursor() as cursor: await cursor.execute( f"SET statement_timeout = {self._statement_timeout_ms}; SET lock_timeout = {self._lock_timeout_ms}" ) - try: + with contextlib.suppress(AttributeError): connection._timeouts_configured = True - except AttributeError: - pass - return await super()._execute_with_connection(connection, query, params, force_readonly) + return await super()._execute_with_connection( + connection, query, params, force_readonly=force_readonly, row_limit=row_limit + ) # NOTE: these two return shapes intentionally avoid ``slots=True`` and field diff --git a/src/mcpg/nl2sql.py b/src/mcpg/nl2sql.py index 5e0cb003..f6faadb6 100644 --- a/src/mcpg/nl2sql.py +++ b/src/mcpg/nl2sql.py @@ -37,7 +37,10 @@ from typing import TYPE_CHECKING, Any, Literal, Protocol, runtime_checkable import httpx +from circuitbreaker import CircuitBreakerError, circuit +from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter +from mcpg.errors import MCPgError from mcpg.introspection import describe_table, list_foreign_keys, list_tables from mcpg.query import DEFAULT_MAX_ROWS, QueryError, explain_query, run_select from mcpg.sql import SqlDriver, obfuscate_password @@ -232,6 +235,38 @@ def _env_hint(env_vars: tuple[str, ...]) -> str: # 30s; pad for slow networks / slower models. DEFAULT_TIMEOUT_SECONDS = 60.0 +# Circuit breaker tuning for each provider's ``complete()`` HTTP call — +# after this many consecutive failures, further calls to that *class* (not +# instance — see the ``@circuit`` note on each provider below) fail fast +# with ``CircuitBreakerError`` for ``recovery_timeout`` seconds instead of +# each one separately paying the full ``timeout`` cost against a degraded +# vendor. Threshold counts once per logical call even once retry-with- +# backoff is layered *inside* the breaker (see each ``complete``'s +# ``@retry``) — only an attempt that exhausts all its retries counts as one +# breaker failure, not one per retry. +NL2SQL_CIRCUIT_FAILURE_THRESHOLD = 5 +NL2SQL_CIRCUIT_RECOVERY_TIMEOUT_SECONDS = 30.0 + +# Retry tuning for each provider's ``complete()`` HTTP call — a handful of +# quick attempts with exponential backoff + jitter before giving up, so a +# single dropped connection or transient 5xx doesn't surface as a user- +# facing failure. `@retry` is applied *inside* `@circuit` below (i.e. +# `@circuit` is the outer decorator) deliberately, not by accident of +# decorator-stacking order: `circuitbreaker.call_async` (see the installed +# package's source) does `with self: return await func(...)` and counts +# exactly one failure per invocation of whatever it wraps. With retry +# innermost, all `NL2SQL_RETRY_STOP_ATTEMPTS` attempts happen *inside* that +# one `with self:` block, so an exhausted retry cycle counts as ONE breaker +# failure. Stacked the other way (`@retry` outer, `@circuit` inner — as a +# stale draft of this task's brief showed), each retry attempt would +# separately enter/exit the breaker's `with self:`, so 3 quick retries +# would burn 3 of the breaker's `failure_threshold` slots for a single +# logical call — letting retries alone trip the breaker. +NL2SQL_RETRY_STOP_ATTEMPTS = 3 +NL2SQL_RETRY_WAIT_INITIAL_SECONDS = 0.1 +NL2SQL_RETRY_WAIT_MAX_SECONDS = 2.0 +NL2SQL_RETRY_WAIT_JITTER_SECONDS = 0.1 + _SUPPORTED_PROVIDERS = frozenset(DEFAULT_MODELS) # Schema-brief sizing — bounded so the prompt doesn't explode on large @@ -290,7 +325,7 @@ def _env_hint(env_vars: tuple[str, ...]) -> str: _DEFAULT_EXPR_MAX_CHARS = 80 -class NL2SQLError(Exception): +class NL2SQLError(MCPgError): """Raised when NL→SQL translation is rejected or fails.""" @@ -299,7 +334,10 @@ class TranslationResult: """Result of :func:`translate_nl_to_sql`. ``sql`` is the generated query; empty when parsing failed. - ``explanation`` is the model's natural-language rationale. When + ``explanation`` is the model's natural-language rationale. + ``schema_context`` is the rendered schema brief the model actually + saw for this call — empty only on the (currently nonexistent) path + where the result is built before schema-gathering runs. When ``execute=True`` and the SQL passed the safety check, ``rows`` / ``columns`` / ``row_count`` are populated and ``executed`` is ``True``. On safety / execution failure, ``error`` carries the @@ -310,6 +348,11 @@ class TranslationResult: explanation: str model: str provider: str + # The rendered schema brief actually sent to the model as part of the + # prompt for this translation (see ``_build_schema_brief``) — kept so a + # generated query's provenance is traceable to the schema evidence that + # informed it, not just which model/provider produced it. + schema_context: str executed: bool rows: list[dict[str, Any]] columns: list[str] @@ -362,12 +405,63 @@ async def complete( user_prompt: str, model: str, max_tokens: int, - timeout: float, + timeout: float, # noqa: ASYNC109 -- forwarded to httpx's per-request timeout, not a manual reimplementation ) -> ProviderCompletion: """Send the prompt; return the completion text + usage. Raises on transport error.""" ... +# Process-wide client shared by all three provider classes below. +# +# `build_provider` is called fresh on every `translate_nl_to_sql` tool +# invocation (provider selection can vary per-call via the `provider=` +# argument), so holding a client on each provider *instance* wouldn't +# help — a new instance still means a new client. Sharing one client at +# module scope instead means every call, regardless of how many +# providers get constructed, reuses the same keep-alive connection pool +# rather than paying a fresh TCP/TLS handshake per call. httpx already +# pools connections per-host internally, so one client shared across +# vendors (each with its own host) is correct usage, not a compromise. +# +# Lazily created on first use so importing this module never opens a +# client; closed via `aclose_shared_client()`, which the server's +# lifespan hook (`mcpg.server.make_lifespan`) calls on shutdown. +_shared_http_client: httpx.AsyncClient | None = None + + +def _get_shared_http_client() -> httpx.AsyncClient: + """Return the process-wide ``httpx.AsyncClient`` shared by all providers.""" + global _shared_http_client + if _shared_http_client is None: + _shared_http_client = httpx.AsyncClient() + return _shared_http_client + + +async def aclose_shared_client() -> None: + """Close the shared HTTP client, if one was ever constructed. + + Call once at server shutdown. Safe to call when no provider call + has happened yet (no-op) and safe to call more than once. + """ + global _shared_http_client + if _shared_http_client is not None: + await _shared_http_client.aclose() + _shared_http_client = None + + +def _reset_shared_http_client() -> None: + """Test-only: drop the cached shared client without closing it. + + Each async test runs its own event loop (pytest-asyncio, + function-scoped), and an ``httpx.AsyncClient`` built against one + loop breaks if reused from another. Tests that mock the transport + per-call must reset this between tests so they don't inherit a + prior test's cached (and differently-mocked) client instance. + """ + global _shared_http_client + _shared_http_client = None + + class AnthropicProvider: """Anthropic Messages API caller — `POST /v1/messages`.""" @@ -377,6 +471,33 @@ def __init__(self, api_key: str, *, base_url: str = "https://api.anthropic.com") self._api_key = api_key self._base_url = base_url.rstrip("/") + # ``@circuit`` decorates the plain function object at class-definition + # time, so its failure count is one object shared by every + # ``AnthropicProvider`` instance for the process's lifetime — not + # per-instance state. That's intentional: `build_provider` constructs a + # fresh provider on every `translate_nl_to_sql` call (see its + # module-level docstring), so per-instance breaker state would never + # accumulate a single failure across calls. `expected_exception` is + # scoped to `httpx.HTTPError` (network/timeout/non-2xx) so a bug in our + # own response parsing can't trip the *network* breaker. + @circuit( # type: ignore[untyped-decorator] + failure_threshold=NL2SQL_CIRCUIT_FAILURE_THRESHOLD, + recovery_timeout=NL2SQL_CIRCUIT_RECOVERY_TIMEOUT_SECONDS, + expected_exception=httpx.HTTPError, + ) + @retry( + reraise=True, # load-bearing: without it, exhaustion raises + # tenacity.RetryError instead of the original httpx.HTTPError, which + # wouldn't match @circuit's expected_exception above — the breaker + # would silently never count a retry-exhausted call as a failure. + stop=stop_after_attempt(NL2SQL_RETRY_STOP_ATTEMPTS), + wait=wait_exponential_jitter( + initial=NL2SQL_RETRY_WAIT_INITIAL_SECONDS, + max=NL2SQL_RETRY_WAIT_MAX_SECONDS, + jitter=NL2SQL_RETRY_WAIT_JITTER_SECONDS, + ), + retry=retry_if_exception_type(httpx.HTTPError), + ) async def complete( self, *, @@ -384,23 +505,24 @@ async def complete( user_prompt: str, model: str, max_tokens: int, - timeout: float, + timeout: float, # noqa: ASYNC109 -- forwarded to httpx's per-request timeout, not a manual reimplementation ) -> ProviderCompletion: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.post( - f"{self._base_url}/v1/messages", - headers={ - "x-api-key": self._api_key, - "anthropic-version": "2023-06-01", - "content-type": "application/json", - }, - json={ - "model": model, - "max_tokens": max_tokens, - "system": system_prompt, - "messages": [{"role": "user", "content": user_prompt}], - }, - ) + client = _get_shared_http_client() + response = await client.post( + f"{self._base_url}/v1/messages", + headers={ + "x-api-key": self._api_key, + "anthropic-version": "2023-06-01", + "content-type": "application/json", + }, + json={ + "model": model, + "max_tokens": max_tokens, + "system": system_prompt, + "messages": [{"role": "user", "content": user_prompt}], + }, + timeout=timeout, + ) response.raise_for_status() body = response.json() usage = body.get("usage") or {} @@ -428,6 +550,30 @@ def __init__(self, api_key: str, *, base_url: str = "https://api.openai.com/v1") self._api_key = api_key self._base_url = base_url.rstrip("/") + # See the matching comment on ``AnthropicProvider.complete`` — shared + # class-level breaker state is intentional given per-call construction. + # Note this one breaker covers every OpenAI-compatible vendor (the + # `OPENAI_COMPATIBLE_BASE_URLS` fleet all route through this class), so + # a run of failures against one vendor's endpoint trips the same + # breaker as another OpenAI-compatible vendor would. + @circuit( # type: ignore[untyped-decorator] + failure_threshold=NL2SQL_CIRCUIT_FAILURE_THRESHOLD, + recovery_timeout=NL2SQL_CIRCUIT_RECOVERY_TIMEOUT_SECONDS, + expected_exception=httpx.HTTPError, + ) + @retry( + reraise=True, # load-bearing: without it, exhaustion raises + # tenacity.RetryError instead of the original httpx.HTTPError, which + # wouldn't match @circuit's expected_exception above — the breaker + # would silently never count a retry-exhausted call as a failure. + stop=stop_after_attempt(NL2SQL_RETRY_STOP_ATTEMPTS), + wait=wait_exponential_jitter( + initial=NL2SQL_RETRY_WAIT_INITIAL_SECONDS, + max=NL2SQL_RETRY_WAIT_MAX_SECONDS, + jitter=NL2SQL_RETRY_WAIT_JITTER_SECONDS, + ), + retry=retry_if_exception_type(httpx.HTTPError), + ) async def complete( self, *, @@ -435,25 +581,26 @@ async def complete( user_prompt: str, model: str, max_tokens: int, - timeout: float, + timeout: float, # noqa: ASYNC109 -- forwarded to httpx's per-request timeout, not a manual reimplementation ) -> ProviderCompletion: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.post( - f"{self._base_url}/chat/completions", - headers={ - "authorization": f"Bearer {self._api_key}", - "content-type": "application/json", - }, - json={ - "model": model, - "max_tokens": max_tokens, - "messages": [ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": user_prompt}, - ], - "response_format": {"type": "json_object"}, - }, - ) + client = _get_shared_http_client() + response = await client.post( + f"{self._base_url}/chat/completions", + headers={ + "authorization": f"Bearer {self._api_key}", + "content-type": "application/json", + }, + json={ + "model": model, + "max_tokens": max_tokens, + "messages": [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt}, + ], + "response_format": {"type": "json_object"}, + }, + timeout=timeout, + ) response.raise_for_status() body = response.json() usage = body.get("usage") or {} @@ -475,6 +622,25 @@ def __init__(self, api_key: str, *, base_url: str = "https://generativelanguage. self._api_key = api_key self._base_url = base_url.rstrip("/") + # See the matching comment on ``AnthropicProvider.complete``. + @circuit( # type: ignore[untyped-decorator] + failure_threshold=NL2SQL_CIRCUIT_FAILURE_THRESHOLD, + recovery_timeout=NL2SQL_CIRCUIT_RECOVERY_TIMEOUT_SECONDS, + expected_exception=httpx.HTTPError, + ) + @retry( + reraise=True, # load-bearing: without it, exhaustion raises + # tenacity.RetryError instead of the original httpx.HTTPError, which + # wouldn't match @circuit's expected_exception above — the breaker + # would silently never count a retry-exhausted call as a failure. + stop=stop_after_attempt(NL2SQL_RETRY_STOP_ATTEMPTS), + wait=wait_exponential_jitter( + initial=NL2SQL_RETRY_WAIT_INITIAL_SECONDS, + max=NL2SQL_RETRY_WAIT_MAX_SECONDS, + jitter=NL2SQL_RETRY_WAIT_JITTER_SECONDS, + ), + retry=retry_if_exception_type(httpx.HTTPError), + ) async def complete( self, *, @@ -482,26 +648,27 @@ async def complete( user_prompt: str, model: str, max_tokens: int, - timeout: float, + timeout: float, # noqa: ASYNC109 -- forwarded to httpx's per-request timeout, not a manual reimplementation ) -> ProviderCompletion: # Gemini accepts the API key as a query string or `x-goog-api-key` # header — we use the header to avoid logging-route leakage. - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.post( - f"{self._base_url}/v1beta/models/{model}:generateContent", - headers={ - "x-goog-api-key": self._api_key, - "content-type": "application/json", + client = _get_shared_http_client() + response = await client.post( + f"{self._base_url}/v1beta/models/{model}:generateContent", + headers={ + "x-goog-api-key": self._api_key, + "content-type": "application/json", + }, + json={ + "systemInstruction": {"parts": [{"text": system_prompt}]}, + "contents": [{"role": "user", "parts": [{"text": user_prompt}]}], + "generationConfig": { + "maxOutputTokens": max_tokens, + "responseMimeType": "application/json", }, - json={ - "systemInstruction": {"parts": [{"text": system_prompt}]}, - "contents": [{"role": "user", "parts": [{"text": user_prompt}]}], - "generationConfig": { - "maxOutputTokens": max_tokens, - "responseMimeType": "application/json", - }, - }, - ) + }, + timeout=timeout, + ) response.raise_for_status() body = response.json() usage = body.get("usageMetadata") or {} @@ -1081,7 +1248,13 @@ async def _explain_preflight(driver: SqlDriver, sql: str) -> str | None: return None -async def translate_nl_to_sql( +# C901 rationale: the end-to-end NL->SQL orchestration (input validation, +# schema-brief building, provider call with circuit-breaker/retry error +# translation, response parsing, refusal detection, EXPLAIN pre-flight, +# optional execute-and-audit) -- each stage has its own distinct failure +# mode that must surface as a specific NL2SQLError, per this security- and +# cost-sensitive tool's own docstring contract. +async def translate_nl_to_sql( # noqa: C901 driver: SqlDriver, *, provider: LLMProvider, @@ -1096,7 +1269,8 @@ async def translate_nl_to_sql( max_tables_in_brief: int = DEFAULT_MAX_TABLES_IN_BRIEF, columns_per_table: int = DEFAULT_COLUMNS_PER_TABLE, max_brief_chars: int = DEFAULT_MAX_BRIEF_CHARS, - timeout: float = DEFAULT_TIMEOUT_SECONDS, + # ASYNC109 rationale: forwarded to provider.complete's own timeout, not a manual reimplementation. + timeout: float = DEFAULT_TIMEOUT_SECONDS, # noqa: ASYNC109 env: Mapping[str, str] | None = None, audit_persist: bool = False, ) -> TranslationResult: @@ -1190,6 +1364,12 @@ async def translate_nl_to_sql( ) except httpx.HTTPError as exc: raise NL2SQLError(f"NL→SQL provider request failed: {exc}") from exc + except CircuitBreakerError as exc: + # The breaker on `provider.complete` is open (too many recent + # failures) — translate to the module's own error type so callers + # only ever see `NL2SQLError` from this function, tripped breaker + # or not. + raise NL2SQLError(f"NL→SQL provider request failed: circuit open ({exc})") from exc raw = completion.text tokens_in, tokens_out = completion.tokens_in, completion.tokens_out @@ -1214,6 +1394,7 @@ async def translate_nl_to_sql( explanation=explanation or refusal_reason, model=model, provider=provider.name, + schema_context=schema_brief, executed=False, rows=[], columns=[], @@ -1232,6 +1413,7 @@ async def translate_nl_to_sql( explanation=explanation, model=model, provider=provider.name, + schema_context=schema_brief, executed=False, rows=[], columns=[], @@ -1246,6 +1428,7 @@ async def translate_nl_to_sql( explanation=explanation, model=model, provider=provider.name, + schema_context=schema_brief, executed=False, rows=[], columns=[], @@ -1267,6 +1450,7 @@ async def translate_nl_to_sql( explanation=explanation, model=model, provider=provider.name, + schema_context=schema_brief, executed=True, rows=exec_result.rows, columns=exec_result.columns, @@ -1286,6 +1470,7 @@ async def translate_nl_to_sql( explanation=explanation, model=model, provider=provider.name, + schema_context=schema_brief, executed=False, rows=[], columns=[], @@ -1320,6 +1505,6 @@ async def translate_nl_to_sql( env=env, ) except Exception as exc: # pragma: no cover - swallowed on purpose - logger.warning("NL→SQL audit persist failed (translation kept): %s", exc) + logger.warning("NL→SQL audit persist failed (translation kept): %s", exc, exc_info=True) return translation diff --git a/src/mcpg/obs_logging.py b/src/mcpg/obs_logging.py index 18f1052a..93801e84 100644 --- a/src/mcpg/obs_logging.py +++ b/src/mcpg/obs_logging.py @@ -8,10 +8,42 @@ from datetime import UTC, datetime from typing import TYPE_CHECKING +from mcpg.sql import obfuscate_password + if TYPE_CHECKING: from mcpg.config import Settings +class RedactionFilter(logging.Filter): + """Backstop redaction: scrubs a password-bearing connection string out of a log call's + message or %-style arguments when a call site reaches a handler without having passed + it through obfuscate_password() first. + + Not a replacement for calling obfuscate_password() explicitly where a value is known to + carry credentials — that per-call-site discipline still matters for accuracy (this filter + only recognizes the same connection-string shapes obfuscate_password() already does). This + is the centralized enforcement layer for the case a future call site forgets. + + Scope: covers only `record.msg` / `record.args` (the log call's own message and its + lazy-formatting arguments). It does NOT cover `record.exc_info` — an exception's traceback + (e.g. from `logger.exception(...)`) is rendered separately by + `logging.Formatter.formatException()`, which never passes through this filter, so a + password embedded in an exception's message can still leak via a formatter's "exception" + output even with this filter attached. + + Runs before formatting (logging.Filter operates on the LogRecord, not rendered output), so + it renders any lazy %-style args via getMessage() up front and clears record.args — this + both avoids leaving unredacted args sitting on the record and prevents a formatter that + calls getMessage() again (e.g. JSONFormatter) from re-applying % substitution to the + already-rendered, now-static message string. + """ + + def filter(self, record: logging.LogRecord) -> bool: + record.msg = obfuscate_password(record.getMessage()) + record.args = () + return True + + class JSONFormatter(logging.Formatter): """Formatter that outputs structured JSON for log events. @@ -73,6 +105,7 @@ def setup_logging(settings: Settings) -> None: formatter = logging.Formatter("%(asctime)s [%(levelname)s] %(name)s: %(message)s") handler.setFormatter(formatter) + handler.addFilter(RedactionFilter()) logger.addHandler(handler) # Disable propagation to prevent double-logging if root has handlers diff --git a/src/mcpg/oidc.py b/src/mcpg/oidc.py index 4e6b0941..b143eed2 100644 --- a/src/mcpg/oidc.py +++ b/src/mcpg/oidc.py @@ -36,7 +36,11 @@ import httpx import jwt +from circuitbreaker import CircuitBreakerError, circuit from jwt import PyJWKClient +from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter + +from mcpg.errors import MCPgError logger = logging.getLogger(__name__) @@ -45,8 +49,36 @@ DEFAULT_JWKS_CACHE_SECONDS = 3600.0 DEFAULT_VERIFY_LEEWAY_SECONDS = 30.0 +# Circuit breaker tuning for the discovery-document fetch in +# `_resolve_jwks_url` — after this many consecutive failures, further calls +# fail fast with `CircuitBreakerError` (translated to `OIDCError` by +# `_ensure_jwks_client` below) for `recovery_timeout` seconds instead of +# every request separately paying the full `discovery_timeout` cost against +# a degraded IdP. Threshold counts once per logical call even once +# retry-with-backoff is layered *inside* the breaker (see +# `_resolve_jwks_url`'s `@retry`) — only an attempt that exhausts all its +# retries counts as one breaker failure, not one per retry. +OIDC_CIRCUIT_FAILURE_THRESHOLD = 5 +OIDC_CIRCUIT_RECOVERY_TIMEOUT_SECONDS = 30.0 + +# Retry tuning for the discovery-document fetch — a handful of quick +# attempts with exponential backoff + jitter before giving up, so a single +# dropped connection doesn't fail a request that would have succeeded on a +# retry. `@retry` is applied *inside* `@circuit` below (i.e. `@circuit` is +# the outer decorator) deliberately: `circuitbreaker.call_async` (see the +# installed package's source) does `with self: return await func(...)` and +# counts exactly one failure per invocation of whatever it wraps. With +# retry innermost, all `OIDC_RETRY_STOP_ATTEMPTS` attempts happen *inside* +# that one `with self:` block, so an exhausted retry cycle counts as ONE +# breaker failure — not one per retry, which is what stacking the two +# decorators the other way around would produce. +OIDC_RETRY_STOP_ATTEMPTS = 3 +OIDC_RETRY_WAIT_INITIAL_SECONDS = 0.1 +OIDC_RETRY_WAIT_MAX_SECONDS = 2.0 +OIDC_RETRY_WAIT_JITTER_SECONDS = 0.1 -class OIDCError(Exception): + +class OIDCError(MCPgError): """Raised when OIDC configuration is wrong or a token fails to verify.""" @@ -147,7 +179,40 @@ def __init__( self._discovery: _DiscoveryCache | None = None self._jwks_client: PyJWKClient | None = None + # Held for this verifier's lifetime rather than opened fresh per + # discovery-document fetch — construction is cheap (no I/O), so + # this is safe even though discovery itself is infrequent (cached + # for jwks_cache_seconds, and skipped entirely when jwks_url is + # supplied explicitly). + self._client = httpx.AsyncClient(timeout=self._discovery_timeout) + # ``@circuit`` decorates the plain function object at class-definition + # time, so its failure count is one object shared by every + # ``OIDCVerifier`` instance for the process's lifetime, not per-instance + # state — acceptable here since a process typically runs one configured + # IdP. `expected_exception=OIDCError` matches this method's own error + # type (the try/except below already normalises every failure mode — + # bad URL, connection error, malformed discovery doc — into `OIDCError` + # before it escapes the function), so only genuine discovery failures + # count toward the threshold. + @circuit( # type: ignore[untyped-decorator] + failure_threshold=OIDC_CIRCUIT_FAILURE_THRESHOLD, + recovery_timeout=OIDC_CIRCUIT_RECOVERY_TIMEOUT_SECONDS, + expected_exception=OIDCError, + ) + @retry( + reraise=True, # load-bearing: without it, exhaustion raises + # tenacity.RetryError instead of OIDCError, which wouldn't match + # @circuit's expected_exception above — the breaker would silently + # never count a retry-exhausted call as a failure. + stop=stop_after_attempt(OIDC_RETRY_STOP_ATTEMPTS), + wait=wait_exponential_jitter( + initial=OIDC_RETRY_WAIT_INITIAL_SECONDS, + max=OIDC_RETRY_WAIT_MAX_SECONDS, + jitter=OIDC_RETRY_WAIT_JITTER_SECONDS, + ), + retry=retry_if_exception_type(OIDCError), + ) async def _resolve_jwks_url(self) -> str: """Return the JWKS URL — explicit override wins, else discovery.""" if self._explicit_jwks_url is not None: @@ -156,8 +221,7 @@ async def _resolve_jwks_url(self) -> str: return self._discovery.jwks_uri url = f"{self._issuer}/.well-known/openid-configuration" try: - async with httpx.AsyncClient(timeout=self._discovery_timeout) as client: - response = await client.get(url) + response = await self._client.get(url) response.raise_for_status() doc = response.json() except Exception as exc: @@ -168,8 +232,20 @@ async def _resolve_jwks_url(self) -> str: self._discovery = _DiscoveryCache(jwks_uri=jwks_uri, fetched_at=time.monotonic()) return jwks_uri + async def aclose(self) -> None: + """Close the underlying HTTP client. Call once when the verifier is no longer needed.""" + await self._client.aclose() + async def _ensure_jwks_client(self) -> PyJWKClient: - url = await self._resolve_jwks_url() + try: + url = await self._resolve_jwks_url() + except CircuitBreakerError as exc: + # The breaker on `_resolve_jwks_url` is open (too many recent + # discovery failures) — translate to `OIDCError` so `verify`'s + # caller (the HTTP auth middleware) only ever needs to catch + # `OIDCError`, tripped breaker or not (see `http_runtime.py`'s + # `except OIDCError` around `verifier.verify`). + raise OIDCError(f"OIDC JWKS resolution failed: circuit open ({exc})") from exc # PyJWKClient caches keys in-process; reuse the same client # for the JWKS-URL lifetime. Recreate when the URL changes # (e.g. discovery doc rotated). diff --git a/src/mcpg/otel_tracing.py b/src/mcpg/otel_tracing.py index eb0c2f21..4ffb417c 100644 --- a/src/mcpg/otel_tracing.py +++ b/src/mcpg/otel_tracing.py @@ -193,7 +193,8 @@ def setup_tracing(settings: Settings) -> TracerHandle | None: except ImportError: _logger.warning( "OpenTelemetry is enabled but the OTLP HTTP exporter is not installed. " - "Spans will be created but not exported." + "Spans will be created but not exported.", + exc_info=True, ) # ``trace.set_tracer_provider`` is set-once-per-process; calling it diff --git a/src/mcpg/partman.py b/src/mcpg/partman.py index 74cd519e..b3c6250d 100644 --- a/src/mcpg/partman.py +++ b/src/mcpg/partman.py @@ -13,6 +13,7 @@ from dataclasses import dataclass +from mcpg.errors import MCPgError from mcpg.extensions import extension_installed from mcpg.sql import SqlDriver @@ -22,7 +23,7 @@ _PARTITION_TYPES = frozenset({"range", "list", "native"}) -class PartmanError(Exception): +class PartmanError(MCPgError): """Raised when a pg_partman operation cannot complete.""" diff --git a/src/mcpg/pg19_ddl.py b/src/mcpg/pg19_ddl.py index 1209c498..9f9cb88e 100644 --- a/src/mcpg/pg19_ddl.py +++ b/src/mcpg/pg19_ddl.py @@ -39,13 +39,14 @@ from dataclasses import dataclass from mcpg.database import Database +from mcpg.errors import MCPgError from mcpg.sql import SqlDriver # PG 19 ships the new pg_get_*def() functions. The version-num boundary. _MIN_PG19_DDL_VERSION = 190000 -class Pg19DdlError(Exception): +class Pg19DdlError(MCPgError): """Raised when a PG 19 DDL helper operation cannot complete.""" diff --git a/src/mcpg/pg19_partitions.py b/src/mcpg/pg19_partitions.py index 6047f3c6..a7178a44 100644 --- a/src/mcpg/pg19_partitions.py +++ b/src/mcpg/pg19_partitions.py @@ -59,13 +59,14 @@ from dataclasses import dataclass from mcpg.database import Database +from mcpg.errors import MCPgError from mcpg.sql import SqlDriver # PG 19 ships both forms. The version-num boundary. _MIN_PG19_PARTITIONS_VERSION = 190000 -class Pg19PartitionsError(Exception): +class Pg19PartitionsError(MCPgError): """Raised when a PG 19 partition reorganisation cannot complete.""" diff --git a/src/mcpg/pg19_runtime.py b/src/mcpg/pg19_runtime.py index 4a6bf601..635e50f2 100644 --- a/src/mcpg/pg19_runtime.py +++ b/src/mcpg/pg19_runtime.py @@ -42,6 +42,7 @@ from dataclasses import dataclass from mcpg.database import Database +from mcpg.errors import MCPgError from mcpg.sql import SqlDriver # PG 19 ships both toggles. The version-num probe is the boundary — @@ -53,7 +54,7 @@ _VALID_WAL_LEVELS = frozenset({"minimal", "replica", "logical"}) -class Pg19RuntimeError(Exception): +class Pg19RuntimeError(MCPgError): """Raised when a PG 19 runtime-toggle operation cannot complete.""" diff --git a/src/mcpg/pg19_skip_scan.py b/src/mcpg/pg19_skip_scan.py index 934c79ae..558e94e9 100644 --- a/src/mcpg/pg19_skip_scan.py +++ b/src/mcpg/pg19_skip_scan.py @@ -46,6 +46,7 @@ from dataclasses import dataclass +from mcpg.errors import MCPgError from mcpg.sql import SqlDriver # PG 19 ships skip-scan as the planner default. The version-num boundary. @@ -63,7 +64,7 @@ _NDV_UNKNOWN = 0 -class Pg19SkipScanError(Exception): +class Pg19SkipScanError(MCPgError): """Raised when a skip-scan advisor operation cannot complete.""" @@ -128,7 +129,7 @@ async def _server_version(driver: SqlDriver) -> tuple[int, str]: return int(cells.get("ver_num") or 0), str(cells.get("ver") or "") -def _absolute_ndv(n_distinct: float | int | None, reltuples: float | int | None) -> int: +def _absolute_ndv(n_distinct: float | None, reltuples: float | None) -> int: """Normalise pg_stats.n_distinct to an absolute integer estimate. Per the PG docs: diff --git a/src/mcpg/pg19_stats.py b/src/mcpg/pg19_stats.py index c3211526..bdc0a78a 100644 --- a/src/mcpg/pg19_stats.py +++ b/src/mcpg/pg19_stats.py @@ -36,6 +36,7 @@ from dataclasses import dataclass, field +from mcpg.errors import MCPgError from mcpg.sql import SqlDriver # Both views landed in PG 19. The version-num probe is the boundary — @@ -56,7 +57,7 @@ _HIGH_WAIT_COUNT = 1_000 -class Pg19StatsError(Exception): +class Pg19StatsError(MCPgError): """Raised when a PG 19 stats operation cannot complete.""" diff --git a/src/mcpg/pg_prewarm.py b/src/mcpg/pg_prewarm.py index 08025926..d799df37 100644 --- a/src/mcpg/pg_prewarm.py +++ b/src/mcpg/pg_prewarm.py @@ -32,6 +32,7 @@ from dataclasses import dataclass, field +from mcpg.errors import MCPgError from mcpg.extensions import extension_installed from mcpg.sql import SqlDriver @@ -52,7 +53,7 @@ _AUTOWARM_JOB_NAME = "mcpg_autowarm" -class PrewarmError(Exception): +class PrewarmError(MCPgError): """Raised when a pg_prewarm operation cannot complete.""" diff --git a/src/mcpg/pg_search.py b/src/mcpg/pg_search.py index 3304da8e..688ed532 100644 --- a/src/mcpg/pg_search.py +++ b/src/mcpg/pg_search.py @@ -67,6 +67,7 @@ from typing import Any from mcpg.database import Database +from mcpg.errors import MCPgError from mcpg.extensions import extension_installed from mcpg.sql import SqlDriver @@ -76,7 +77,7 @@ _IDENTIFIER = re.compile(r"\A[A-Za-z_][A-Za-z0-9_]*\Z") -class PgSearchError(Exception): +class PgSearchError(MCPgError): """Raised when a pg_search operation cannot complete.""" @@ -366,7 +367,7 @@ def _validate_positive_int(name: str, value: int, *, allow_zero: bool = False) - raise PgSearchError(f"{name} must be an int {op} 0; got {value!r}") -def _validate_bool(value: bool, kind: str) -> None: +def _validate_bool(*, value: bool, kind: str) -> None: if not isinstance(value, bool): raise PgSearchError(f"{kind} must be a bool; got {value!r}") @@ -566,7 +567,7 @@ async def pg_search_run( _validate_identifier(schema, "schema") _validate_identifier(table, "table") _validate_identifier(key_field, "key_field") - _validate_bool(return_snippets, "return_snippets") + _validate_bool(value=return_snippets, kind="return_snippets") _validate_limit(limit) if not isinstance(query, str): raise PgSearchError(f"query must be str; got {type(query).__name__}") @@ -831,8 +832,8 @@ async def pg_search_parse_query( """ if not isinstance(query_string, str): raise PgSearchError(f"query_string must be str; got {type(query_string).__name__}") - _validate_bool(lenient, "lenient") - _validate_bool(conjunction_mode, "conjunction_mode") + _validate_bool(value=lenient, kind="lenient") + _validate_bool(value=conjunction_mode, kind="conjunction_mode") if not await extension_installed(driver, "pg_search"): raise PgSearchError("pg_search extension is not installed in this database") @@ -1275,7 +1276,13 @@ def _utc_iso_now() -> str: """ -async def create_pg_search_index( +# C901 rationale: per-identifier injection-defense validation (schema/table/ +# columns/index_name/key_field, each through _validate_identifier + +# _pg_quote_ident) plus independent bounds/type validation for each of the +# 13 documented bm25 reloptions before DDL construction -- the branching is +# the validation matrix itself; the docstring's own "Identifier safety" +# section is the reason each check stays separate and explicit. +async def create_pg_search_index( # noqa: C901 database: Database, schema: str, table: str, diff --git a/src/mcpg/pgq.py b/src/mcpg/pgq.py index dc5834f0..7827b475 100644 --- a/src/mcpg/pgq.py +++ b/src/mcpg/pgq.py @@ -52,6 +52,7 @@ from dataclasses import dataclass from typing import Any +from mcpg.errors import MCPgError from mcpg.sql import SqlDriver # SQL/PGQ landed in the PG 19 series. We use the version-num probe rather @@ -80,7 +81,7 @@ _DEFAULT_MAX_ROWS = 200 -class PgqError(Exception): +class PgqError(MCPgError): """Raised when a SQL/PGQ operation cannot complete.""" @@ -333,9 +334,7 @@ def _is_safe_pgq_query(query: str) -> bool: body = stripped.rstrip(";").strip() if ";" in body: return False - if not _GRAPH_TABLE_RE.search(body): - return False - return True + return bool(_GRAPH_TABLE_RE.search(body)) async def run_pgq( diff --git a/src/mcpg/pitr.py b/src/mcpg/pitr.py index ce06d6d5..2a97cc73 100644 --- a/src/mcpg/pitr.py +++ b/src/mcpg/pitr.py @@ -117,14 +117,21 @@ async def check_pitr_readiness(driver: SqlDriver) -> PitrReadinessReport: # 1. Continuous archiving. if archive.archiving_enabled and archive.healthy: - gates.append(PitrGate("archiving", True, f"archive_mode={archive.archive_mode}, healthy", "")) + gates.append( + PitrGate( + "archiving", + ok=True, + observed=f"archive_mode={archive.archive_mode}, healthy", + remediation="", + ) + ) elif not archive.archiving_enabled: gates.append( PitrGate( "archiving", - False, - f"archive_mode={archive.archive_mode}", - "Enable continuous archiving: set archive_mode = on and a working " + ok=False, + observed=f"archive_mode={archive.archive_mode}", + remediation="Enable continuous archiving: set archive_mode = on and a working " "archive_command / archive_library, then restart.", ) ) @@ -132,9 +139,9 @@ async def check_pitr_readiness(driver: SqlDriver) -> PitrReadinessReport: gates.append( PitrGate( "archiving", - False, - "archive_mode on but archiver is failing", - "Fix the failing archive_command / archive_library " + ok=False, + observed="archive_mode on but archiver is failing", + remediation="Fix the failing archive_command / archive_library " "(see get_wal_archive_status) before relying on PITR.", ) ) @@ -144,9 +151,9 @@ async def check_pitr_readiness(driver: SqlDriver) -> PitrReadinessReport: gates.append( PitrGate( "wal_level", - wal_ok, - wal_level, - "" + ok=wal_ok, + observed=wal_level, + remediation="" if wal_ok else "Set wal_level = replica (or logical) and restart — minimal omits records PITR replay needs.", ) @@ -157,9 +164,11 @@ async def check_pitr_readiness(driver: SqlDriver) -> PitrReadinessReport: gates.append( PitrGate( "base_backup_capable", - senders_ok, - f"max_wal_senders={max_wal_senders}", - "" if senders_ok else "Set max_wal_senders >= 1 (and restart) so pg_basebackup can stream a base backup.", + ok=senders_ok, + observed=f"max_wal_senders={max_wal_senders}", + remediation="" + if senders_ok + else "Set max_wal_senders >= 1 (and restart) so pg_basebackup can stream a base backup.", ) ) @@ -168,9 +177,9 @@ async def check_pitr_readiness(driver: SqlDriver) -> PitrReadinessReport: gates.append( PitrGate( "full_page_writes", - fpw_ok, - full_page_writes, - "" + ok=fpw_ok, + observed=full_page_writes, + remediation="" if fpw_ok else "Set full_page_writes = on — recovery replay can hit torn " "pages otherwise (unless the storage guarantees atomic 8kB writes).", diff --git a/src/mcpg/policy.py b/src/mcpg/policy.py index 7a2bd560..8a94de2f 100644 --- a/src/mcpg/policy.py +++ b/src/mcpg/policy.py @@ -10,6 +10,7 @@ from enum import StrEnum from mcpg.config import AccessMode +from mcpg.errors import MCPgError class Capability(StrEnum): @@ -61,7 +62,7 @@ def is_permitted(access_mode: AccessMode, capability: Capability) -> bool: return capability in _PERMITTED[access_mode] -class PermissionError(Exception): +class PermissionError(MCPgError): """Raised when an operation is requested but the access mode does not permit it.""" diff --git a/src/mcpg/prisma.py b/src/mcpg/prisma.py index e898f689..c5e1590b 100644 --- a/src/mcpg/prisma.py +++ b/src/mcpg/prisma.py @@ -29,6 +29,7 @@ import re from collections.abc import Iterable +from mcpg.errors import MCPgError from mcpg.introspection import ( ColumnInfo, EnumInfo, @@ -90,7 +91,7 @@ _LITERAL_DEFAULT = re.compile(r"^'((?:[^']|'')*)'::") -class PrismaError(Exception): +class PrismaError(MCPgError): """Raised when a Prisma schema cannot be emitted.""" @@ -306,7 +307,12 @@ def _disambiguate_relation_names(pairs: Iterable[tuple[str, ForeignKeyInfo]]) -> return names -async def generate_prisma_schema(driver: SqlDriver, schema: str) -> str: +# C901 rationale: same code-generator family as diesel.py / sqlc.py, but at +# complexity 23 -- Prisma's relation model needs both forward FKs and +# synthesized back-relations (`_back_relations_by_target`) rendered inline +# per column, which threads more state through the per-table loop than the +# diesel/sqlc cheap-win shape allows without a larger restructuring. +async def generate_prisma_schema(driver: SqlDriver, schema: str) -> str: # noqa: C901 """Emit a Prisma schema string covering the base tables of ``schema``. Views, foreign tables, partitions, triggers, functions, policies, diff --git a/src/mcpg/py.typed b/src/mcpg/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/src/mcpg/query.py b/src/mcpg/query.py index 345ae15e..5e4a880a 100644 --- a/src/mcpg/query.py +++ b/src/mcpg/query.py @@ -15,6 +15,7 @@ from dataclasses import dataclass from typing import Any +from mcpg.errors import MCPgError from mcpg.sql import SafeSqlDriver, SqlDriver # Default per-query execution timeout, in seconds. @@ -34,7 +35,7 @@ _MEMORY_CAP_KB = 2 * 1024 * 1024 # 2 GiB expressed in kB -class QueryError(Exception): +class QueryError(MCPgError): """Raised when a query is rejected as unsafe or fails to execute.""" @@ -104,7 +105,8 @@ async def run_select( driver: SqlDriver, sql: str, *, - timeout: float = DEFAULT_TIMEOUT_SECONDS, + # ASYNC109 rationale: forwarded to SafeSqlDriver, which already wraps execution in asyncio.timeout() itself. + timeout: float = DEFAULT_TIMEOUT_SECONDS, # noqa: ASYNC109 max_rows: int = DEFAULT_MAX_ROWS, ) -> QueryResult: """Validate and execute a read-only SQL query. @@ -126,8 +128,14 @@ async def run_select( safe_driver = SafeSqlDriver(sql_driver=driver, timeout=timeout) try: - # SafeSqlDriver parses and validates this runtime SQL before running it. - rows = await safe_driver.execute_query(sql) + # SafeSqlDriver parses and validates this runtime SQL before running + # it. row_limit bounds how many rows are converted to Python objects + # (fetchmany(max_rows + 1) instead of fetchall()) instead of + # materializing the whole result set. Note: psycopg's client-side + # cursor already pulls the full result set into libpq's buffer + # during cursor.execute(), so this bounds Python-side object + # allocation, not the server-side network transfer. + rows = await safe_driver.execute_query(sql, row_limit=max_rows + 1) except Exception as exc: if _is_timeout_exc(exc): raise QueryTimeoutError(str(exc)) from exc @@ -176,7 +184,11 @@ async def run_select_tuned( *, work_mem: str, maintenance_work_mem: str | None = None, - timeout: float = DEFAULT_TIMEOUT_SECONDS, + # ASYNC109 rationale: SafeSqlDriver is only used here for its pglast + # validator (._validate), not for execution -- the timeout= passed to + # it is dead on this path. Real enforcement is the asyncio.wait_for(..., + # timeout=timeout) around the actual execute_query call below. + timeout: float = DEFAULT_TIMEOUT_SECONDS, # noqa: ASYNC109 max_rows: int = DEFAULT_MAX_ROWS, ) -> QueryResult: """Run a read-only SELECT with an elevated, bounded ``work_mem``. @@ -229,9 +241,12 @@ async def run_select_tuned( try: # One call so SET LOCAL and the SELECT share a transaction; # force_readonly wraps it in BEGIN READ ONLY. Reinstate the timeout - # bound the raw driver doesn't apply itself. + # bound the raw driver doesn't apply itself. row_limit bounds how + # many rows are converted to Python objects (max_rows + 1) instead + # of materializing everything — see run_select's comment above for + # the client-side-cursor caveat. rows = await asyncio.wait_for( - driver.execute_query(tuned_sql, force_readonly=True), + driver.execute_query(tuned_sql, force_readonly=True, row_limit=max_rows + 1), timeout=timeout, ) except Exception as exc: @@ -262,7 +277,8 @@ async def explain_query( driver: SqlDriver, sql: str, *, - timeout: float = DEFAULT_TIMEOUT_SECONDS, + # ASYNC109 rationale: forwarded to SafeSqlDriver / asyncio.wait_for at the execution boundary below. + timeout: float = DEFAULT_TIMEOUT_SECONDS, # noqa: ASYNC109 io: bool = False, ) -> ExplainResult: """Return the PostgreSQL execution plan for a query. @@ -392,7 +408,7 @@ async def analyze_query_plan( driver: SqlDriver, sql: str, *, - timeout: float = DEFAULT_TIMEOUT_SECONDS, + timeout: float = DEFAULT_TIMEOUT_SECONDS, # noqa: ASYNC109 -- forwarded to explain_query io: bool = False, ) -> QueryPlanAnalysis: """Summarise a query's execution plan into a structured analysis. @@ -484,7 +500,8 @@ async def run_select_parallel( driver: SqlDriver, statements: list[str], *, - timeout: float = DEFAULT_TIMEOUT_SECONDS, + # ASYNC109 rationale: forwarded per-statement to run_select, which owns the actual timeout enforcement. + timeout: float = DEFAULT_TIMEOUT_SECONDS, # noqa: ASYNC109 max_rows: int = DEFAULT_MAX_ROWS, parallel_limit: int = DEFAULT_PARALLEL_LIMIT, ) -> ParallelQueryResult: diff --git a/src/mcpg/rag_efficiency.py b/src/mcpg/rag_efficiency.py index 989554cf..e0ffcfc0 100644 --- a/src/mcpg/rag_efficiency.py +++ b/src/mcpg/rag_efficiency.py @@ -49,6 +49,7 @@ from dataclasses import dataclass, field from typing import Any +from mcpg.errors import MCPgError from mcpg.extensions import extension_installed from mcpg.sql import SqlDriver @@ -102,7 +103,7 @@ _THRESHOLD_PRUNING_INEFFECTIVE = 0.10 -class VectorEfficiencyError(Exception): +class VectorEfficiencyError(MCPgError): """Raised when a vector-efficiency analysis cannot complete.""" @@ -675,7 +676,13 @@ async def _approx_top_k_turboquant( # --- main entry point ------------------------------------------------------ -async def analyze_vector_search_efficiency( +# C901 rationale: identifier + numeric-bound validation matrix (schema/ +# table/column/id_column/metric/k/sample_size/candidate_multipliers, each +# with its own VectorEfficiencyError per the docstring's Raises list), +# followed by the ANN recall-sweep algorithm itself -- both halves are +# individually simple but independent, and inlining is what keeps each +# precondition's error message specific. +async def analyze_vector_search_efficiency( # noqa: C901 driver: SqlDriver, schema: str, table: str, @@ -929,7 +936,12 @@ async def _detect_single_column_pk(driver: SqlDriver, schema: str, table: str) - return str(rows[0].cells["pk_column"]) -async def audit_vector_indexes(driver: SqlDriver) -> Any: +# C901 rationale: per-index audit loop where a missing PK or a per-index +# efficiency-sweep failure must skip-and-continue rather than sink the whole +# scorecard category (see the docstring: "the audit reports what it can, +# not what it can't") -- same degrade-gracefully shape as the audit_* +# functions in mcpg/audit.py. +async def audit_vector_indexes(driver: SqlDriver) -> Any: # noqa: C901 """Scorecard adapter — returns a CategoryResult or None. Returns ``None`` when there are no ANN indexes (HNSW / IVFFlat / diff --git a/src/mcpg/rag_telemetry.py b/src/mcpg/rag_telemetry.py index b237ceab..b2741db1 100644 --- a/src/mcpg/rag_telemetry.py +++ b/src/mcpg/rag_telemetry.py @@ -38,6 +38,7 @@ from mcpg.audit_nl2sql import _check_interval as _shared_check_interval from mcpg.audit_nl2sql import detect_backend as _detect_backend from mcpg.database import Database +from mcpg.errors import MCPgError from mcpg.sql import SqlDriver _SCHEMA_NAME = "mcpg_rag" @@ -109,7 +110,7 @@ ) -class RagTelemetryError(Exception): +class RagTelemetryError(MCPgError): """Raised when a RAG telemetry operation cannot complete.""" @@ -634,7 +635,13 @@ def _percentile(values: list[float], q: float) -> float: return s[lo] + (s[hi] - s[lo]) * (pos - lo) -async def recommend_efficiency_thresholds( +# C901 rationale: 3 independent optional filters (backend/metric/k) each +# validated then conditionally appended to the WHERE clause + params list, +# followed by 3 independent percentile-vs-corpus-size fallback decisions +# (recall/spearman/pruning) -- both halves are per-field repetition of the +# same small pattern, not entangled logic, but there are enough fields that +# the count adds up. +async def recommend_efficiency_thresholds( # noqa: C901 driver: SqlDriver, *, days: int = 30, @@ -994,10 +1001,7 @@ async def _rag_data_range(driver: SqlDriver, spec: _RagTableSpec) -> tuple[datet def _rag_native_monthly_partition_sql(spec: _RagTableSpec, month_start: datetime) -> str: start = month_start.replace(day=1, hour=0, minute=0, second=0, microsecond=0) - if start.month == 12: - end = start.replace(year=start.year + 1, month=1) - else: - end = start.replace(month=start.month + 1) + end = start.replace(year=start.year + 1, month=1) if start.month == 12 else start.replace(month=start.month + 1) return ( f"CREATE TABLE IF NOT EXISTS {_SCHEMA_NAME}.{spec.table}_p{start.strftime('%Y%m')} " f"PARTITION OF {_SCHEMA_NAME}.{spec.table} " diff --git a/src/mcpg/redis_fdw.py b/src/mcpg/redis_fdw.py index e6432603..39e67b1d 100644 --- a/src/mcpg/redis_fdw.py +++ b/src/mcpg/redis_fdw.py @@ -37,6 +37,7 @@ from collections.abc import Mapping from dataclasses import dataclass, field +from mcpg.errors import MCPgError from mcpg.extensions import extension_installed from mcpg.introspection import _parse_options from mcpg.secrets import SecretsProvider @@ -62,7 +63,7 @@ _IDENT_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") -class RedisFdwError(Exception): +class RedisFdwError(MCPgError): """Raised when a redis_fdw operation cannot complete.""" diff --git a/src/mcpg/repack.py b/src/mcpg/repack.py index a78f4df1..55ead03f 100644 --- a/src/mcpg/repack.py +++ b/src/mcpg/repack.py @@ -40,13 +40,14 @@ from dataclasses import dataclass from mcpg.database import Database +from mcpg.errors import MCPgError from mcpg.sql import SqlDriver # PG 19 ships REPACK; older versions don't recognise the keyword. _MIN_REPACK_VERSION = 190000 -class RepackError(Exception): +class RepackError(MCPgError): """Raised when a REPACK request is rejected or fails.""" diff --git a/src/mcpg/replicas.py b/src/mcpg/replicas.py index bf075ef8..5ec8cbdd 100644 --- a/src/mcpg/replicas.py +++ b/src/mcpg/replicas.py @@ -31,12 +31,14 @@ from __future__ import annotations import asyncio +import contextlib import itertools import logging import time from dataclasses import dataclass -from typing import Any +from typing import Any, Self +from mcpg.errors import MCPgError from mcpg.sql import DbConnPool, SqlDriver, obfuscate_password from mcpg.tenancy import TenantSqlDriver @@ -49,7 +51,7 @@ DEFAULT_DEGRADED_RETRY_SECONDS = 30.0 -class ReplicaError(Exception): +class ReplicaError(MCPgError): """Raised when replica configuration is invalid.""" @@ -148,7 +150,7 @@ async def close(self) -> None: try: await state.pool.close() except Exception as exc: - logger.warning("Error closing replica %d pool: %s", state.index, exc) + logger.warning("Error closing replica %d pool: %s", state.index, exc, exc_info=True) async def next_healthy(self) -> _ReplicaState | None: """Return the next healthy replica, or ``None`` if all are degraded. @@ -192,7 +194,7 @@ async def snapshot(self) -> list[ReplicaInfo]: for state in self._states ] - async def __aenter__(self) -> ReplicaPool: + async def __aenter__(self) -> Self: await self.connect() return self @@ -219,18 +221,20 @@ async def _execute_with_connection( # type: ignore[no-untyped-def] connection, query, params, + *, force_readonly, + row_limit=None, ): if not getattr(connection, "_timeouts_configured", False): async with connection.cursor() as cursor: await cursor.execute( f"SET statement_timeout = {self._statement_timeout_ms}; SET lock_timeout = {self._lock_timeout_ms}" ) - try: + with contextlib.suppress(AttributeError): connection._timeouts_configured = True - except AttributeError: - pass - return await super()._execute_with_connection(connection, query, params, force_readonly) + return await super()._execute_with_connection( + connection, query, params, force_readonly=force_readonly, row_limit=row_limit + ) class TenantTimeoutSqlDriver(TenantSqlDriver): @@ -252,18 +256,20 @@ async def _execute_with_connection( # type: ignore[no-untyped-def] connection, query, params, + *, force_readonly, + row_limit=None, ): if not getattr(connection, "_timeouts_configured", False): async with connection.cursor() as cursor: await cursor.execute( f"SET statement_timeout = {self._statement_timeout_ms}; SET lock_timeout = {self._lock_timeout_ms}" ) - try: + with contextlib.suppress(AttributeError): connection._timeouts_configured = True - except AttributeError: - pass - return await super()._execute_with_connection(connection, query, params, force_readonly) # type: ignore[no-untyped-call] + return await super()._execute_with_connection( # type: ignore[no-untyped-call] + connection, query, params, force_readonly=force_readonly, row_limit=row_limit + ) def _make_driver_for_pool( @@ -327,7 +333,9 @@ async def execute_query( self, query: str, params: list[Any] | None = None, + *, force_readonly: bool = False, + row_limit: int | None = None, ) -> list[SqlDriver.RowResult] | None: from mcpg.observability import get_metrics @@ -335,17 +343,19 @@ async def execute_query( if not force_readonly: metrics.record_call("__replica_route", "primary", 0.0) - return await self._primary.execute_query(query, params, force_readonly) + return await self._primary.execute_query(query, params, force_readonly=force_readonly, row_limit=row_limit) candidate = await self._replica_pool.next_healthy() if candidate is None: # Every replica degraded — fall through to primary. metrics.record_call("__replica_route", "primary_no_healthy", 0.0) - return await self._primary.execute_query(query, params, force_readonly) + return await self._primary.execute_query(query, params, force_readonly=force_readonly, row_limit=row_limit) replica_driver = self._replicas[candidate.index] try: - result = await replica_driver.execute_query(query, params, force_readonly) + result = await replica_driver.execute_query( + query, params, force_readonly=force_readonly, row_limit=row_limit + ) except Exception as exc: await self._replica_pool.mark_degraded(candidate.index, str(exc)) metrics.record_call("__replica_route", "fallback", 0.0) @@ -354,7 +364,7 @@ async def execute_query( candidate.index, obfuscate_password(str(exc)), ) - return await self._primary.execute_query(query, params, force_readonly) + return await self._primary.execute_query(query, params, force_readonly=force_readonly, row_limit=row_limit) metrics.record_call("__replica_route", f"replica_{candidate.index}", 0.0) return result diff --git a/src/mcpg/resources.py b/src/mcpg/resources.py index 8519501d..162f0869 100644 --- a/src/mcpg/resources.py +++ b/src/mcpg/resources.py @@ -59,10 +59,11 @@ from typing import Any from mcpg.about import CAPABILITIES, build_capability_summary +from mcpg.errors import MCPgError from mcpg.introspection import get_compact_schema -class MCPgResourceError(Exception): +class MCPgResourceError(MCPgError): """Raised when a resource lookup fails.""" diff --git a/src/mcpg/rls.py b/src/mcpg/rls.py index 2d876043..22dd7490 100644 --- a/src/mcpg/rls.py +++ b/src/mcpg/rls.py @@ -24,6 +24,7 @@ from dataclasses import dataclass from typing import Any +from mcpg.errors import MCPgError from mcpg.sql import SqlDriver _IDENTIFIER = re.compile(r"\A[A-Za-z_][A-Za-z0-9_]*\Z") @@ -31,7 +32,7 @@ DEFAULT_RLS_SAMPLE_SIZE = 25 -class RLSError(Exception): +class RLSError(MCPgError): """Raised when an RLS-tester input fails validation or execution fails.""" diff --git a/src/mcpg/schema_docs.py b/src/mcpg/schema_docs.py index 53775d0e..19b4e40d 100644 --- a/src/mcpg/schema_docs.py +++ b/src/mcpg/schema_docs.py @@ -76,7 +76,13 @@ def _escape_cell(val: Any) -> str: return s.replace("\n", "
") -async def generate_schema_docs(driver: SqlDriver, schema: str, *, include_samples: bool = False) -> str: +# C901 rationale: complexity 42, the second-highest in the repo -- a +# comprehensive Markdown doc-generator covering tables/views/foreign tables/ +# enums/comments/FKs/optional row-sampling in one pass. A safe extraction +# into per-section helpers is plausible (same family as diesel.py/sqlc.py) +# but at this size and branching depth it needs a dedicated refactor pass +# with its own test-diffing, not a cheap-win within this lint sweep. +async def generate_schema_docs(driver: SqlDriver, schema: str, *, include_samples: bool = False) -> str: # noqa: C901 """Build a detailed Markdown reference for all objects in ``schema``. Optionally samples the first 10 rows of each base table to extract up to 3 diff --git a/src/mcpg/secrets.py b/src/mcpg/secrets.py index 6f330ba2..dfe48b3a 100644 --- a/src/mcpg/secrets.py +++ b/src/mcpg/secrets.py @@ -41,12 +41,15 @@ from collections import OrderedDict from collections.abc import Mapping from dataclasses import dataclass, field +from pathlib import Path from typing import Any, Protocol, runtime_checkable +from mcpg.errors import MCPgError + _SUPPORTED_BACKENDS = frozenset({"env", "file", "vault", "aws", "gcp"}) -class SecretsError(Exception): +class SecretsError(MCPgError): """Raised when the secrets backend is misconfigured or unreadable.""" @@ -97,10 +100,16 @@ def get(self, name: str) -> str | None: return self.env.get(name) -def _load_overlay(path: str) -> dict[str, str]: +# C901 rationale: YAML/JSON secrets-file parsing where every error message +# is deliberately hand-scoped to exclude the parser's own exception text +# (see the inline comments: YAML/JSON error internals "can echo source text +# and would leak secret values into logs") -- the branching is what keeps +# secret values out of error messages and logs; consolidating it risks +# reintroducing a secret-leak path. +def _load_overlay(path: str) -> dict[str, str]: # noqa: C901 """Load + validate a flat ``name -> value`` secrets file (JSON / YAML).""" try: - with open(path, encoding="utf-8") as handle: + with Path(path).open(encoding="utf-8") as handle: raw_text = handle.read() except OSError as exc: raise SecretsError(f"could not read MCPG_SECRETS_FILE_PATH ({path!r}): {exc}") from exc diff --git a/src/mcpg/server.py b/src/mcpg/server.py index 4954b441..c68b1c77 100644 --- a/src/mcpg/server.py +++ b/src/mcpg/server.py @@ -75,6 +75,12 @@ class AuditedMCPServer(MCPServer[AppContext]): rate_limiter: RateLimiter mcpg_settings: Settings + # The primary Database, so the HTTP transport's /readyz probe can read + # Database.is_connected without threading a new constructor parameter + # through create_server -> AuditedMCPServer -> build_http_app. Same + # "stash it directly on the server object" pattern as mcpg_settings / + # otel_tracer / rate_limiter below. + mcpg_database: Database in_flight_calls: int = 0 # OpenTelemetry tracer. ``None`` when MCPG_OTEL_ENABLED=false or # the ``mcpg[otel]`` extra isn't installed — :func:`tool_span` @@ -276,6 +282,14 @@ async def lifespan(_server: MCPServer[AppContext]) -> AsyncIterator[AppContext]: await cache_manager.close() + # NL→SQL providers share one process-wide httpx.AsyncClient + # (see mcpg.nl2sql._get_shared_http_client) rather than each + # opening a fresh client per translate_nl_to_sql call — close + # it out symmetrically so it doesn't leak past shutdown. + from mcpg.nl2sql import aclose_shared_client + + await aclose_shared_client() + # Flush pending OTel spans so a clean shutdown doesn't # drop the last batch of traces. Tracer is process-wide # global but the provider hung off the server lets us @@ -348,6 +362,7 @@ def create_server( middleware=middleware, ) server.mcpg_settings = settings + server.mcpg_database = db server.otel_tracer = setup_tracing(settings) # Instantiate and register the RateLimiter server.rate_limiter = RateLimiter( diff --git a/src/mcpg/session_advisor.py b/src/mcpg/session_advisor.py index 14cfb4fd..3a0725a8 100644 --- a/src/mcpg/session_advisor.py +++ b/src/mcpg/session_advisor.py @@ -36,6 +36,7 @@ from dataclasses import dataclass, field +from mcpg.errors import MCPgError from mcpg.sql import SqlDriver # Tools that return small, well-defined slices of the catalogue — calling @@ -60,7 +61,7 @@ REASON_IDLE_SESSION = "idle_session" -class SessionAdvisorError(Exception): +class SessionAdvisorError(MCPgError): """Raised when the advisor can't run — e.g. the audit table is missing.""" diff --git a/src/mcpg/session_intent.py b/src/mcpg/session_intent.py index 766ca29a..13e8ce4d 100644 --- a/src/mcpg/session_intent.py +++ b/src/mcpg/session_intent.py @@ -245,9 +245,7 @@ def resolved_tool_names( """ kept: set[str] = set() for name in candidate_names: - if name in always_keep or name in resolution.tool_names: - kept.add(name) - elif classify_tool(name) in resolution.buckets: + if name in always_keep or name in resolution.tool_names or classify_tool(name) in resolution.buckets: kept.add(name) return frozenset(kept) diff --git a/src/mcpg/shell.py b/src/mcpg/shell.py index a7091e70..06262160 100644 --- a/src/mcpg/shell.py +++ b/src/mcpg/shell.py @@ -33,8 +33,11 @@ from collections.abc import Callable, Iterator from contextlib import contextmanager from dataclasses import dataclass, field +from pathlib import Path from typing import Final +from mcpg.errors import MCPgError + try: # POSIX-only; absent on Windows. Guarded so the import never breaks startup. import resource except ImportError: # pragma: no cover - exercised only on non-POSIX platforms @@ -68,7 +71,7 @@ _REDACTED_VALUE = "****" -class ShellError(Exception): +class ShellError(MCPgError): """Raised when a subprocess invocation is rejected or fails.""" @@ -140,7 +143,7 @@ def _resolve_binary(name: str, bin_allowlist: tuple[str, ...] = ()) -> str: # e.g. /usr/bin/pg_dump -> pg_wrapper outside the bin dir. We only # normalise the directory for symlinks so a PATH shim in an # untrusted dir is still rejected. - resolved_dir = os.path.realpath(os.path.dirname(resolved)) + resolved_dir = os.path.realpath(Path(resolved).parent) allowed = {os.path.realpath(d) for d in bin_allowlist} if resolved_dir not in allowed: raise ShellError( @@ -218,7 +221,14 @@ def _filter_env(env: dict[str, str] | None) -> dict[str, str]: return merged -async def run_pg_binary( +# C901 rationale: the SHELL-capability subprocess-hardening path -- binary +# allowlisting, env-var filtering (drops everything but allowlisted PG*/ +# LANG/LC_ALL/PATH so the child can't inherit a polluted environment), +# sandboxed throwaway workdir, preexec rlimits, timeout-kill, and +# output-byte capping all happen in one call because they're one security +# boundary; splitting the checks apart doesn't reduce what must be gotten +# right for a subprocess spawn to stay sandboxed. +async def run_pg_binary( # noqa: C901 binary: str, *argv: str, env: dict[str, str] | None = None, diff --git a/src/mcpg/sql/driver.py b/src/mcpg/sql/driver.py index ac357b9a..53b55231 100644 --- a/src/mcpg/sql/driver.py +++ b/src/mcpg/sql/driver.py @@ -135,9 +135,8 @@ async def pool_connect(self, connection_url: str | None = None) -> AsyncConnecti await self.pool.open() # Prove the pool works before handing it out. - async with self.pool.connection() as conn: - async with conn.cursor() as cursor: - await cursor.execute("SELECT 1") + async with self.pool.connection() as conn, conn.cursor() as cursor: + await cursor.execute("SELECT 1") self._is_valid = True self._last_error = None @@ -210,7 +209,9 @@ async def execute_query( self, query: LiteralString, params: list[Any] | None = None, + *, force_readonly: bool = False, + row_limit: int | None = None, ) -> list[RowResult] | None: """Run ``query`` and return its rows (or ``None`` for no result set). @@ -219,6 +220,12 @@ async def execute_query( connection is checked out per call; on error the pool is marked invalid (or a direct connection is dropped) and the exception re-raised. + + ``row_limit``, when given, bounds the fetch itself: at most + ``row_limit`` rows are pulled from the cursor via ``fetchmany`` + instead of materializing the entire result set with ``fetchall``. + ``None`` (the default) preserves the historical full-fetch + behaviour for callers that rely on it. """ try: if self.conn is None: @@ -229,9 +236,11 @@ async def execute_query( if self.is_pool: # pragma: no cover - real pool checkout; integration-tested pool = await self.conn.pool_connect() async with pool.connection() as connection: - return await self._execute_with_connection(connection, query, params, force_readonly=force_readonly) + return await self._execute_with_connection( + connection, query, params, force_readonly=force_readonly, row_limit=row_limit + ) return await self._execute_with_connection( # pragma: no cover - real connection; integration-tested - self.conn, query, params, force_readonly=force_readonly + self.conn, query, params, force_readonly=force_readonly, row_limit=row_limit ) except Exception as e: # A connection-level failure invalidates the pool / drops the conn. @@ -242,8 +251,15 @@ async def execute_query( self.conn = None raise - async def _execute_with_connection( # pragma: no cover - real psycopg execution; integration-tested - self, connection: Any, query: Any, params: Any, force_readonly: bool + # C901 rationale: part of the first-party SQL-safety kernel (CLAUDE.md: + # "sql/driver.py -- pool + execution + credential redaction"). The + # branching here is the transaction start/commit/rollback state machine + # plus the obfuscate_password-sanitised error-logging path (see the + # inline CodeQL note below, verified against + # tests/unit/test_sql_kernel_driver.py) -- restructuring it risks a + # dropped rollback or a credential-leak regression for no benefit. + async def _execute_with_connection( # pragma: no cover - real psycopg execution; integration-tested # noqa: C901 + self, connection: Any, query: Any, params: Any, *, force_readonly: bool, row_limit: int | None = None ) -> list[RowResult] | None: """Execute on a specific connection with read-only + txn handling.""" transaction_started = False @@ -270,7 +286,7 @@ async def _execute_with_connection( # pragma: no cover - real psycopg execution transaction_started = False return None - rows = await cursor.fetchall() + rows = await cursor.fetchmany(row_limit) if row_limit is not None else await cursor.fetchall() if not force_readonly: await cursor.execute("COMMIT") diff --git a/src/mcpg/sql/safety.py b/src/mcpg/sql/safety.py index 05eca4d1..4fe23849 100644 --- a/src/mcpg/sql/safety.py +++ b/src/mcpg/sql/safety.py @@ -72,7 +72,15 @@ def __init__(self, sql_driver: SqlDriver, timeout: float | None = None) -> None: self.sql_driver = sql_driver self.timeout = timeout - def _validate_node(self, node: Node) -> None: + # C901 rationale: the recursive pglast AST walker enforcing the + # mcpg.sql.allowlist policy. Per this module's own docstring, it is + # "re-authored from the vendored crystaldba/postgres-mcp safe_sql.py + # (MIT); behaviour is pinned identical by the adversarial suite + + # differential parity test" — a security-critical, fuzz-tested, + # adversarially-pinned function where restructuring the branching is + # pure risk (any behavioural drift is a SQL-safety regression) for no + # safety benefit. + def _validate_node(self, node: Node) -> None: # noqa: C901 """Recursively validate a node and all of its children.""" if not isinstance(node, tuple(self.ALLOWED_NODE_TYPES)): raise ValueError(f"Node type {type(node)} is not allowed") @@ -111,9 +119,8 @@ def _validate_node(self, node: Node) -> None: raise ValueError("EXPLAIN ANALYZE is not supported") # CREATE EXTENSION only for allowlisted extensions. - if isinstance(node, CreateExtensionStmt): - if node.extname not in self.ALLOWED_EXTENSIONS: - raise ValueError(f"CREATE EXTENSION {node.extname} is not supported") + if isinstance(node, CreateExtensionStmt) and node.extname not in self.ALLOWED_EXTENSIONS: + raise ValueError(f"CREATE EXTENSION {node.extname} is not supported") # Recurse into every child node. pglast's concrete Node subclasses # carry their fields in ``__slots__``; the base ``Node`` type (typed @@ -128,11 +135,7 @@ def _validate_node(self, node: Node) -> None: except AttributeError: continue # normal in pglast - if isinstance(attr, list): - for item in attr: - if isinstance(item, Node): - self._validate_node(item) - elif isinstance(attr, tuple): + if isinstance(attr, (list, tuple)): for item in attr: if isinstance(item, Node): self._validate_node(item) @@ -167,9 +170,18 @@ async def execute_query( self, query: LiteralString, params: list[Any] | None = None, + *, force_readonly: bool = True, # ignored — SafeSqlDriver always forces read-only + row_limit: int | None = None, ) -> list[SqlDriver.RowResult] | None: - """Validate ``query`` is safe, then execute it read-only.""" + """Validate ``query`` is safe, then execute it read-only. + + ``row_limit``, when given, is threaded through to the wrapped + driver so the fetch itself is bounded (``fetchmany``) rather than + materializing the full result set before any caller-side + truncation runs. ``None`` preserves the historical full-fetch + behaviour. + """ self._validate(query) # Always force read-only regardless of the argument. @@ -180,6 +192,7 @@ async def execute_query( f"/* crystaldba */ {query}", params=params, force_readonly=True, + row_limit=row_limit, ) except TimeoutError as e: logger.warning("Query execution timed out after %s seconds: %s...", self.timeout, query[:100]) @@ -194,6 +207,7 @@ async def execute_query( f"/* crystaldba */ {query}", params=params, force_readonly=True, + row_limit=row_limit, ) @staticmethod diff --git a/src/mcpg/sqlalchemy_export.py b/src/mcpg/sqlalchemy_export.py index e62c8bd6..e4ac26bc 100644 --- a/src/mcpg/sqlalchemy_export.py +++ b/src/mcpg/sqlalchemy_export.py @@ -16,6 +16,7 @@ import re +from mcpg.errors import MCPgError from mcpg.introspection import ( ColumnInfo, ForeignKeyInfo, @@ -31,7 +32,7 @@ _SERIAL_DEFAULT_RE = re.compile(r"nextval\(['\"]([^'\"]+)['\"]") -class SqlAlchemyExportError(Exception): +class SqlAlchemyExportError(MCPgError): """Raised when a SQLAlchemy export call is rejected or fails.""" @@ -159,7 +160,13 @@ def _parse_columns_in_definition(definition: str, regex: re.Pattern[str]) -> lis return [col.strip().strip('"') for col in match.group(1).split(",")] -def _render_column( +# C901 rationale: builds a single Mapped[...]/mapped_column(...) line while +# accumulating 3 separate import sets (core/pg/typing) alongside the +# rendered SQLAlchemy args/kwargs -- each `if` contributes to shared local +# state (args, kwargs, and the 3 import sets), so extracting pieces would +# still need to thread all of that state back together; not the clean +# independent-steps shape that diesel.py/sqlc.py's cheap wins had. +def _render_column( # noqa: C901 column: ColumnInfo, *, schema: str, @@ -321,7 +328,12 @@ def _render_table_args(schema: str, composite_uniques: list[tuple[str, list[str] return f" __table_args__ = ({joined}, {schema_kwarg})" -async def generate_sqlalchemy_models(driver: SqlDriver, schema: str) -> str: +# C901 rationale: same code-generator family as diesel.py / sqlc.py, but at +# complexity 23 -- the 3 accumulated import sets (core/pg/typing) plus +# composite-FK and composite-unique special-casing thread state through the +# per-table loop in a way that resists the diesel/sqlc "extract a step" +# refactor without a larger restructuring. +async def generate_sqlalchemy_models(driver: SqlDriver, schema: str) -> str: # noqa: C901 """Emit a SQLAlchemy 2.0 declarative model file for ``schema``. Returns a Python source string. The model classes use diff --git a/src/mcpg/sqlc.py b/src/mcpg/sqlc.py index 4e4d548a..ea3e7699 100644 --- a/src/mcpg/sqlc.py +++ b/src/mcpg/sqlc.py @@ -22,8 +22,10 @@ import re +from mcpg.errors import MCPgError from mcpg.introspection import ( ColumnInfo, + TableInfo, describe_table, list_constraints, list_enums, @@ -36,7 +38,7 @@ _IDENTIFIER = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\Z") -class SqlcExportError(Exception): +class SqlcExportError(MCPgError): """Raised when an sqlc export call is rejected or fails.""" @@ -96,6 +98,51 @@ def _render_index(definition: str) -> str: _CONSTRAINT_ORDER = {"primary_key": 0, "unique": 1, "check": 2, "foreign_key": 3, "exclusion": 4} +async def _build_create_tables( + driver: SqlDriver, schema: str, tables: list[TableInfo] +) -> tuple[list[str], dict[str, list[ColumnInfo]]]: + """Step 3: ``CREATE TABLE`` statements (columns only) for each base table.""" + blocks: list[str] = [] + table_columns: dict[str, list[ColumnInfo]] = {} + for table in tables: + columns = await describe_table(driver, schema, table.name) + for col in columns: + _check_identifier(col.name, "column") + table_columns[table.name] = columns + blocks.append(_render_table(schema, table.name, columns)) + return blocks, table_columns + + +async def _build_constraints( + driver: SqlDriver, schema: str, tables: list[TableInfo] +) -> tuple[list[str], dict[str, list[tuple[str, str, str]]]]: + """Step 4: ``ALTER TABLE ADD CONSTRAINT``, ordered PK before FK.""" + constraint_blocks: list[str] = [] + constraints_by_table: dict[str, list[tuple[str, str, str]]] = {} + for table in tables: + cons = await list_constraints(driver, schema, table.name) + triples = [(c.type, c.name, c.definition) for c in cons] + triples.sort(key=lambda triple: (_CONSTRAINT_ORDER.get(triple[0], 9), triple[1])) + constraints_by_table[table.name] = triples + for _ctype, name, definition in triples: + constraint_blocks.append(_render_constraint(schema, table.name, name, definition)) + return constraint_blocks, constraints_by_table + + +async def _build_indexes( + driver: SqlDriver, schema: str, tables: list[TableInfo], constraints_by_table: dict[str, list[tuple[str, str, str]]] +) -> list[str]: + """Step 5: ``CREATE INDEX`` for indexes not created by PK / unique constraints.""" + blocks: list[str] = [] + for table in tables: + constraint_names = {name for _, name, _ in constraints_by_table[table.name]} + for idx in await list_indexes(driver, schema, table.name): + if idx.name in constraint_names: + continue + blocks.append(_render_index(idx.definition)) + return blocks + + async def generate_sqlc_schema(driver: SqlDriver, schema: str) -> str: """Emit a ``schema.sql`` for sqlc covering the base tables of ``schema``. @@ -129,25 +176,12 @@ async def generate_sqlc_schema(driver: SqlDriver, schema: str) -> str: blocks.append(_render_enum(enum.name, list(enum.values))) # 3. CREATE TABLE statements — columns only. - table_columns: dict[str, list[ColumnInfo]] = {} - for table in tables: - columns = await describe_table(driver, schema, table.name) - for col in columns: - _check_identifier(col.name, "column") - table_columns[table.name] = columns - blocks.append(_render_table(schema, table.name, columns)) + table_blocks, _table_columns = await _build_create_tables(driver, schema, tables) + blocks.extend(table_blocks) # 4. ALTER TABLE ADD CONSTRAINT, ordered by constraint type so PK lands # before FK (and unique indexes are created implicitly with PK/unique). - constraint_blocks: list[str] = [] - constraints_by_table: dict[str, list[tuple[str, str, str]]] = {} - for table in tables: - cons = await list_constraints(driver, schema, table.name) - triples = [(c.type, c.name, c.definition) for c in cons] - triples.sort(key=lambda triple: (_CONSTRAINT_ORDER.get(triple[0], 9), triple[1])) - constraints_by_table[table.name] = triples - for _ctype, name, definition in triples: - constraint_blocks.append(_render_constraint(schema, table.name, name, definition)) + constraint_blocks, constraints_by_table = await _build_constraints(driver, schema, tables) # FK constraints on intra-schema targets come back via list_constraints # already (they're table constraints, not free-floating). The @@ -159,11 +193,6 @@ async def generate_sqlc_schema(driver: SqlDriver, schema: str) -> str: blocks.extend(constraint_blocks) # 5. CREATE INDEX statements for indexes not created by PK / unique constraints. - for table in tables: - constraint_names = {name for _, name, _ in constraints_by_table[table.name]} - for idx in await list_indexes(driver, schema, table.name): - if idx.name in constraint_names: - continue - blocks.append(_render_index(idx.definition)) + blocks.extend(await _build_indexes(driver, schema, tables, constraints_by_table)) return "\n\n".join(blocks) + "\n" diff --git a/src/mcpg/tenancy.py b/src/mcpg/tenancy.py index c2a196ce..1e5032f4 100644 --- a/src/mcpg/tenancy.py +++ b/src/mcpg/tenancy.py @@ -43,6 +43,7 @@ from mcp.server.context import CallNext, HandlerResult, ServerMiddleware, ServerRequestContext from psycopg.rows import dict_row +from mcpg.errors import MCPgError from mcpg.sql import SqlDriver logger = logging.getLogger(__name__) @@ -65,7 +66,7 @@ _ROLE_SCOPE_KEY = "mcpg.tenant_role" -class TenancyError(ValueError): +class TenancyError(MCPgError, ValueError): """Raised when a role name fails validation.""" @@ -173,20 +174,34 @@ async def _execute_with_connection( # type: ignore[no-untyped-def] connection, query, params, + *, force_readonly, + row_limit=None, ): role = resolve_role(self._default_role) if role is None: - return await super()._execute_with_connection(connection, query, params, force_readonly) - return await _execute_with_role(connection, query, params, force_readonly, role) - - -async def _execute_with_role( + return await super()._execute_with_connection( + connection, query, params, force_readonly=force_readonly, row_limit=row_limit + ) + return await _execute_with_role( + connection, query, params, force_readonly=force_readonly, role=role, row_limit=row_limit + ) + + +# C901 rationale: multi-tenant RLS execution path -- role validation, +# explicit transaction lifecycle so `SET LOCAL ROLE` is valid on write +# paths too, and the same transaction-commit/rollback state machine as +# sql/driver.py's `_execute_with_connection` (mirrored intentionally, per +# the docstring) -- restructuring risks a tenant-isolation regression (the +# wrong role active, or a transaction left open) for no benefit. +async def _execute_with_role( # noqa: C901 connection: Any, query: str, params: Any, + *, force_readonly: bool, role: str, + row_limit: int | None = None, ) -> Any: """Run ``query`` inside an explicit transaction with ``SET LOCAL ROLE``. @@ -226,7 +241,7 @@ async def _execute_with_role( transaction_started = False return None - rows = await cursor.fetchall() + rows = await cursor.fetchmany(row_limit) if row_limit is not None else await cursor.fetchall() if force_readonly: await cursor.execute("ROLLBACK") else: @@ -242,5 +257,9 @@ async def _execute_with_role( # actually unwinds; never swallow it inside a fallback. raise except Exception as rollback_error: - logger.error("Error rolling back transaction during role-wrapped execute: %s", rollback_error) + logger.error( + "Error rolling back transaction during role-wrapped execute: %s", + rollback_error, + exc_info=True, + ) raise diff --git a/src/mcpg/test_data.py b/src/mcpg/test_data.py index 2f22784d..0d195682 100644 --- a/src/mcpg/test_data.py +++ b/src/mcpg/test_data.py @@ -32,6 +32,7 @@ from dataclasses import dataclass from datetime import UTC, datetime, timedelta +from mcpg.errors import MCPgError from mcpg.introspection import ColumnInfo, describe_table from mcpg.sql import SqlDriver @@ -41,7 +42,7 @@ HARD_ROW_CAP = 10_000 -class TestDataError(Exception): +class TestDataError(MCPgError): """Raised when test-data generation is rejected or fails.""" # Pytest treats classes named ``Test*`` as test collection targets; @@ -140,6 +141,22 @@ def _synth_value(column: ColumnInfo, rng: random.Random) -> object: ) +def _format_column_value(col: ColumnInfo, rng: random.Random) -> str: + """Render one column's value for an INSERT's ``VALUES`` list. + + ``DEFAULT`` when synthesis failed but PG can fill the column in + (has a default, or is NOT NULL with none — the agent sees the + resulting failure when the INSERT runs); ``NULL`` when synthesis + failed and the column is nullable; otherwise the quoted literal. + """ + value = _synth_value(col, rng) + if value is None: + if col.default is not None or not col.nullable: + return "DEFAULT" + return "NULL" + return _quote_literal(value) + + async def generate_test_data( driver: SqlDriver, schema: str, @@ -193,21 +210,7 @@ async def generate_test_data( column_list = ", ".join(f'"{c.name}"' for c in target_columns) statements: list[str] = [] for _ in range(rows): - values: list[str] = [] - for col in target_columns: - v = _synth_value(col, rng) - if v is None and col.default is not None: - # Has a default; let PG fill it in. - values.append("DEFAULT") - elif v is None and not col.nullable: - # Synthesis failed and column is NOT NULL with no default - # — fall back to a placeholder that's likely to fit. The - # agent will see the failure when the INSERT runs. - values.append("DEFAULT") - elif v is None: - values.append("NULL") - else: - values.append(_quote_literal(v)) + values = [_format_column_value(col, rng) for col in target_columns] statements.append(f'INSERT INTO "{schema}"."{table}" ({column_list}) VALUES ({", ".join(values)})') return GeneratedDataset( diff --git a/src/mcpg/test_row_factory.py b/src/mcpg/test_row_factory.py index 42a3fe3c..97299827 100644 --- a/src/mcpg/test_row_factory.py +++ b/src/mcpg/test_row_factory.py @@ -55,16 +55,18 @@ import re import string import uuid +from collections.abc import Callable from dataclasses import dataclass from datetime import UTC, datetime, timedelta +from mcpg.errors import MCPgError from mcpg.introspection import ColumnInfo, describe_table from mcpg.sql import SqlDriver _IDENTIFIER = re.compile(r"\A[A-Za-z_][A-Za-z0-9_]*\Z") -class TestRowFactoryError(Exception): +class TestRowFactoryError(MCPgError): """Raised when the factory is rejected or cannot produce a row.""" __test__ = False # opt out of pytest collection — class name starts with Test @@ -141,71 +143,144 @@ def _quote_literal(value: object) -> str: _CURRENCIES = ("USD", "EUR", "GBP", "JPY", "INR", "BRL", "ZAR", "AUD") +# Column-name heuristics, tried in order (first match wins). Time-bearing +# names must land first so e.g. ``last_login_at`` wins on ``_at`` rather +# than falling through to a less specific pattern. +_NAME_PATTERNS: list[tuple[Callable[[str], bool], Callable[[random.Random], tuple[object, str]]]] = [ + ( + lambda name: ( + name.endswith("_at") + or name in {"created", "updated", "last_seen", "last_login"} + or name.endswith("_timestamp") + ), + lambda rng: (datetime.now(UTC) - timedelta(seconds=rng.randint(0, 60 * 60 * 24 * 30)), "timestamp pattern"), + ), + ( + lambda name: name == "email" or name.endswith("_email"), + lambda rng: (f"user_{rng.randint(1, 9999)}@example.com", "email pattern"), + ), + ( + lambda name: name == "url" or name.endswith("_url") or name.endswith("_uri"), + lambda rng: (f"https://example.com/r/{rng.randint(1, 9999)}", "url pattern"), + ), + ( + lambda name: name == "phone" or name.endswith("_phone"), + lambda rng: (f"+1-555-{rng.randint(1000, 9999)}", "phone pattern"), + ), + ( + lambda name: name in {"country", "country_code"}, + lambda rng: (rng.choice(_COUNTRIES), "country pattern"), + ), + ( + lambda name: name in {"currency", "currency_code"}, + lambda rng: (rng.choice(_CURRENCIES), "currency pattern"), + ), + ( + lambda name: name in {"ip", "ip_address"}, + lambda rng: (f"192.0.2.{rng.randint(1, 254)}", "ip pattern (RFC 5737 docs range)"), + ), + ( + lambda name: name == "slug", + lambda rng: ( + "-".join(rng.choice(_NAME_FIRST) for _ in range(rng.randint(2, 3))), + "slug pattern", + ), + ), + ( + lambda name: name in {"first_name", "given_name"}, + lambda rng: (rng.choice(_NAME_FIRST), "first_name pattern"), + ), + ( + lambda name: name in {"last_name", "family_name", "surname"}, + lambda rng: (rng.choice(_NAME_LAST), "last_name pattern"), + ), + ( + lambda name: name in {"full_name", "name", "display_name"}, + lambda rng: (f"{rng.choice(_NAME_FIRST)} {rng.choice(_NAME_LAST)}", "name pattern"), + ), +] + + def _synth_by_name(col_name: str, rng: random.Random) -> tuple[object, str] | None: """Return ``(value, heuristic_label)`` if ``col_name`` matches a well-known pattern; otherwise ``None`` so the caller falls through to type-based synthesis.""" name = col_name.lower() - # Time-bearing names land first so ``last_login_at`` wins on ``_at``. - if name.endswith("_at") or name in {"created", "updated", "last_seen", "last_login"} or name.endswith("_timestamp"): - ts = datetime.now(UTC) - timedelta(seconds=rng.randint(0, 60 * 60 * 24 * 30)) - return ts, "timestamp pattern" - if name == "email" or name.endswith("_email"): - return f"user_{rng.randint(1, 9999)}@example.com", "email pattern" - if name == "url" or name.endswith("_url") or name.endswith("_uri"): - return f"https://example.com/r/{rng.randint(1, 9999)}", "url pattern" - if name == "phone" or name.endswith("_phone"): - return f"+1-555-{rng.randint(1000, 9999)}", "phone pattern" - if name in {"country", "country_code"}: - return rng.choice(_COUNTRIES), "country pattern" - if name in {"currency", "currency_code"}: - return rng.choice(_CURRENCIES), "currency pattern" - if name == "ip" or name == "ip_address": - return f"192.0.2.{rng.randint(1, 254)}", "ip pattern (RFC 5737 docs range)" - if name == "slug": - return "-".join(rng.choice(_NAME_FIRST) for _ in range(rng.randint(2, 3))), "slug pattern" - if name in {"first_name", "given_name"}: - return rng.choice(_NAME_FIRST), "first_name pattern" - if name in {"last_name", "family_name", "surname"}: - return rng.choice(_NAME_LAST), "last_name pattern" - if name in {"full_name", "name", "display_name"}: - return f"{rng.choice(_NAME_FIRST)} {rng.choice(_NAME_LAST)}", "name pattern" + for predicate, generate in _NAME_PATTERNS: + if predicate(name): + return generate(rng) return None +def _synth_text_value(column: ColumnInfo, rng: random.Random) -> tuple[object, str]: + # Honour the (N) length cap on varchar(N) / char(N) — generating + # a 4-16 char string into a varchar(2) lands a real "value too + # long" failure on the INSERT (gemini review on #178). + max_len: int | None = None + if "(" in column.data_type: + try: + max_len = int(column.data_type.split("(", 1)[1].split(")", 1)[0].strip()) + except (ValueError, IndexError): + max_len = None + lo, hi = 4, 16 + if max_len is not None: + hi = min(hi, max(1, max_len)) + lo = min(lo, hi) + length = rng.randint(lo, hi) + return "".join(rng.choice(string.ascii_lowercase) for _ in range(length)), "text type" + + +# Type-driven fallback synthesis, tried in order (first matching base-type +# set wins). Each generator receives the full column (needed for the +# varchar/char length cap) plus the shared RNG. +_TYPE_SYNTHESIZERS: list[tuple[frozenset[str], Callable[[ColumnInfo, random.Random], tuple[object, str]]]] = [ + ( + frozenset({"integer", "int", "int4", "int2", "smallint", "bigint", "int8"}), + lambda _column, rng: (rng.randint(1, 1_000_000), "int type"), + ), + ( + frozenset({"numeric", "decimal", "real", "double precision", "float4", "float8"}), + lambda _column, rng: (round(rng.random() * 1000, 2), "numeric type"), + ), + ( + frozenset({"boolean", "bool"}), + lambda _column, rng: (rng.choice([True, False]), "bool type"), + ), + ( + frozenset({"uuid"}), + lambda _column, rng: (str(uuid.UUID(int=rng.getrandbits(128))), "uuid type"), + ), + ( + frozenset({"date"}), + lambda _column, rng: ( + (datetime.now(UTC).date() - timedelta(days=rng.randint(0, 365))).isoformat(), + "date type", + ), + ), + ( + frozenset({"timestamp", "timestamp without time zone", "timestamptz", "timestamp with time zone"}), + lambda _column, rng: ( + datetime.now(UTC) - timedelta(seconds=rng.randint(0, 60 * 60 * 24 * 365)), + "timestamp type", + ), + ), + ( + frozenset({"text", "varchar", "character varying", "char", "character", "citext", "name"}), + _synth_text_value, + ), + ( + frozenset({"json", "jsonb"}), + lambda _column, rng: ('{"k": "v"}', "json type"), + ), +] + + def _synth_by_type(column: ColumnInfo, rng: random.Random) -> tuple[object, str] | None: """Fall-back type-driven synthesis.""" base = column.data_type.lower().split("(", 1)[0].strip() - if base in {"integer", "int", "int4", "int2", "smallint", "bigint", "int8"}: - return rng.randint(1, 1_000_000), "int type" - if base in {"numeric", "decimal", "real", "double precision", "float4", "float8"}: - return round(rng.random() * 1000, 2), "numeric type" - if base in {"boolean", "bool"}: - return rng.choice([True, False]), "bool type" - if base == "uuid": - return str(uuid.UUID(int=rng.getrandbits(128))), "uuid type" - if base == "date": - return (datetime.now(UTC).date() - timedelta(days=rng.randint(0, 365))).isoformat(), "date type" - if base in {"timestamp", "timestamp without time zone", "timestamptz", "timestamp with time zone"}: - return datetime.now(UTC) - timedelta(seconds=rng.randint(0, 60 * 60 * 24 * 365)), "timestamp type" - if base in {"text", "varchar", "character varying", "char", "character", "citext", "name"}: - # Honour the (N) length cap on varchar(N) / char(N) — generating - # a 4-16 char string into a varchar(2) lands a real "value too - # long" failure on the INSERT (gemini review on #178). - max_len: int | None = None - if "(" in column.data_type: - try: - max_len = int(column.data_type.split("(", 1)[1].split(")", 1)[0].strip()) - except (ValueError, IndexError): - max_len = None - lo, hi = 4, 16 - if max_len is not None: - hi = min(hi, max(1, max_len)) - lo = min(lo, hi) - length = rng.randint(lo, hi) - return "".join(rng.choice(string.ascii_lowercase) for _ in range(length)), "text type" - if base in {"json", "jsonb"}: - return '{"k": "v"}', "json type" + for bases, generate in _TYPE_SYNTHESIZERS: + if base in bases: + return generate(column, rng) return None @@ -293,7 +368,14 @@ async def _sample_fk_row( # --------------------------------------------------------------------------- -async def generate_test_row_for( +# C901 rationale: the 5-step column-fill priority order documented in the +# docstring (identity/generated skip -> FK sampling with composite-FK +# consistency -> name-pattern -> type-based synthesis -> NULL/DEFAULT/raise +# fallback) plus per-referenced-identifier validation for FK targets -- the +# `_synth_by_name`/`_synth_by_type` dispatch tables already extracted the +# two steps that were cleanly separable (see above); the remaining +# branching is the priority-order decision itself. +async def generate_test_row_for( # noqa: C901 driver: SqlDriver, schema: str, table: str, diff --git a/src/mcpg/textsearch.py b/src/mcpg/textsearch.py index 9c3df7b3..9e06bfdd 100644 --- a/src/mcpg/textsearch.py +++ b/src/mcpg/textsearch.py @@ -19,6 +19,7 @@ from dataclasses import dataclass from typing import Any +from mcpg.errors import MCPgError from mcpg.extensions import extension_installed from mcpg.sql import SqlDriver @@ -41,7 +42,7 @@ DEFAULT_VECTOR_METRIC = "l2" -class SearchError(Exception): +class SearchError(MCPgError): """Raised when a search request is invalid.""" @@ -391,7 +392,13 @@ def _cosine_similarity(a: list[float], b: list[float]) -> float: return dot / math.sqrt(norm_a * norm_b) -async def mmr_search( +# C901 rationale: input validation (metric/finite-values/k/fetch_k/lambda +# bounds) followed by the O(k^2) greedy Maximal Marginal Relevance selection +# loop itself -- the greedy re-ranking algorithm's correctness depends on +# the exact relevance-vs-diversity bookkeeping across iterations, so +# restructuring it is a correctness risk to the ranking result for a +# lint-only benefit. +async def mmr_search( # noqa: C901 driver: SqlDriver, schema: str, table: str, diff --git a/src/mcpg/timescaledb.py b/src/mcpg/timescaledb.py index 069a2781..f6da30f0 100644 --- a/src/mcpg/timescaledb.py +++ b/src/mcpg/timescaledb.py @@ -18,6 +18,7 @@ import re from dataclasses import dataclass +from mcpg.errors import MCPgError from mcpg.extensions import extension_installed from mcpg.sql import SqlDriver @@ -30,7 +31,7 @@ ) -class TimescaleError(Exception): +class TimescaleError(MCPgError): """Raised when a TimescaleDB tool call is rejected or fails.""" diff --git a/src/mcpg/tools.py b/src/mcpg/tools.py index bfa61898..eaf0340b 100644 --- a/src/mcpg/tools.py +++ b/src/mcpg/tools.py @@ -406,10 +406,8 @@ async def list_session_intents(ctx: _Ctx) -> dict[str, Any]: entries: list[dict[str, Any]] = [] for preset_name, buckets in INTENT_PRESETS.items(): - if buckets: - count = sum(1 for name in registered if classify_tool(name) in buckets) - else: - count = len(registered) # "admin" -- no filter + # "admin" -- no filter + count = sum(1 for name in registered if classify_tool(name) in buckets) if buckets else len(registered) entries.append( { "name": preset_name, @@ -450,7 +448,11 @@ async def enable_session_intent(name: str, ctx: _Ctx) -> dict[str, Any]: return {"ok": True, "enabled": sorted(enabled_intents(session_key, default_intent=default_intent))} -def _register_introspection(server: MCPServer[AppContext]) -> None: +# C901 rationale: one `@server.tool` closure per exposed MCP tool, gated by +# the frozen `tests/contract/tool_surface.snapshot.json` contract — splitting +# this up would restructure registration order/grouping for the entire +# introspection tool surface for no functional benefit. +def _register_introspection(server: MCPServer[AppContext]) -> None: # noqa: C901 @server.tool( name="list_schemas", description=_with_example( @@ -460,7 +462,7 @@ def _register_introspection(server: MCPServer[AppContext]) -> None: ), ) async def list_schemas( - ctx: _Ctx, include_system: bool = False, database: _DatabaseArg = None + ctx: _Ctx, *, include_system: bool = False, database: _DatabaseArg = None ) -> list[introspection.SchemaInfo]: async def _run() -> list[introspection.SchemaInfo]: schemas = await introspection.list_schemas(_driver(ctx, database), include_system=include_system) @@ -495,7 +497,7 @@ async def _run() -> list[introspection.TableInfo]: ), ) async def describe_table( - ctx: _Ctx, schema: str, table: str, database: _DatabaseArg = None, fresh: bool = False + ctx: _Ctx, schema: str, table: str, database: _DatabaseArg = None, *, fresh: bool = False ) -> list[introspection.ColumnInfo]: async def _run() -> list[introspection.ColumnInfo]: columns = await introspection.describe_table(_driver(ctx, database), schema, table) @@ -515,7 +517,7 @@ async def _run() -> list[introspection.ColumnInfo]: ), ) async def list_indexes( - ctx: _Ctx, schema: str, table: str, database: _DatabaseArg = None, fresh: bool = False + ctx: _Ctx, schema: str, table: str, database: _DatabaseArg = None, *, fresh: bool = False ) -> list[introspection.IndexInfo]: async def _run() -> list[introspection.IndexInfo]: indexes = await introspection.list_indexes(_driver(ctx, database), schema, table) @@ -533,7 +535,7 @@ async def _run() -> list[introspection.IndexInfo]: ), ) async def list_constraints( - ctx: _Ctx, schema: str, table: str, database: _DatabaseArg = None, fresh: bool = False + ctx: _Ctx, schema: str, table: str, database: _DatabaseArg = None, *, fresh: bool = False ) -> list[introspection.ConstraintInfo]: async def _run() -> list[introspection.ConstraintInfo]: constraints = await introspection.list_constraints(_driver(ctx, database), schema, table) @@ -552,7 +554,7 @@ async def _run() -> list[introspection.ConstraintInfo]: ), ) async def list_foreign_keys( - ctx: _Ctx, schema: str, database: _DatabaseArg = None, fresh: bool = False + ctx: _Ctx, schema: str, database: _DatabaseArg = None, *, fresh: bool = False ) -> list[introspection.ForeignKeyInfo]: async def _run() -> list[introspection.ForeignKeyInfo]: fks = await introspection.list_foreign_keys(_driver(ctx, database), schema) @@ -635,7 +637,7 @@ async def _run() -> introspection.PartitionSet: ), ) async def list_roles( - ctx: _Ctx, include_system: bool = False, database: _DatabaseArg = None + ctx: _Ctx, *, include_system: bool = False, database: _DatabaseArg = None ) -> list[introspection.RoleInfo]: async def _run() -> list[introspection.RoleInfo]: roles = await introspection.list_roles(_driver(ctx, database), include_system=include_system) @@ -1001,10 +1003,11 @@ async def read_pg_wal_stats( ctx: _Ctx, start_lsn: str, end_lsn: str | None = None, + *, per_record: bool = False, database: _DatabaseArg = None, ) -> walinspect.WalStatsReport: - return await walinspect.read_pg_wal_stats(_driver(ctx, database), start_lsn, end_lsn, per_record) + return await walinspect.read_pg_wal_stats(_driver(ctx, database), start_lsn, end_lsn, per_record=per_record) @server.tool( name="get_wal_archive_status", @@ -1061,7 +1064,7 @@ async def check_pitr_readiness(ctx: _Ctx, database: _DatabaseArg = None) -> pitr "Set `fresh=true` to bypass the cache and re-read live (e.g. after a schema change)." ), ) - async def get_compact_schema(ctx: _Ctx, schema: str, database: _DatabaseArg = None, fresh: bool = False) -> str: + async def get_compact_schema(ctx: _Ctx, schema: str, database: _DatabaseArg = None, *, fresh: bool = False) -> str: async def _run() -> str: return await introspection.get_compact_schema(_driver(ctx, database), schema) @@ -1098,7 +1101,7 @@ def _register_diagrams(server: MCPServer[AppContext]) -> None: ), ) async def generate_schema_diagram( - ctx: _Ctx, schema: str, include_partitions: bool = False, database: _DatabaseArg = None + ctx: _Ctx, schema: str, *, include_partitions: bool = False, database: _DatabaseArg = None ) -> str: _check_heavy_diagnostics(ctx, "generate_schema_diagram") @@ -1125,7 +1128,7 @@ async def _run() -> str: ), ) async def generate_fk_cascade_graph( - ctx: _Ctx, schema: str, include_all: bool = False, database: _DatabaseArg = None + ctx: _Ctx, schema: str, *, include_all: bool = False, database: _DatabaseArg = None ) -> str: _check_heavy_diagnostics(ctx, "generate_fk_cascade_graph") @@ -1146,7 +1149,7 @@ async def _run() -> str: ), ) async def generate_schema_docs( - ctx: _Ctx, schema: str, include_samples: bool = False, database: _DatabaseArg = None + ctx: _Ctx, schema: str, *, include_samples: bool = False, database: _DatabaseArg = None ) -> str: _check_heavy_diagnostics(ctx, "generate_schema_docs") @@ -1356,7 +1359,9 @@ async def recommend_rerank_strategy( return report -def _register_vector_tuning(server: MCPServer[AppContext]) -> None: +# C901 rationale: one `@server.tool` closure per exposed MCP tool, gated by +# the frozen tool-surface contract — same shape as `_register_introspection`. +def _register_vector_tuning(server: MCPServer[AppContext]) -> None: # noqa: C901 @server.tool( name="tune_vector_index", description=( @@ -1676,6 +1681,7 @@ async def retrieve_with_context( query_vector: list[float], k: int = vector_ops.DEFAULT_CONTEXT_K, metric: str = "l2", + *, include_parents: bool = True, include_children: bool = True, max_related: int = vector_ops.DEFAULT_MAX_RELATED, @@ -1989,7 +1995,9 @@ async def generate_sqlc_schema(ctx: _Ctx, schema: str, database: _DatabaseArg = return await sqlc.generate_sqlc_schema(_driver(ctx, database), schema) -def _register_advisors(server: MCPServer[AppContext]) -> None: +# C901 rationale: one `@server.tool` closure per exposed MCP tool, gated by +# the frozen tool-surface contract — same shape as `_register_introspection`. +def _register_advisors(server: MCPServer[AppContext]) -> None: # noqa: C901 @server.tool( name="run_advisors", description=( @@ -2201,6 +2209,7 @@ async def generate_test_row_for( schema: str, table: str, seed: int | None = None, + *, follow_foreign_keys: bool = True, database: _DatabaseArg = None, ) -> test_row_factory.GeneratedTestRow: @@ -2496,6 +2505,7 @@ async def import_csv( schema: str, table: str, content: str, + *, header: bool = True, delimiter: str = ",", columns: list[str] | None = None, @@ -2816,6 +2826,7 @@ async def create_hypertable( table: str, time_column: str, chunk_time_interval: str = "7 days", + *, if_not_exists: bool = True, ) -> timescaledb.TimescaleWriteResult: result = await timescaledb.create_hypertable( @@ -2941,6 +2952,7 @@ def _register_data_movement_shell(server: MCPServer[AppContext]) -> None: async def dump_database( ctx: _Ctx, format: str = "plain", + *, schema_only: bool = False, schemas: list[str] | None = None, ) -> data_movement.DumpResult: @@ -3002,6 +3014,7 @@ async def copy_table_between_databases( source_url: str, schema: str, table: str, + *, include_schema: bool, include_data: bool, ) -> data_movement.CopyTableResult: @@ -3051,7 +3064,9 @@ async def verify_audit_chain(ctx: _Ctx, database: _DatabaseArg = None) -> dict[s return await vac(_driver(ctx, database)) -def _register_query(server: MCPServer[AppContext]) -> None: +# C901 rationale: one `@server.tool` closure per exposed MCP tool, gated by +# the frozen tool-surface contract — same shape as `_register_introspection`. +def _register_query(server: MCPServer[AppContext]) -> None: # noqa: C901 @server.tool( name="run_select", description=_with_example( @@ -3218,7 +3233,7 @@ async def list_cursors(ctx: _Ctx) -> list[dict[str, Any]]: ), ) async def explain_query( - ctx: _Ctx, sql: str, io: bool = False, database: _DatabaseArg = None + ctx: _Ctx, sql: str, *, io: bool = False, database: _DatabaseArg = None ) -> query.ExplainResult: result = await query.explain_query(_driver(ctx, database), sql, io=io) return result @@ -3237,7 +3252,7 @@ async def explain_query( ), ) async def analyze_query_plan( - ctx: _Ctx, sql: str, io: bool = False, database: _DatabaseArg = None + ctx: _Ctx, sql: str, *, io: bool = False, database: _DatabaseArg = None ) -> query.QueryPlanAnalysis: result = await query.analyze_query_plan(_driver(ctx, database), sql, io=io) return result @@ -3275,6 +3290,7 @@ async def translate_nl_to_sql( question: str, schema: str, provider: str | None = None, + *, execute: bool = False, explain_preflight: bool = True, table_filter: list[str] | None = None, @@ -3334,7 +3350,9 @@ async def run_analytical_query( return await runner.run(sql, timeout_ms=timeout_ms, max_rows=max_rows, work_mem=work_mem) -def _register_health(server: MCPServer[AppContext]) -> None: +# C901 rationale: one `@server.tool` closure per exposed MCP tool, gated by +# the frozen tool-surface contract — same shape as `_register_introspection`. +def _register_health(server: MCPServer[AppContext]) -> None: # noqa: C901 @server.tool( name="check_database_health", description=_with_example( @@ -3367,6 +3385,7 @@ async def analyze_table_bloat( ctx: _Ctx, schema: str, limit: int = health.DEFAULT_BLOAT_LIMIT, + *, precise: bool = False, database: _DatabaseArg = None, ) -> health.TableBloatReport: @@ -3425,7 +3444,7 @@ async def list_databases(ctx: _Ctx) -> multidb.DatabaseList: ), ) async def audit_database( - ctx: _Ctx, schema: str, log_table: str | None = None, database: _DatabaseArg = None, fresh: bool = False + ctx: _Ctx, schema: str, log_table: str | None = None, database: _DatabaseArg = None, *, fresh: bool = False ) -> audit.AuditReport: _check_heavy_diagnostics(ctx, "audit_database") @@ -3534,6 +3553,7 @@ async def recommend_indexes( ctx: _Ctx, min_live_tuples: int = indexing.DEFAULT_MIN_LIVE_TUPLES, database: _DatabaseArg = None, + *, fresh: bool = False, ) -> list[indexing.IndexRecommendation]: _check_heavy_diagnostics(ctx, "recommend_indexes") @@ -4019,6 +4039,7 @@ async def turboquant_approx_candidates( candidate_limit: int, probes: int | None = None, oversample_factor: int | None = None, + *, half_precision: bool = False, database: _DatabaseArg = None, ) -> list[turboquant.TurboQuantCandidate]: @@ -4059,6 +4080,7 @@ async def turboquant_rerank_candidates( final_limit: int, probes: int | None = None, oversample_factor: int | None = None, + *, half_precision: bool = False, database: _DatabaseArg = None, ) -> list[turboquant.TurboQuantRerankedCandidate]: @@ -4199,6 +4221,7 @@ async def pg_search_run( key_field: str, limit: int, columns: list[str] | None = None, + *, return_snippets: bool = False, snippet_field: str | None = None, snippet_start_tag: str = "", @@ -4288,6 +4311,7 @@ async def pg_search_more_like_this( async def pg_search_parse_query( ctx: _Ctx, query_string: str, + *, lenient: bool = False, conjunction_mode: bool = False, database: _DatabaseArg = None, @@ -4404,6 +4428,7 @@ async def log_rerank_event( cross_encoder_score: float, cross_encoder_rank: int, reranker_model: str, + *, used_in_context: bool = False, ground_truth_relevance: int | None = None, extra: dict[str, Any] | None = None, @@ -4591,6 +4616,7 @@ async def create_turboquant_index( bits: int | None = None, lists: int | None = None, transform: str | None = None, + *, normalized: bool | None = None, concurrently: bool = True, ) -> turboquant.CreateIndexResult: @@ -4628,6 +4654,7 @@ async def reindex_turboquant_index( ctx: _Ctx, schema: str, index: str, + *, concurrently: bool = True, ) -> turboquant.ReindexResult: database = ctx.request_context.lifespan_context.database @@ -4677,6 +4704,7 @@ async def create_pg_search_index( mutable_segment_rows: int | None = None, sort_by: str | None = None, search_tokenizer: dict[str, Any] | None = None, + *, concurrently: bool = True, ) -> pg_search.CreatePgSearchIndexResult: database = ctx.request_context.lifespan_context.database @@ -4722,6 +4750,7 @@ async def reindex_pg_search_index( ctx: _Ctx, schema: str, index: str, + *, concurrently: bool = True, ) -> pg_search.ReindexPgSearchResult: database = ctx.request_context.lifespan_context.database @@ -4889,6 +4918,7 @@ async def create_redis_cache_server( address: str, port: int = 6379, database: int = 0, + *, tls: bool = True, allow_insecure_tls: bool = False, ) -> redis_fdw.CreateRedisServerResult: @@ -5038,6 +5068,7 @@ async def schedule_logical_backup( destination: str, database: str, format: str = "plain", + *, schema_only: bool = False, compress: bool = False, pg_dump_path: str = "pg_dump", @@ -5195,6 +5226,7 @@ async def drop_property_graph( ctx: _Ctx, schema: str, name: str, + *, if_exists: bool = True, ) -> pgq.DropPropertyGraphResult: result = await pgq.drop_property_graph( @@ -5369,6 +5401,7 @@ async def prewarm_recommended( min_heap_blks_read: int = 1000, limit: int = 20, prewarm_mode: str = "buffer", + *, dry_run: bool = False, ) -> pg_prewarm.BulkPrewarmResult: result = await pg_prewarm.prewarm_recommended( @@ -6093,6 +6126,7 @@ async def repack_table( ctx: _Ctx, schema: str, table: str, + *, concurrently: bool = True, ) -> repack.RepackResult: database = ctx.request_context.lifespan_context.database @@ -6164,6 +6198,7 @@ async def partman_drop_partition( ctx: _Ctx, parent_table: str, retention: str, + *, control_is_time: bool = True, ) -> dict[str, Any]: dropped = await partman.partman_drop_partition( @@ -6453,7 +6488,7 @@ async def create_graph(ctx: _Ctx, graph_name: str) -> dict[str, Any]: "Returns an object with `graph_name` and `dropped` (bool)." ), ) - async def drop_graph(ctx: _Ctx, graph_name: str, cascade: bool = True) -> dict[str, Any]: + async def drop_graph(ctx: _Ctx, graph_name: str, *, cascade: bool = True) -> dict[str, Any]: app = ctx.request_context.lifespan_context res = await graph_mgmt.drop_graph(app, graph_name, cascade=cascade) await app.cache.clear() @@ -6639,7 +6674,9 @@ def review_rls_policy_prompt( return mcpg_prompts._build_review_rls_policy(schema, table) -def _register_warehousepg_reads(server: MCPServer[AppContext]) -> None: +# C901 rationale: one `@server.tool` closure per exposed MCP tool, gated by +# the frozen tool-surface contract — same shape as `_register_introspection`. +def _register_warehousepg_reads(server: MCPServer[AppContext]) -> None: # noqa: C901 @server.tool( name="get_warehousepg_status", description=_with_example( @@ -6801,6 +6838,7 @@ def _register_logical_replication_writes(server: MCPServer[AppContext]) -> None: async def create_publication( ctx: _Ctx, name: str, + *, all_tables: bool = False, tables: tuple[str, ...] = (), ) -> logical_replication.CreatePublicationResult: @@ -6826,6 +6864,7 @@ async def create_publication( async def drop_publication( ctx: _Ctx, name: str, + *, if_exists: bool = False, cascade: bool = False, ) -> logical_replication.DropPublicationResult: @@ -6859,6 +6898,7 @@ async def create_subscription( name: str, connection_string: str, publications: tuple[str, ...], + *, enabled: bool = True, copy_data: bool = True, create_slot: bool = True, @@ -6893,6 +6933,7 @@ async def create_subscription( async def drop_subscription( ctx: _Ctx, name: str, + *, if_exists: bool = False, ) -> logical_replication.DropSubscriptionResult: result = await logical_replication.drop_subscription( @@ -7103,7 +7144,10 @@ def _apply_tool_wire_metadata(server: MCPServer[AppContext], read_only_names: se tool.annotations = existing.model_copy(update=derived) -def register_tools( +# C901 rationale: one policy-gated `if is_permitted(...)` branch per capability +# / access-mode tier, dispatching to the `_register_*` helpers above — the +# branching is the access-mode policy itself, not incidental complexity. +def register_tools( # noqa: C901 server: MCPServer[AppContext], settings: Settings, *, analytical_available: bool | None = None ) -> None: """Register the MCP tools permitted by the configured access mode. diff --git a/src/mcpg/turboquant.py b/src/mcpg/turboquant.py index 5143cfa1..d4b35820 100644 --- a/src/mcpg/turboquant.py +++ b/src/mcpg/turboquant.py @@ -59,6 +59,7 @@ from typing import Any from mcpg.database import Database +from mcpg.errors import MCPgError from mcpg.extensions import extension_installed from mcpg.sql import SqlDriver @@ -68,7 +69,7 @@ _IDENTIFIER = re.compile(r"\A[A-Za-z_][A-Za-z0-9_]*\Z") -class TurboQuantError(Exception): +class TurboQuantError(MCPgError): """Raised when a pg_turboquant operation cannot complete.""" @@ -814,7 +815,7 @@ def _validate_transform(transform: str | None) -> None: ) -def _validate_bool(value: bool | None, kind: str) -> None: +def _validate_bool(*, value: bool | None, kind: str) -> None: if value is None: return if not isinstance(value, bool): @@ -896,8 +897,8 @@ async def create_turboquant_index( _validate_int_option("bits", bits, _BITS_MIN, _BITS_MAX) _validate_int_option("lists", lists, _LISTS_MIN, _LISTS_MAX) _validate_transform(transform) - _validate_bool(normalized, "normalized") - _validate_bool(concurrently, "concurrently") + _validate_bool(value=normalized, kind="normalized") + _validate_bool(value=concurrently, kind="concurrently") if not await extension_installed(database.driver(), "pg_turboquant"): raise TurboQuantError("pg_turboquant extension is not installed in this database") @@ -966,7 +967,7 @@ async def reindex_turboquant_index( """ _validate_identifier(schema, "schema") _validate_identifier(index, "index") - _validate_bool(concurrently, "concurrently") + _validate_bool(value=concurrently, kind="concurrently") driver = database.driver() if not await extension_installed(driver, "pg_turboquant"): @@ -1145,7 +1146,7 @@ async def turboquant_approx_candidates( _validate_positive_int("candidate_limit", candidate_limit) _validate_int_option("probes", probes, 1, 1_000_000) _validate_int_option("oversample_factor", oversample_factor, 1, 1_000_000) - _validate_bool(half_precision, "half_precision") + _validate_bool(value=half_precision, kind="half_precision") if not await extension_installed(driver, "pg_turboquant"): raise TurboQuantError("pg_turboquant extension is not installed in this database") @@ -1213,7 +1214,7 @@ async def turboquant_rerank_candidates( _validate_positive_int("final_limit", final_limit) _validate_int_option("probes", probes, 1, 1_000_000) _validate_int_option("oversample_factor", oversample_factor, 1, 1_000_000) - _validate_bool(half_precision, "half_precision") + _validate_bool(value=half_precision, kind="half_precision") if not await extension_installed(driver, "pg_turboquant"): raise TurboQuantError("pg_turboquant extension is not installed in this database") diff --git a/src/mcpg/vector_ops.py b/src/mcpg/vector_ops.py index 85c60865..1240175e 100644 --- a/src/mcpg/vector_ops.py +++ b/src/mcpg/vector_ops.py @@ -24,6 +24,7 @@ from typing import Any from mcpg import introspection +from mcpg.errors import MCPgError from mcpg.extensions import extension_installed from mcpg.sql import SqlDriver @@ -80,7 +81,7 @@ def _validate_sample_size(value: int) -> None: _NORM_TOLERANCE = 0.02 -class VectorOpsError(Exception): +class VectorOpsError(MCPgError): """Raised when a vector-analytics request is rejected.""" @@ -358,7 +359,12 @@ class CrossTableSimilarityResult: matches: list[CrossTableMatch] -async def cross_table_similarity( +# C901 rationale: per-identifier validation (6 schema/table/column names, +# each checked before the catalog lookup so an invalid identifier fails with +# its own error rather than being masked as "column missing" -- see the +# docstring) followed by the cross-table dimension-mismatch check and the +# k-NN query itself. +async def cross_table_similarity( # noqa: C901 driver: SqlDriver, *, source_schema: str, @@ -618,7 +624,13 @@ def _kmeans_plus_plus_init( return centroids -def _kmeans( +# C901 rationale: Lloyd's k-means algorithm (assignment step + update step + +# convergence check) with a hand-tuned hot-path distance function (the +# inline comment cites "Gemini PR #52" for the unit-norm cosine shortcut) -- +# this is exactly the numerical-algorithm-inherent-complexity case; touching +# the loop structure risks the performance property the comment documents +# or the convergence math itself. +def _kmeans( # noqa: C901 vectors: list[list[float]], k: int, *, @@ -709,7 +721,13 @@ def _cosine_distance_unit(a: list[float], b: list[float]) -> float: return centroids, labels, distances, iteration, converged, inertia -async def cluster_vectors( +# C901 rationale: identifier + numeric-bound validation (k/sample_size/ +# max_iterations, metric) feeding into the cosine-normalisation branch +# (vectors normalised before clustering, centroids re-normalised each +# iteration so Lloyd's update still converges -- see the docstring) around +# the `_kmeans` call; the branching is what keeps the metric-specific +# normalisation contract correct. +async def cluster_vectors( # noqa: C901 driver: SqlDriver, schema: str, table: str, @@ -910,7 +928,13 @@ class VectorOutlierResult: cluster_stats: list[ClusterOutlierStats] -async def detect_vector_outliers( +# C901 rationale: complexity 23 -- identifier/bound validation, the shared +# k-means clustering call, then within-cluster z-score computation and the +# sort-and-cap-at-max_results output shaping. The statistical logic (z-score +# per cluster, not a global threshold -- see the docstring's rationale for +# why) is the correctness-sensitive part; splitting it from its validation +# risks silently changing what counts as "far from its cluster." +async def detect_vector_outliers( # noqa: C901 driver: SqlDriver, schema: str, table: str, diff --git a/src/mcpg/vector_tuner_advanced.py b/src/mcpg/vector_tuner_advanced.py index d1409431..b00072be 100644 --- a/src/mcpg/vector_tuner_advanced.py +++ b/src/mcpg/vector_tuner_advanced.py @@ -196,7 +196,13 @@ def _percentile(values: list[float], pct: float) -> float: return ordered[rank] -async def recommend_hnsw_ef_search( +# C901 rationale: brute-force ground-truth construction, an ef_search sweep +# measuring mean recall@k + p50/p95 latency per value, and the +# smallest-value-clearing-target_recall selection -- the recall-measurement +# methodology (excluding a query row's own vector from both ground truth +# and approximate results, per the docstring) is the correctness-sensitive +# part of the branching. +async def recommend_hnsw_ef_search( # noqa: C901 driver: SqlDriver, schema: str, table: str, @@ -458,7 +464,11 @@ class IvfflatProbesRecommendation: detail: str = "" -async def recommend_ivfflat_probes( +# C901 rationale: mirrors recommend_hnsw_ef_search (same recall-sweep +# methodology, probe_values instead of ef_values) -- see that function's +# rationale above for why the branching is the correctness-sensitive +# recall-measurement logic, not incidental nesting. +async def recommend_ivfflat_probes( # noqa: C901 driver: SqlDriver, schema: str, table: str, diff --git a/src/mcpg/vector_tuning.py b/src/mcpg/vector_tuning.py index a14c7c99..30b273e1 100644 --- a/src/mcpg/vector_tuning.py +++ b/src/mcpg/vector_tuning.py @@ -19,6 +19,7 @@ import re from dataclasses import dataclass +from mcpg.errors import MCPgError from mcpg.extensions import extension_installed from mcpg.introspection import describe_table from mcpg.sql import SqlDriver @@ -51,7 +52,7 @@ _IDENTIFIER = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\Z") -class VectorTuningError(Exception): +class VectorTuningError(MCPgError): """Raised when a pgvector tuning operation cannot complete.""" diff --git a/src/mcpg/wait_for_lsn.py b/src/mcpg/wait_for_lsn.py index 5d563481..59c2f710 100644 --- a/src/mcpg/wait_for_lsn.py +++ b/src/mcpg/wait_for_lsn.py @@ -50,6 +50,7 @@ import re from dataclasses import dataclass +from mcpg.errors import MCPgError from mcpg.sql import SqlDriver # PG 19 ships WAIT FOR LSN. The version-num boundary. @@ -61,7 +62,7 @@ _LSN_PATTERN = re.compile(r"^[0-9A-Fa-f]+/[0-9A-Fa-f]+$") -class WaitForLsnError(Exception): +class WaitForLsnError(MCPgError): """Raised when a WAIT FOR LSN request is rejected or fails.""" @@ -352,7 +353,14 @@ async def wait_for_lsn(driver: SqlDriver, *, lsn: str, timeout_ms: int = 0) -> W # --------------------------------------------------------------------------- -async def recommend_read_your_writes(driver: SqlDriver) -> ReadYourWritesRecommendation: +# C901 rationale: six reason-code classification branches over an +# atomically-fetched version/recovery/lag probe. Per the function's own +# "Implementation note", the single-query atomicity fixed a real +# misclassification bug ("gemini critical on PR #146" -- a standby could be +# routed to `primary_no_wait_needed`) -- splitting the classification back +# into separate probes/branches is the exact regression this function's +# docstring documents avoiding. +async def recommend_read_your_writes(driver: SqlDriver) -> ReadYourWritesRecommendation: # noqa: C901 """Advisor — should the caller use WAIT FOR LSN for read-your-writes? Read-only; never raises. Returns a structured recommendation with diff --git a/src/mcpg/walinspect.py b/src/mcpg/walinspect.py index ba71928e..b44e9342 100644 --- a/src/mcpg/walinspect.py +++ b/src/mcpg/walinspect.py @@ -143,6 +143,7 @@ async def read_pg_wal_stats( driver: SqlDriver, start_lsn: str, end_lsn: str | None = None, + *, per_record: bool = False, ) -> WalStatsReport: """Read WAL record statistics using ``pg_walinspect``. diff --git a/src/mcpg/warehousepg.py b/src/mcpg/warehousepg.py index 41fd66d6..69032104 100644 --- a/src/mcpg/warehousepg.py +++ b/src/mcpg/warehousepg.py @@ -97,7 +97,13 @@ class WarehousePGStatus: _MPP_VERSION_MARKERS: tuple[str, ...] = ("warehousepg", "greenplum") -async def get_warehousepg_status(driver: SqlDriver) -> WarehousePGStatus: +# C901 rationale: multi-step MPP detection (version-string marker check, +# then a second gp_segment_configuration catalog confirmation "since an +# operator could in principle put 'WarehousePG' in their version string +# without being on the real product" -- see the inline comment), where +# every step needs its own try/except to uphold the docstring's "never +# raises; available=False on every error path" contract. +async def get_warehousepg_status(driver: SqlDriver) -> WarehousePGStatus: # noqa: C901 """Probe the connected server for WarehousePG / MPP signature. Read-only; never raises. Returns ``available=False`` on every @@ -749,7 +755,12 @@ def _walk_mpp_plan(node: dict[str, Any]) -> Iterator[dict[str, Any]]: yield from _walk_mpp_plan(child) -async def analyze_mpp_query_plan(driver: SqlDriver, sql: str) -> MppQueryPlanAnalysis: +# C901 rationale: same "available=False on every error path" contract as +# get_warehousepg_status (vanilla-PG gate, EXPLAIN failure, unexpected plan +# shape each need their own guard) plus the depth-first MPP plan-tree walk +# rolling up slice/motion/redistribute/broadcast/gather counts -- the +# per-node-type counting is the plan-analysis logic itself. +async def analyze_mpp_query_plan(driver: SqlDriver, sql: str) -> MppQueryPlanAnalysis: # noqa: C901 """Run ``EXPLAIN (ANALYZE, FORMAT JSON)`` and roll up MPP plan facts. Reuses :func:`mcpg.query.explain_query` with ``io=True`` (which diff --git a/src/mcpg/write.py b/src/mcpg/write.py index dd4424f4..fb0eb44e 100644 --- a/src/mcpg/write.py +++ b/src/mcpg/write.py @@ -16,12 +16,14 @@ from __future__ import annotations +import contextlib from dataclasses import asdict, dataclass, field from typing import Any import pglast from mcpg.audit_trail import SchemaDiffSnapshot, capture_columns, record_audit +from mcpg.errors import MCPgError from mcpg.sql import SqlDriver # pglast statement node names accepted by run_write. @@ -47,7 +49,7 @@ ) -class WriteError(Exception): +class WriteError(MCPgError): """Raised when a write is rejected or fails to execute.""" @@ -107,7 +109,9 @@ async def _persist_audit( ) -> None: """Best-effort audit persistence — failures must not mask the real result.""" result_payload = asdict(result) if result is not None else None - try: + # Audit persistence is best-effort; never let it shadow the real + # write error or fabricate one for a successful write. + with contextlib.suppress(Exception): await record_audit( driver, tool=tool, @@ -116,10 +120,6 @@ async def _persist_audit( error=error, result=result_payload, ) - except Exception: - # Audit persistence is best-effort; never let it shadow the real - # write error or fabricate one for a successful write. - pass async def run_write(driver: SqlDriver, sql: str, *, audit_persist: bool = False) -> WriteResult: diff --git a/tests/_mcp_test_helpers.py b/tests/_mcp_test_helpers.py index f9090945..98eaa60f 100644 --- a/tests/_mcp_test_helpers.py +++ b/tests/_mcp_test_helpers.py @@ -29,6 +29,7 @@ async def create_connected_server_and_client_session( logging_callback: LoggingFnT | None = None, message_handler: MessageHandlerFnT | None = None, client_info: types.Implementation | None = None, + *, raise_exceptions: bool = False, elicitation_callback: ElicitationFnT | None = None, ) -> AsyncGenerator[ClientSession, None]: diff --git a/tests/contract/test_pg19_sql_characterisation.py b/tests/contract/test_pg19_sql_characterisation.py index bf34bafa..149f70b6 100644 --- a/tests/contract/test_pg19_sql_characterisation.py +++ b/tests/contract/test_pg19_sql_characterisation.py @@ -45,6 +45,7 @@ from __future__ import annotations +from pathlib import Path from typing import Any import pglast @@ -235,7 +236,7 @@ def test_stats_reset_propagates_through_pg19_stats_reads() -> None: # the purpose of the propagation guard (gemini review on #177). src = (pg19_stats.__file__ or "").replace(".pyc", ".py") try: - with open(src, encoding="utf-8") as fh: + with Path(src).open(encoding="utf-8") as fh: module_text = fh.read() except OSError as exc: pytest.fail(f"failed to read pg19_stats source from {src!r}: {exc}") diff --git a/tests/contract/tool_return_shapes.snapshot.json b/tests/contract/tool_return_shapes.snapshot.json index 081b4227..32caf889 100644 --- a/tests/contract/tool_return_shapes.snapshot.json +++ b/tests/contract/tool_return_shapes.snapshot.json @@ -2234,6 +2234,7 @@ "refused", "row_count", "rows", + "schema_context", "sql", "tokens_in", "tokens_out" diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 4b1bf2b6..7a3ec009 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -56,7 +56,7 @@ async def is_warehousepg(connected_database: Database) -> bool: @pytest.fixture -async def distributed_replicated_clause(is_warehousepg: bool) -> str: +async def distributed_replicated_clause(*, is_warehousepg: bool) -> str: """DDL suffix for tables needing >1 independent UNIQUE/PK constraint. WarehousePG requires every UNIQUE/PRIMARY KEY constraint on a table to diff --git a/tests/integration/test_cron_integration.py b/tests/integration/test_cron_integration.py index c979eecc..0d6dd380 100644 --- a/tests/integration/test_cron_integration.py +++ b/tests/integration/test_cron_integration.py @@ -1,5 +1,7 @@ """Integration tests for pg_cron wrappers — gated on the extension.""" +import contextlib + import pytest from mcpg.cron import list_cron_jobs, schedule_cron_job, unschedule_cron_job @@ -29,10 +31,8 @@ async def test_schedule_then_unschedule_roundtrip(connected_database: Database) job_name = "mcpg_cron_it_heartbeat" # Best-effort cleanup if a prior run left the job behind. - try: + with contextlib.suppress(Exception): await unschedule_cron_job(driver, job_name) - except Exception: - pass scheduled = await schedule_cron_job(driver, job_name, "*/5 * * * *", "SELECT 1") try: diff --git a/tests/integration/test_data_movement_integration.py b/tests/integration/test_data_movement_integration.py index c2d3523e..55320750 100644 --- a/tests/integration/test_data_movement_integration.py +++ b/tests/integration/test_data_movement_integration.py @@ -38,7 +38,10 @@ async def _skip_when_pg_dump_too_old_for_server(database: Database) -> None: if shutil.which("pg_dump") is None: pytest.skip("pg_dump is not on PATH on this runner") try: - client_out = subprocess.check_output(["pg_dump", "--version"], text=True, timeout=5) + # ASYNC221 rationale: test-only, single fast local probe run once per test + # setup, not a hot path; asyncio.create_subprocess_exec would add + # complexity for no measurable benefit here. + client_out = subprocess.check_output(["pg_dump", "--version"], text=True, timeout=5) # noqa: ASYNC221 except (subprocess.SubprocessError, OSError) as exc: pytest.skip(f"pg_dump --version probe failed: {exc}") client_match = re.search(r"\b(\d+)(?:\.\d+)?\b", client_out) diff --git a/tests/integration/test_demo_integration.py b/tests/integration/test_demo_integration.py index 9af7897d..647c79f8 100644 --- a/tests/integration/test_demo_integration.py +++ b/tests/integration/test_demo_integration.py @@ -31,7 +31,7 @@ async def _force_drop(database_url: str) -> None: await conn.commit() -async def test_demo_lifecycle_and_planted_findings(database_url: str, is_warehousepg: bool) -> None: +async def test_demo_lifecycle_and_planted_findings(database_url: str, *, is_warehousepg: bool) -> None: if is_warehousepg: pytest.skip("demo dataset targets stock PostgreSQL") await _force_drop(database_url) @@ -122,7 +122,7 @@ async def test_demo_lifecycle_and_planted_findings(database_url: str, is_warehou await _force_drop(database_url) -async def test_drop_refuses_a_schema_mcpg_did_not_create(database_url: str, is_warehousepg: bool) -> None: +async def test_drop_refuses_a_schema_mcpg_did_not_create(database_url: str, *, is_warehousepg: bool) -> None: if is_warehousepg: pytest.skip("demo dataset targets stock PostgreSQL") await _force_drop(database_url) diff --git a/tests/integration/test_graph_integration.py b/tests/integration/test_graph_integration.py index 22ca3698..39b75c78 100644 --- a/tests/integration/test_graph_integration.py +++ b/tests/integration/test_graph_integration.py @@ -1,5 +1,6 @@ """Integration tests for Apache AGE graph tools against a live PostgreSQL database.""" +import contextlib from collections.abc import AsyncIterator import pytest @@ -46,10 +47,8 @@ async def integration_graph( ) # Teardown any leftover graph space - try: + with contextlib.suppress(Exception): await drop_graph(context, graph_name, cascade=True) - except Exception: - pass # Create fresh graph await create_graph(context, graph_name) @@ -58,10 +57,8 @@ async def integration_graph( yield graph_name finally: # Cleanup - try: + with contextlib.suppress(Exception): await drop_graph(context, graph_name, cascade=True) - except Exception: - pass async def test_graph_lifecycle_and_cypher_queries( diff --git a/tests/unit/_fakes.py b/tests/unit/_fakes.py index 4eccb669..e02d341e 100644 --- a/tests/unit/_fakes.py +++ b/tests/unit/_fakes.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import Any +from typing import Any, Self from mcpg.sql import SqlDriver @@ -39,14 +39,25 @@ def __init__(self, rows: list[dict[str, Any]] | None = None, *, fail: bool = Fal self._rows = rows or [] self.fail = fail self.calls: list[tuple[str, Any, bool]] = [] + # Recorded separately (not in ``calls``, to avoid touching every + # existing call-site assertion): the ``row_limit`` each + # ``execute_query`` call was made with, in call order. + self.row_limits: list[int | None] = [] async def execute_query( - self, query: str, params: list[Any] | None = None, force_readonly: bool = False + self, + query: str, + params: list[Any] | None = None, + *, + force_readonly: bool = False, + row_limit: int | None = None, ) -> list[SqlDriver.RowResult]: self.calls.append((query, params, force_readonly)) + self.row_limits.append(row_limit) if self.fail: raise RuntimeError("execution failed") - return [SqlDriver.RowResult(cells=dict(row)) for row in self._rows] + rows = self._rows if row_limit is None else self._rows[:row_limit] + return [SqlDriver.RowResult(cells=dict(row)) for row in rows] class FakeRoutingDriver: @@ -57,12 +68,18 @@ def __init__(self, routes: dict[str, list[dict[str, Any]]]) -> None: self.calls: list[tuple[str, Any, bool]] = [] async def execute_query( - self, query: str, params: list[Any] | None = None, force_readonly: bool = False + self, + query: str, + params: list[Any] | None = None, + *, + force_readonly: bool = False, + row_limit: int | None = None, ) -> list[SqlDriver.RowResult]: self.calls.append((query, params, force_readonly)) for substring, rows in self._routes.items(): if substring in query: - return [SqlDriver.RowResult(cells=dict(row)) for row in rows] + result = rows if row_limit is None else rows[:row_limit] + return [SqlDriver.RowResult(cells=dict(row)) for row in result] return [] @@ -80,7 +97,12 @@ def __init__(self, routes: dict[tuple[str, tuple[Any, ...] | None], list[dict[st self.calls: list[tuple[str, Any, bool]] = [] async def execute_query( - self, query: str, params: list[Any] | None = None, force_readonly: bool = False + self, + query: str, + params: list[Any] | None = None, + *, + force_readonly: bool = False, + row_limit: int | None = None, ) -> list[SqlDriver.RowResult]: self.calls.append((query, params, force_readonly)) params_key = tuple(params) if params is not None else () @@ -88,7 +110,8 @@ async def execute_query( if substring not in query: continue if route_params is None or route_params == params_key: - return [SqlDriver.RowResult(cells=dict(row)) for row in rows] + result = rows if row_limit is None else rows[:row_limit] + return [SqlDriver.RowResult(cells=dict(row)) for row in result] return [] @@ -143,7 +166,7 @@ async def execute_many(self, sql: str, params_seq: Any) -> int: self.execute_many_calls.append((sql, rows)) return self._execute_many_rowcount if self._execute_many_rowcount is not None else len(rows) - async def __aenter__(self) -> FakeDatabase: + async def __aenter__(self) -> Self: await self.connect() return self diff --git a/tests/unit/test_about.py b/tests/unit/test_about.py index 5f6dca72..f2627001 100644 --- a/tests/unit/test_about.py +++ b/tests/unit/test_about.py @@ -57,7 +57,7 @@ def test_capability_ids_are_unique() -> None: tuple.""" ids = [c.id for c in CAPABILITIES] assert len(ids) == len(set(ids)), f"duplicate bucket ids: {ids}" - assert BUCKET_IDS == frozenset(ids) + assert frozenset(ids) == BUCKET_IDS def test_capability_summaries_are_short_enough_for_an_llm() -> None: diff --git a/tests/unit/test_audit.py b/tests/unit/test_audit.py index 5c1128a8..86b97f46 100644 --- a/tests/unit/test_audit.py +++ b/tests/unit/test_audit.py @@ -403,3 +403,32 @@ def test_audit_record_redacts_password_in_error_text(caplog) -> None: assert "db.internal" in messages # non-secret host is preserved finally: logger.propagate = old_propagate + + +async def test_get_version_and_db_logs_debug_on_query_failure(caplog) -> None: + """A failed version()/current_database() probe logs at debug and falls back.""" + import logging + + from mcpg.audit import _get_version_and_db + + driver = FakeDriver(fail=True) + logger = logging.getLogger("mcpg") + old_propagate = logger.propagate + logger.propagate = True + try: + caplog.set_level(logging.DEBUG, logger="mcpg.audit") + + version, dbname = await _get_version_and_db(driver) # type: ignore[arg-type] + + assert version == "PostgreSQL Unknown" + assert dbname == "unknown" + matches = [ + r + for r in caplog.records + if r.name == "mcpg.audit" and "Version/dbname query failed" in r.message and r.levelno == logging.DEBUG + ] + assert len(matches) == 1 + # exc_info=True must actually attach a traceback, not just the message. + assert matches[0].exc_info is not None + finally: + logger.propagate = old_propagate diff --git a/tests/unit/test_audit_authentication.py b/tests/unit/test_audit_authentication.py index c69b4c18..1b64e13f 100644 --- a/tests/unit/test_audit_authentication.py +++ b/tests/unit/test_audit_authentication.py @@ -85,7 +85,7 @@ async def test_pg_authid_permission_denied_degrades_to_warning_per_metric() -> N class _DeniedDriver: async def execute_query( - self, query: str, params: list[Any] | None = None, force_readonly: bool = False + self, query: str, params: list[Any] | None = None, *, force_readonly: bool = False ) -> list[Any]: del query, params, force_readonly raise PermissionError("pg_authid: permission denied") diff --git a/tests/unit/test_audit_events_migration.py b/tests/unit/test_audit_events_migration.py index 29affdea..cce58046 100644 --- a/tests/unit/test_audit_events_migration.py +++ b/tests/unit/test_audit_events_migration.py @@ -12,11 +12,31 @@ AUDIT_TABLE, AuditTrailError, EventsAuditMigrationResult, + _events_migrate_timescaledb, _resolve_events_settings, migrate_audit_events_to_partitioned, ) +class _SelectiveFailDriver: + """Routes like FakeRoutingDriver, but raises for queries matching any + of ``fail_substrings`` — used to simulate a TimescaleDB policy call + (``add_compression_policy`` / ``add_retention_policy``) rejecting on + an unsupported TSDB edition/version, distinct from every other + statement in the same migration succeeding.""" + + def __init__(self, routes: dict[str, list[dict[str, Any]]], fail_substrings: tuple[str, ...]) -> None: + self._routing = FakeRoutingDriver(routes) + self._fail_substrings = fail_substrings + self.calls: list[Any] = [] + + async def execute_query(self, query: str, params: Any = None, *, force_readonly: bool = False) -> Any: + self.calls.append((query, params, force_readonly)) + if any(s in query for s in self._fail_substrings): + raise RuntimeError("simulated TimescaleDB policy failure") + return await self._routing.execute_query(query, params, force_readonly=force_readonly) + + def _table_exists_routes() -> dict[str, list[dict[str, Any]]]: """The events table exists (pg_class probe returns a row).""" return { @@ -282,3 +302,78 @@ async def test_migrate_native_skips_lz4_on_pg_13() -> None: # The DROP TABLE legacy step (which follows compression) still # runs — transaction integrity preserved. assert "DROP TABLE mcpg_audit.events_migration_legacy" in queries + + +async def test_timescaledb_migrate_logs_debug_when_compression_policy_fails( + caplog: pytest.LogCaptureFixture, +) -> None: + """A rejected add_compression_policy call is swallowed (best-effort — + not every TSDB edition supports compression) but now logs at debug.""" + import logging + + driver = _SelectiveFailDriver({}, fail_substrings=("add_compression_policy",)) + + root_logger = logging.getLogger("mcpg") + old_propagate = root_logger.propagate + root_logger.propagate = True + try: + caplog.set_level(logging.DEBUG, logger="mcpg.audit_trail") + + _rows_copied, compression_enabled, _statements = await _events_migrate_timescaledb( + driver, # type: ignore[arg-type] + chunk_interval="7 days", + compress_after="30 days", + retention_days=None, + rls=False, + reader_role=None, + ) + finally: + root_logger.propagate = old_propagate + + assert compression_enabled is False + matches = [ + r + for r in caplog.records + if r.name == "mcpg.audit_trail" and "add_compression_policy" in r.message and r.levelno == logging.DEBUG + ] + assert len(matches) == 1 + # exc_info=True must actually attach a traceback, not just the message. + assert matches[0].exc_info is not None + + +async def test_timescaledb_migrate_logs_debug_when_retention_policy_fails( + caplog: pytest.LogCaptureFixture, +) -> None: + """A rejected add_retention_policy call is swallowed (best-effort — + the operator opted in but the TSDB edition rejected it) but now logs + at debug.""" + import logging + + driver = _SelectiveFailDriver({}, fail_substrings=("add_retention_policy",)) + + root_logger = logging.getLogger("mcpg") + old_propagate = root_logger.propagate + root_logger.propagate = True + try: + caplog.set_level(logging.DEBUG, logger="mcpg.audit_trail") + + _rows_copied, _compression_enabled, statements = await _events_migrate_timescaledb( + driver, # type: ignore[arg-type] + chunk_interval="7 days", + compress_after="30 days", + retention_days=90, + rls=False, + reader_role=None, + ) + finally: + root_logger.propagate = old_propagate + + assert "add_retention_policy" not in " | ".join(statements) + matches = [ + r + for r in caplog.records + if r.name == "mcpg.audit_trail" and "add_retention_policy" in r.message and r.levelno == logging.DEBUG + ] + assert len(matches) == 1 + # exc_info=True must actually attach a traceback, not just the message. + assert matches[0].exc_info is not None diff --git a/tests/unit/test_audit_integrity.py b/tests/unit/test_audit_integrity.py index 5d753dae..d0d31b1e 100644 --- a/tests/unit/test_audit_integrity.py +++ b/tests/unit/test_audit_integrity.py @@ -453,7 +453,9 @@ class _PagedFake: def __init__(self) -> None: self.calls: list[tuple[str, list[Any] | None, bool]] = [] - async def execute_query(self, query: str, params: list[Any] | None = None, force_readonly: bool = False) -> Any: + async def execute_query( + self, query: str, params: list[Any] | None = None, *, force_readonly: bool = False + ) -> Any: from mcpg.sql import SqlDriver self.calls.append((query, params, force_readonly)) diff --git a/tests/unit/test_audit_trail.py b/tests/unit/test_audit_trail.py index 2f06a5a0..539b59e7 100644 --- a/tests/unit/test_audit_trail.py +++ b/tests/unit/test_audit_trail.py @@ -388,25 +388,46 @@ async def test_record_audit_reads_integrity_config_from_driver_settings() -> Non ) -async def test_record_audit_treats_malformed_integrity_flag_as_disabled(monkeypatch: pytest.MonkeyPatch) -> None: +async def test_record_audit_treats_malformed_integrity_flag_as_disabled( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: # A non-boolean MCPG_AUDIT_INTEGRITY must not crash record_audit; the - # parse error is swallowed and integrity stays off (HMAC columns NULL). + # parse error is swallowed and integrity stays off (HMAC columns NULL), + # and the swallowed parse failure is logged at debug rather than silent. + import logging + monkeypatch.setenv("MCPG_AUDIT_INTEGRITY", "not-a-bool") monkeypatch.setenv("MCPG_AUDIT_HMAC_KEY", "secret_key") _reset_audit_init_cache() driver = FakeRoutingDriver({}) - await record_audit( # type: ignore[arg-type] - driver, - tool="run_write", - arguments={"sql": "SELECT 1"}, - status="ok", - ) + root_logger = logging.getLogger("mcpg") + old_propagate = root_logger.propagate + root_logger.propagate = True + try: + caplog.set_level(logging.DEBUG, logger="mcpg.audit_trail") + + await record_audit( # type: ignore[arg-type] + driver, + tool="run_write", + arguments={"sql": "SELECT 1"}, + status="ok", + ) + finally: + root_logger.propagate = old_propagate insert = next(call for call in driver.calls if "INSERT INTO" in call[0]) params = insert[1] assert params is not None assert params[7] is None # event_hmac stays NULL + matches = [ + r + for r in caplog.records + if r.name == "mcpg.audit_trail" and "MCPG_AUDIT_INTEGRITY" in r.message and r.levelno == logging.DEBUG + ] + assert len(matches) == 1 + # exc_info=True must actually attach a traceback, not just the message. + assert matches[0].exc_info is not None async def test_record_audit_leaves_hmac_columns_null_when_integrity_disabled(monkeypatch: pytest.MonkeyPatch) -> None: @@ -606,7 +627,7 @@ async def test_prune_audit_events_tool_is_registered_in_unrestricted_mode() -> N ("mode", "present"), [("read-only", False), ("restricted", True), ("unrestricted", True)], ) -async def test_prune_audit_events_tool_needs_write_capability(mode: str, present: bool) -> None: +async def test_prune_audit_events_tool_needs_write_capability(mode: str, *, present: bool) -> None: # prune deletes rows -> WRITE capability -> present in the read-write tiers # (restricted + unrestricted), absent in read-only. settings = load_settings({"MCPG_DATABASE_URL": "postgresql://u:p@localhost/db", "MCPG_ACCESS_MODE": mode}) diff --git a/tests/unit/test_bench_tier_b.py b/tests/unit/test_bench_tier_b.py index f7c789ad..9d96d6cf 100644 --- a/tests/unit/test_bench_tier_b.py +++ b/tests/unit/test_bench_tier_b.py @@ -41,7 +41,7 @@ def test_naming_grader_matches_camelcase_column() -> None: def _trial( - arm: str, task_id: str, tin: int, tout: int, tools: int, turns: int, passed: bool, error=None + arm: str, task_id: str, tin: int, tout: int, tools: int, turns: int, *, passed: bool, error=None ) -> TrialResult: return TrialResult( task_id=task_id, @@ -60,11 +60,11 @@ def _trial( def test_aggregate_token_ratio_and_correctness() -> None: trials = [ # baseline: 10k tokens, 1/2 correct - _trial(ARM_BASELINE, "a", 8000, 2000, 5, 5, True), - _trial(ARM_BASELINE, "b", 8000, 2000, 5, 5, False), + _trial(ARM_BASELINE, "a", 8000, 2000, 5, 5, passed=True), + _trial(ARM_BASELINE, "b", 8000, 2000, 5, 5, passed=False), # mcpg: 2.5k tokens, 2/2 correct - _trial(ARM_MCPG, "a", 2000, 500, 1, 2, True), - _trial(ARM_MCPG, "b", 2000, 500, 1, 2, True), + _trial(ARM_MCPG, "a", 2000, 500, 1, 2, passed=True), + _trial(ARM_MCPG, "b", 2000, 500, 1, 2, passed=True), ] agg = aggregate(trials) assert agg["baseline"]["mean_total_tokens"] == pytest.approx(10000) @@ -78,8 +78,8 @@ def test_aggregate_token_ratio_and_correctness() -> None: def test_aggregate_excludes_errored_trials_and_counts_them() -> None: trials = [ - _trial(ARM_MCPG, "a", 2000, 500, 1, 2, True), - _trial(ARM_MCPG, "a", 999999, 999999, 9, 9, False, error="boom"), # must not pollute means + _trial(ARM_MCPG, "a", 2000, 500, 1, 2, passed=True), + _trial(ARM_MCPG, "a", 999999, 999999, 9, 9, passed=False, error="boom"), # must not pollute means ] agg = aggregate(trials) assert agg["mcpg"]["trials"] == 1 diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index dcead15c..1093c001 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -956,7 +956,7 @@ def test_new_config_parameters_loads_and_validates() -> None: settings = load_settings({"MCPG_DATABASE_URL": _DB_URL}) assert settings.http_max_body_bytes == 1048576 assert settings.http_allowed_origins == () - assert settings.http_hsts_max_age == 31536000 + assert settings.http_hsts_max_age == 63072000 assert settings.shutdown_drain_seconds == 30 assert settings.audit_hmac_key is None assert settings.audit_integrity is False @@ -1076,3 +1076,33 @@ def test_http_request_timeout_rejects_negative() -> None: "MCPG_HTTP_REQUEST_TIMEOUT_SECONDS": "-5", } ) + + +def test_http_allow_unauthenticated_defaults_false() -> None: + settings = load_settings({"MCPG_DATABASE_URL": _DB_URL}) + assert settings.http_allow_unauthenticated is False + + +def test_http_allow_unauthenticated_env_var_parses() -> None: + settings = load_settings( + { + "MCPG_DATABASE_URL": _DB_URL, + "MCPG_HTTP_ALLOW_UNAUTHENTICATED": "true", + } + ) + assert settings.http_allow_unauthenticated is True + + +def test_rate_limit_enabled_defaults_true() -> None: + settings = load_settings({"MCPG_DATABASE_URL": _DB_URL}) + assert settings.rate_limit_enabled is True + + +def test_rate_limit_enabled_can_still_be_disabled_explicitly() -> None: + settings = load_settings( + { + "MCPG_DATABASE_URL": _DB_URL, + "MCPG_RATE_LIMIT_ENABLED": "false", + } + ) + assert settings.rate_limit_enabled is False diff --git a/tests/unit/test_config_advisor.py b/tests/unit/test_config_advisor.py index 6e1a7a86..1a6eaae8 100644 --- a/tests/unit/test_config_advisor.py +++ b/tests/unit/test_config_advisor.py @@ -24,7 +24,7 @@ # =========================================================================== -def _seq_present(present: bool) -> dict[str, list[dict[str, object]]]: +def _seq_present(*, present: bool) -> dict[str, list[dict[str, object]]]: return {"to_regclass('pg_catalog.pg_sequences')": [{"present": present}]} @@ -45,7 +45,7 @@ async def test_sequences_rejects_critical_below_warning() -> None: async def test_sequences_unavailable_pre_pg10() -> None: - driver = FakeRoutingDriver(_seq_present(False)) + driver = FakeRoutingDriver(_seq_present(present=False)) result = await audit_sequences(driver) # type: ignore[arg-type] assert isinstance(result, SequenceAuditResult) assert result.available is False @@ -54,7 +54,7 @@ async def test_sequences_unavailable_pre_pg10() -> None: async def test_sequences_flags_critical_and_warning() -> None: routes: dict[str, list[dict[str, object]]] = {} - routes.update(_seq_present(True)) + routes.update(_seq_present(present=True)) routes.update( _seq_rows( [ @@ -96,7 +96,7 @@ async def test_sequences_flags_critical_and_warning() -> None: async def test_sequences_never_advanced_not_flagged() -> None: """A sequence with NULL last_value is counted but never at-risk.""" routes: dict[str, list[dict[str, object]]] = {} - routes.update(_seq_present(True)) + routes.update(_seq_present(present=True)) routes.update( _seq_rows( [{"schemaname": "public", "sequencename": "fresh_seq", "last_value": None, "max_value": 2_147_483_647}] @@ -111,7 +111,7 @@ async def test_sequences_never_advanced_not_flagged() -> None: async def test_sequences_remaining_headroom_computed() -> None: routes: dict[str, list[dict[str, object]]] = {} - routes.update(_seq_present(True)) + routes.update(_seq_present(present=True)) # min_value=0 so used_pct = (95-0)/(100-0) = 95.0 exactly. routes.update( _seq_rows( @@ -139,7 +139,7 @@ async def test_sequences_descending_supported() -> None: not max_value — used_pct must reflect the direction of travel. Regression for gemini review on #181.""" routes: dict[str, list[dict[str, object]]] = {} - routes.update(_seq_present(True)) + routes.update(_seq_present(present=True)) routes.update( _seq_rows( [ @@ -169,7 +169,7 @@ async def test_sequences_fresh_descending_not_flagged() -> None: """A just-started descending sequence near max_value has its whole range left — it must NOT be flagged (the false-positive case).""" routes: dict[str, list[dict[str, object]]] = {} - routes.update(_seq_present(True)) + routes.update(_seq_present(present=True)) routes.update( _seq_rows( [ diff --git a/tests/unit/test_cron.py b/tests/unit/test_cron.py index 5caad870..8f7965de 100644 --- a/tests/unit/test_cron.py +++ b/tests/unit/test_cron.py @@ -50,7 +50,7 @@ async def test_list_cron_jobs_maps_rows_when_extension_present() -> None: ) assert await list_cron_jobs(driver) == [ # type: ignore[arg-type] - CronJob(7, "*/5 * * * *", "SELECT 1", "app", "app_owner", True, "heartbeat") + CronJob(7, "*/5 * * * *", "SELECT 1", "app", "app_owner", active=True, jobname="heartbeat") ] diff --git a/tests/unit/test_cursor_cap.py b/tests/unit/test_cursor_cap.py index 129b8302..256dd9d1 100644 --- a/tests/unit/test_cursor_cap.py +++ b/tests/unit/test_cursor_cap.py @@ -10,7 +10,7 @@ from __future__ import annotations import asyncio -from typing import Any +from typing import Any, Self from unittest.mock import MagicMock import pytest @@ -19,7 +19,7 @@ class _FakeCursor: - async def __aenter__(self) -> _FakeCursor: + async def __aenter__(self) -> Self: return self async def __aexit__(self, *exc: object) -> None: diff --git a/tests/unit/test_data_movement.py b/tests/unit/test_data_movement.py index 9a269437..5150a7c8 100644 --- a/tests/unit/test_data_movement.py +++ b/tests/unit/test_data_movement.py @@ -182,7 +182,7 @@ async def test_export_table_rejects_invalid_identifier_characters() -> None: def test_export_formats_set_is_complete() -> None: - assert EXPORT_FORMATS == {"csv", "json"} + assert {"csv", "json"} == EXPORT_FORMATS def test_default_export_limit_is_a_sensible_ceiling() -> None: @@ -975,7 +975,7 @@ async def test_import_json_derives_columns_and_runs_parametrised_insert() -> Non assert result.format == "json" assert result.rows_imported == 2 sql, rows = db.execute_many_calls[0] - assert 'INSERT INTO "app"."widget" ("id", "name") VALUES (%s, %s)' == sql + assert sql == 'INSERT INTO "app"."widget" ("id", "name") VALUES (%s, %s)' assert rows == [(1, "alpha"), (2, "beta")] diff --git a/tests/unit/test_database.py b/tests/unit/test_database.py index 9109008d..08789bdc 100644 --- a/tests/unit/test_database.py +++ b/tests/unit/test_database.py @@ -80,7 +80,11 @@ def __init__(self) -> None: self.autocommit_calls: list[bool] = [] self.executed: list[str] = [] - async def set_autocommit(self, value: bool) -> None: + async def set_autocommit(self, value) -> None: + # Mirrors psycopg's AsyncConnection.set_autocommit(value: bool), + # called positionally by src/mcpg/database.py -- unannotated (rather + # than made keyword-only) so this fake keeps accepting the same + # calling convention as the real connection object it stands in for. self.autocommit_calls.append(value) async def execute(self, sql: str) -> None: diff --git a/tests/unit/test_ddl_dryrun.py b/tests/unit/test_ddl_dryrun.py index f88d98b5..8f56243e 100644 --- a/tests/unit/test_ddl_dryrun.py +++ b/tests/unit/test_ddl_dryrun.py @@ -4,7 +4,7 @@ from collections.abc import AsyncIterator from contextlib import asynccontextmanager -from typing import Any +from typing import Any, Self import pytest @@ -46,7 +46,7 @@ def __init__( self._last: str = "" self._lsn_calls = 0 - async def __aenter__(self) -> FakeCursor: + async def __aenter__(self) -> Self: return self async def __aexit__(self, *exc: object) -> None: diff --git a/tests/unit/test_demo.py b/tests/unit/test_demo.py index 9fb54af0..6ff7a956 100644 --- a/tests/unit/test_demo.py +++ b/tests/unit/test_demo.py @@ -45,7 +45,13 @@ def test_referential_integrity_of_generated_rows() -> None: assert 1 <= product_id <= len(dataset.products) assert 1 <= customer_id <= len(dataset.customers) assert 1 <= rating <= 5 - assert text + # Verify template interpolation actually happened: the product name OR + # at least one feature for that product type must be in the text. + # This catches bugs where template rendering failed or data was corrupted. + _, product_name, _, _, _ = dataset.products[product_id - 1] # (sku, name, category, price, desc) + product_type = product_name.split(" ", 1)[1] # Extract product type from "Adjective ProductType" + features = _FEATURES_BY_TYPE[product_type] + assert product_name in text or any(feature in text for feature in features) assert source in {"web", "mobile", "email_campaign"} diff --git a/tests/unit/test_dynamic_session_intent.py b/tests/unit/test_dynamic_session_intent.py index 8b9b95bf..d305809b 100644 --- a/tests/unit/test_dynamic_session_intent.py +++ b/tests/unit/test_dynamic_session_intent.py @@ -549,8 +549,8 @@ async def test_e2e_enable_session_intent_grows_the_real_tools_list() -> None: assert len(before_names) == 14 assert len(after_names) == 71, f"expected 71 tools after enabling monitor, got {len(after_names)}" assert before_names < after_names # strict growth: every core/always-kept tool survives, nothing lost - assert ALWAYS_KEEP <= before_names - assert ALWAYS_KEEP <= after_names + assert before_names >= ALWAYS_KEEP + assert after_names >= ALWAYS_KEEP assert "list_tables" in after_names # a core headline tool -- must still be visible after enabling monitor assert "list_active_queries" in after_names # operations_and_health, newly visible via monitor assert "list_active_queries" not in before_names # not visible under the core default alone diff --git a/tests/unit/test_errors.py b/tests/unit/test_errors.py new file mode 100644 index 00000000..76db0881 --- /dev/null +++ b/tests/unit/test_errors.py @@ -0,0 +1,40 @@ +"""MCPgError is the common ancestor every domain-specific error subclasses.""" + +from __future__ import annotations + +import importlib +import inspect +import pkgutil + +import mcpg +from mcpg.errors import MCPgError + + +def _iter_mcpg_modules() -> list[str]: + return [ + name + for _, name, is_pkg in pkgutil.walk_packages(mcpg.__path__, prefix="mcpg.") + if not is_pkg and "_vendor" not in name + ] + + +def test_every_domain_error_class_subclasses_mcpg_error() -> None: + offenders: list[str] = [] + for module_name in _iter_mcpg_modules(): + module = importlib.import_module(module_name) + for obj_name, obj in vars(module).items(): + if ( + inspect.isclass(obj) + and obj_name.endswith("Error") + and obj.__module__ == module_name + and issubclass(obj, Exception) + and obj is not MCPgError + and not issubclass(obj, MCPgError) + ): + offenders.append(f"{module_name}.{obj_name}") + assert not offenders, f"Exception classes not subclassing MCPgError: {offenders}" + + +def test_mcpg_error_is_a_plain_exception_subclass() -> None: + assert issubclass(MCPgError, Exception) + assert MCPgError.__doc__ # documented, not a bare pass-through diff --git a/tests/unit/test_graph.py b/tests/unit/test_graph.py index d095feac..c46357d1 100644 --- a/tests/unit/test_graph.py +++ b/tests/unit/test_graph.py @@ -5,12 +5,13 @@ import pytest from _fakes import FakeDatabase, FakeDriver, FakeRoutingDriver -from mcpg.config import Settings +from mcpg.config import AccessMode, Settings from mcpg.context import AppContext from mcpg.cursors import CursorManager from mcpg.database import DatabaseError from mcpg.graph import GraphError, describe_graph, list_graphs, parse_agtype from mcpg.listen import ListenManager +from mcpg.policy import Capability def test_parse_agtype_strips_vertex_edge_path() -> None: @@ -165,3 +166,80 @@ async def test_describe_graph_fetches_stats() -> None: assert len(stats["edge_labels"]) == 1 assert stats["edge_labels"][0]["label"] == "KNOWS" assert stats["edge_labels"][0]["count"] == 15 + + # The per-label row-count query must run READ ONLY. + count_calls = [c for c in fake_routing.calls if '"my_graph"."Person"' in c[0]] + assert count_calls and count_calls[0][2] is True + + +async def test_describe_graph_rejects_malicious_label_name() -> None: + """A label name read back from ag_catalog.ag_label is not identifier-safe. + + AGE/Postgres quoted identifiers can contain arbitrary characters, + including embedded double quotes. describe_graph interpolates the + label name directly into an f-string SQL query + (``FROM "{graph_name}"."{name}"``), so a label such as + ``Person"; DROP TABLE secrets; --`` must be rejected before it ever + reaches that query — not silently degrade to a zero count. + """ + malicious_name = 'Person"; DROP TABLE secrets; --' + fake_routing = FakeRoutingDriver( + { + "ag_label": [ + {"name": malicious_name, "kind": "v"}, + ], + "ag_graph": [{"name": "my_graph", "namespace": "my_graph"}], + } + ) + fake_db = FakeDatabase(fake_routing) # type: ignore[arg-type] + url = "postgresql://localhost/db" + settings = Settings(database_url=url) + context = AppContext( + settings=settings, + database=fake_db, # type: ignore[arg-type] + listen_manager=ListenManager(url), + cursor_manager=CursorManager(url), + ) + + with pytest.raises(GraphError, match="invalid label name"): + await describe_graph(context, "my_graph") + + # No query built from the malicious label name may have reached the driver. + assert not any("DROP TABLE" in str(call[0]) for call in fake_routing.calls) + + +async def test_describe_graph_enforces_access_mode_capability(monkeypatch: pytest.MonkeyPatch) -> None: + """describe_graph must gate on the same READ capability as its sibling + graph-read tools (cypher.run_cypher's read path, generate_graph_diagram). + + Capability.READ is permitted in every current access mode, so this + cannot be proven by a mode that actually blocks the call — instead we + spy on mcpg.graph.check_permission to confirm describe_graph invokes + the gate at all (it did not, prior to this fix). + """ + calls: list[tuple[Capability, AccessMode]] = [] + + def _spy(capability: Capability, access_mode: AccessMode) -> None: + calls.append((capability, access_mode)) + + monkeypatch.setattr("mcpg.graph.check_permission", _spy) + + fake_routing = FakeRoutingDriver( + { + "ag_label": [], + "ag_graph": [{"name": "my_graph", "namespace": "my_graph"}], + } + ) + fake_db = FakeDatabase(fake_routing) # type: ignore[arg-type] + url = "postgresql://localhost/db" + settings = Settings(database_url=url, access_mode=AccessMode.READ_ONLY) + context = AppContext( + settings=settings, + database=fake_db, # type: ignore[arg-type] + listen_manager=ListenManager(url), + cursor_manager=CursorManager(url), + ) + + await describe_graph(context, "my_graph") + + assert calls == [(Capability.READ, AccessMode.READ_ONLY)] diff --git a/tests/unit/test_graph_diagram.py b/tests/unit/test_graph_diagram.py index ee5e6ea6..18dee8c6 100644 --- a/tests/unit/test_graph_diagram.py +++ b/tests/unit/test_graph_diagram.py @@ -5,12 +5,13 @@ import pytest from _fakes import FakeDatabase, FakeRoutingDriver -from mcpg.config import Settings +from mcpg.config import AccessMode, Settings from mcpg.context import AppContext from mcpg.cursors import CursorManager from mcpg.graph import GraphError from mcpg.graph_diagram import generate_graph_diagram from mcpg.listen import ListenManager +from mcpg.policy import Capability async def test_generate_graph_diagram_validates_inputs() -> None: @@ -81,3 +82,77 @@ async def test_generate_graph_diagram_renders_mermaid() -> None: assert 'v844424930131969["Charlie"]' in mermaid assert 'v844424930131970["Dennis"]' in mermaid assert "v844424930131969 -->|KNOWS| v844424930131970" in mermaid + + # The per-label vertex/edge fetch queries must run READ ONLY. + vertex_calls = [c for c in fake_routing.calls if 'FROM "my_graph"."Person"' in c[0]] + edge_calls = [c for c in fake_routing.calls if 'FROM "my_graph"."KNOWS"' in c[0]] + assert vertex_calls and vertex_calls[0][2] is True + assert edge_calls and edge_calls[0][2] is True + + +async def test_generate_graph_diagram_rejects_malicious_label_name() -> None: + """A label name read back from ag_catalog.ag_label is not identifier-safe. + + generate_graph_diagram interpolates the label name directly into an + f-string SQL query (``FROM "{graph_name}"."{tbl}"``) and into the + generated Mermaid text, so a label such as + ``Person"; DROP TABLE secrets; --`` must be rejected up front rather + than silently reaching either sink. + """ + malicious_name = 'Person"; DROP TABLE secrets; --' + fake_routing = FakeRoutingDriver( + { + "ag_label": [ + {"name": malicious_name, "kind": "v"}, + ], + "ag_graph": [{"name": "my_graph"}], + } + ) + fake_db = FakeDatabase(fake_routing) # type: ignore[arg-type] + url = "postgresql://localhost/db" + settings = Settings(database_url=url) + context = AppContext( + settings=settings, + database=fake_db, # type: ignore[arg-type] + listen_manager=ListenManager(url), + cursor_manager=CursorManager(url), + ) + + with pytest.raises(GraphError, match="invalid label name"): + await generate_graph_diagram(context, "my_graph") + + # No query built from the malicious label name may have reached the driver. + assert not any("DROP TABLE" in str(call[0]) for call in fake_routing.calls) + + +async def test_generate_graph_diagram_enforces_access_mode_capability(monkeypatch: pytest.MonkeyPatch) -> None: + """generate_graph_diagram already gates on Capability.READ (graph_diagram.py:44); + this is a regression test confirming it stays wired to check_permission, + mirroring the equivalent test added for graph.describe_graph. + """ + calls: list[tuple[Capability, AccessMode]] = [] + + def _spy(capability: Capability, access_mode: AccessMode) -> None: + calls.append((capability, access_mode)) + + monkeypatch.setattr("mcpg.graph_diagram.check_permission", _spy) + + fake_routing = FakeRoutingDriver( + { + "ag_label": [], + "ag_graph": [{"name": "my_graph"}], + } + ) + fake_db = FakeDatabase(fake_routing) # type: ignore[arg-type] + url = "postgresql://localhost/db" + settings = Settings(database_url=url, access_mode=AccessMode.READ_ONLY) + context = AppContext( + settings=settings, + database=fake_db, # type: ignore[arg-type] + listen_manager=ListenManager(url), + cursor_manager=CursorManager(url), + ) + + await generate_graph_diagram(context, "my_graph") + + assert calls == [(Capability.READ, AccessMode.READ_ONLY)] diff --git a/tests/unit/test_graph_projection.py b/tests/unit/test_graph_projection.py index 5aedf7bb..faa86b97 100644 --- a/tests/unit/test_graph_projection.py +++ b/tests/unit/test_graph_projection.py @@ -25,14 +25,14 @@ def __init__(self, routes: dict[str, list[dict[str, Any]]], *, age: bool = True) self._age = age async def execute_query( - self, query: str, params: list[Any] | None = None, force_readonly: bool = False + self, query: str, params: list[Any] | None = None, *, force_readonly: bool = False ) -> list[SqlDriver.RowResult]: if "ag_catalog.ag_graph" in query: self.calls.append((query, params, force_readonly)) if not self._age: raise RuntimeError("relation ag_catalog.ag_graph does not exist") return [SqlDriver.RowResult(cells={"?column?": 1})] - return await super().execute_query(query, params, force_readonly) + return await super().execute_query(query, params, force_readonly=force_readonly) # Column-describe rows for two tables: authors(id) and books(id, author_id, title). @@ -98,11 +98,14 @@ def __init__( self._book_pk = book_pk self._table_rows = rows or {} - async def execute_query( - self, query: str, params: list[Any] | None = None, force_readonly: bool = False + # C901 rationale: test-only fake-driver query router matching on SQL + # substrings to return canned fixture rows -- refactoring test fixtures + # for a lint score is pure churn. + async def execute_query( # noqa: C901 + self, query: str, params: list[Any] | None = None, *, force_readonly: bool = False ) -> list[SqlDriver.RowResult]: if "ag_catalog.ag_graph" in query: - return await super().execute_query(query, params, force_readonly) + return await super().execute_query(query, params, force_readonly=force_readonly) self.calls.append((query, params, force_readonly)) p = params or [] table = p[1] if len(p) > 1 else None diff --git a/tests/unit/test_headline_curator.py b/tests/unit/test_headline_curator.py index 0b9bace5..c72f421e 100644 --- a/tests/unit/test_headline_curator.py +++ b/tests/unit/test_headline_curator.py @@ -13,7 +13,7 @@ ) -def _audit_present(present: bool) -> dict[str, list[dict[str, object]]]: +def _audit_present(*, present: bool) -> dict[str, list[dict[str, object]]]: return {"to_regclass('mcpg_audit.events')": [{"present": present}]} @@ -56,7 +56,7 @@ async def test_rejects_top_n_over_50() -> None: async def test_returns_diagnostic_when_audit_table_missing() -> None: - driver = FakeRoutingDriver(_audit_present(False)) + driver = FakeRoutingDriver(_audit_present(present=False)) report = await recommend_headline_tools(driver) # type: ignore[arg-type] assert isinstance(report, HeadlineRecommendationReport) assert report.audit_table_present is False @@ -66,7 +66,7 @@ async def test_returns_diagnostic_when_audit_table_missing() -> None: async def test_idle_window_returns_empty_recommendations_with_table_present() -> None: routes: dict[str, list[dict[str, object]]] = {} - routes.update(_audit_present(True)) + routes.update(_audit_present(present=True)) routes.update(_events_route([])) driver = FakeRoutingDriver(routes) report = await recommend_headline_tools(driver) # type: ignore[arg-type] @@ -86,7 +86,7 @@ async def test_top_n_per_bucket_respects_call_count_ranking() -> None: """Three schema_introspection tools at different call counts — they land in the schema_introspection bucket in DESC order.""" routes: dict[str, list[dict[str, object]]] = {} - routes.update(_audit_present(True)) + routes.update(_audit_present(present=True)) routes.update( _events_route( [ @@ -105,7 +105,7 @@ async def test_top_n_per_bucket_respects_call_count_ranking() -> None: async def test_top_n_caps_per_bucket() -> None: """top_n=2 → only the top two from each bucket reach `recommended`.""" routes: dict[str, list[dict[str, object]]] = {} - routes.update(_audit_present(True)) + routes.update(_audit_present(present=True)) routes.update( _events_route( [ @@ -123,7 +123,7 @@ async def test_top_n_caps_per_bucket() -> None: async def test_call_counts_are_propagated_to_recommendation() -> None: routes: dict[str, list[dict[str, object]]] = {} - routes.update(_audit_present(True)) + routes.update(_audit_present(present=True)) routes.update(_events_route([{"tool": "list_tables", "call_count": 42}])) driver = FakeRoutingDriver(routes) report = await recommend_headline_tools(driver) # type: ignore[arg-type] @@ -138,7 +138,7 @@ async def test_call_counts_are_propagated_to_recommendation() -> None: async def test_newcomers_flag_recommended_not_in_current() -> None: routes: dict[str, list[dict[str, object]]] = {} - routes.update(_audit_present(True)) + routes.update(_audit_present(present=True)) routes.update(_events_route([{"tool": "list_tables", "call_count": 1}])) driver = FakeRoutingDriver(routes) # Current headline doesn't include list_tables — it should land as a newcomer. @@ -152,7 +152,7 @@ async def test_newcomers_flag_recommended_not_in_current() -> None: async def test_departures_flag_current_not_in_recommended() -> None: routes: dict[str, list[dict[str, object]]] = {} - routes.update(_audit_present(True)) + routes.update(_audit_present(present=True)) routes.update(_events_route([{"tool": "list_tables", "call_count": 1}])) driver = FakeRoutingDriver(routes) report = await recommend_headline_tools( # type: ignore[arg-type] @@ -174,7 +174,7 @@ async def test_bucket_order_matches_curated_display_order() -> None: from mcpg.about import CAPABILITIES routes: dict[str, list[dict[str, object]]] = {} - routes.update(_audit_present(True)) + routes.update(_audit_present(present=True)) routes.update(_events_route([])) driver = FakeRoutingDriver(routes) report = await recommend_headline_tools(driver) # type: ignore[arg-type] @@ -192,7 +192,7 @@ async def test_query_filters_on_success_status() -> None: """The recommender only considers successful events — failures shouldn't push a flaky tool into a bucket's headline.""" routes: dict[str, list[dict[str, object]]] = {} - routes.update(_audit_present(True)) + routes.update(_audit_present(present=True)) routes.update(_events_route([{"tool": "list_tables", "call_count": 1}])) driver = FakeRoutingDriver(routes) await recommend_headline_tools(driver) # type: ignore[arg-type] diff --git a/tests/unit/test_http_runtime.py b/tests/unit/test_http_runtime.py index dcd3a8d7..ffdcf22e 100644 --- a/tests/unit/test_http_runtime.py +++ b/tests/unit/test_http_runtime.py @@ -16,6 +16,7 @@ build_http_app, ) from mcpg.observability import get_metrics, reset_metrics +from mcpg.oidc import OIDCVerifier from mcpg.tenancy import current_role @@ -94,11 +95,12 @@ async def send(message: dict[str, object]) -> None: def test_bearer_middleware_exempts_metrics_path_even_without_token() -> None: """A Prometheus scraper hits /metrics without the MCP bearer token.""" sent_messages: list[dict[str, object]] = [] - inner_invoked = False + captured_args: dict[str, object] = {} - async def inner(_scope: object, _receive: object, send_fn: object) -> None: - nonlocal inner_invoked - inner_invoked = True + async def inner(scope: object, receive: object, send_fn: object) -> None: + captured_args["scope"] = scope + captured_args["receive"] = receive + captured_args["send_fn"] = send_fn await send_fn( # type: ignore[operator] { "type": "http.response.start", @@ -121,17 +123,21 @@ async def send(message: dict[str, object]) -> None: asyncio.run(middleware(scope, receive, send)) - # The middleware passed through to the inner app without checking auth. - assert inner_invoked + # The middleware passed through to the inner app without checking auth, + # and without modifying the scope/receive/send objects. + assert captured_args["scope"] is scope + assert captured_args["receive"] is receive + assert captured_args["send_fn"] is send assert sent_messages[0]["status"] == 200 def test_bearer_middleware_passes_non_http_scopes_through_unmodified() -> None: - inner_invoked = False + captured_args: dict[str, object] = {} - async def inner(scope: object, _receive: object, _send: object) -> None: - nonlocal inner_invoked - inner_invoked = True + async def inner(scope: object, receive: object, send: object) -> None: + captured_args["scope"] = scope + captured_args["receive"] = receive + captured_args["send"] = send # Lifespan scopes must reach the underlying ASGI app or the # server never starts up. assert scope["type"] == "lifespan" # type: ignore[index] @@ -139,11 +145,20 @@ async def inner(scope: object, _receive: object, _send: object) -> None: middleware = _BearerAuthMiddleware(inner, token="s3cr3t") scope = {"type": "lifespan"} + async def receive_fn() -> dict[str, object]: + return {} + + async def send_fn(_message: dict[str, object]) -> None: + pass + import asyncio - asyncio.run(middleware(scope, lambda: None, lambda _: None)) # type: ignore[arg-type] + asyncio.run(middleware(scope, receive_fn, send_fn)) # type: ignore[arg-type] - assert inner_invoked + # Verify non-HTTP scope is passed through unmodified, without wrapping. + assert captured_args["scope"] is scope + assert captured_args["receive"] is receive_fn + assert captured_args["send"] is send_fn def test_auth_exempt_paths_includes_metrics_and_health_endpoints() -> None: @@ -171,7 +186,12 @@ def test_build_http_app_serves_metrics_with_observability_payload() -> None: # Record one observation so the /metrics body has something to assert on. get_metrics().record_call("smoke_test", "ok", 0.05) - settings = load_settings({"MCPG_DATABASE_URL": "postgresql://u:p@localhost/db"}) + settings = load_settings( + { + "MCPG_DATABASE_URL": "postgresql://u:p@localhost/db", + "MCPG_HTTP_ALLOW_UNAUTHENTICATED": "true", + } + ) class _Stub: def streamable_http_app(self, *, host: str = "127.0.0.1") -> Starlette: @@ -187,7 +207,12 @@ def streamable_http_app(self, *, host: str = "127.0.0.1") -> Starlette: def test_build_http_app_serves_healthz_unauthenticated() -> None: - settings = load_settings({"MCPG_DATABASE_URL": "postgresql://u:p@localhost/db"}) + settings = load_settings( + { + "MCPG_DATABASE_URL": "postgresql://u:p@localhost/db", + "MCPG_HTTP_ALLOW_UNAUTHENTICATED": "true", + } + ) class _Stub: def streamable_http_app(self, *, host: str = "127.0.0.1") -> Starlette: @@ -200,6 +225,77 @@ def streamable_http_app(self, *, host: str = "127.0.0.1") -> Starlette: assert response.text.startswith("ok") +def test_readyz_returns_200_when_database_is_connected() -> None: + """/readyz reports ready once Database.is_connected is True.""" + settings = load_settings( + { + "MCPG_DATABASE_URL": "postgresql://u:p@localhost/db", + "MCPG_HTTP_ALLOW_UNAUTHENTICATED": "true", + } + ) + + class _FakeDatabase: + is_connected = True + + class _Stub: + mcpg_database = _FakeDatabase() + + def streamable_http_app(self, *, host: str = "127.0.0.1") -> Starlette: + return _bare_app() + + wrapped = build_http_app(_Stub(), settings, kind="streamable-http") + with TestClient(wrapped) as client: + response = client.get("/readyz") + assert response.status_code == 200 + assert response.text.startswith("ready") + + +def test_readyz_returns_503_when_database_is_not_connected() -> None: + """/readyz reports not-ready when Database.is_connected is False.""" + settings = load_settings( + { + "MCPG_DATABASE_URL": "postgresql://u:p@localhost/db", + "MCPG_HTTP_ALLOW_UNAUTHENTICATED": "true", + } + ) + + class _FakeDatabase: + is_connected = False + + class _Stub: + mcpg_database = _FakeDatabase() + + def streamable_http_app(self, *, host: str = "127.0.0.1") -> Starlette: + return _bare_app() + + wrapped = build_http_app(_Stub(), settings, kind="streamable-http") + with TestClient(wrapped) as client: + response = client.get("/readyz") + assert response.status_code == 503 + assert response.text.startswith("not ready") + + +def test_readyz_returns_200_when_server_never_wired_a_database() -> None: + """A build_http_app caller that never set mcpg_database (e.g. the bare + stubs used elsewhere in this file) has nothing to assess, so /readyz + reports ready rather than failing a check that was never wired up.""" + settings = load_settings( + { + "MCPG_DATABASE_URL": "postgresql://u:p@localhost/db", + "MCPG_HTTP_ALLOW_UNAUTHENTICATED": "true", + } + ) + + class _Stub: + def streamable_http_app(self, *, host: str = "127.0.0.1") -> Starlette: + return _bare_app() + + wrapped = build_http_app(_Stub(), settings, kind="streamable-http") + with TestClient(wrapped) as client: + response = client.get("/readyz") + assert response.status_code == 200 + + def test_build_http_app_with_token_blocks_unauthenticated_requests() -> None: settings = load_settings( { @@ -228,6 +324,37 @@ def streamable_http_app(self, *, host: str = "127.0.0.1") -> Starlette: assert response.status_code == 200 +def test_build_http_app_raises_without_auth_or_opt_out() -> None: + """HTTP transport refuses to start unauthenticated unless explicitly opted out.""" + from mcpg.config import ConfigError + + settings = load_settings({"MCPG_DATABASE_URL": "postgresql://u:p@localhost/db"}) + + class _Stub: + def streamable_http_app(self, *, host: str = "127.0.0.1") -> Starlette: + return _bare_app() + + with pytest.raises(ConfigError, match="unauthenticated"): + build_http_app(_Stub(), settings, kind="streamable-http") + + +def test_build_http_app_starts_with_explicit_opt_out() -> None: + """The MCPG_HTTP_ALLOW_UNAUTHENTICATED escape hatch still works, loudly logged.""" + settings = load_settings( + { + "MCPG_DATABASE_URL": "postgresql://u:p@localhost/db", + "MCPG_HTTP_ALLOW_UNAUTHENTICATED": "true", + } + ) + + class _Stub: + def streamable_http_app(self, *, host: str = "127.0.0.1") -> Starlette: + return _bare_app() + + app = build_http_app(_Stub(), settings, kind="streamable-http") + assert app is not None + + # --- per-request role multi-tenancy (Phase 1.4) -------------------------- @@ -352,11 +479,12 @@ async def send(_message: dict[str, object]) -> None: def test_tenant_role_middleware_exempts_health_paths() -> None: """A probe to /healthz doesn't need the X-MCPG-Role header.""" - inner_invoked = False + captured_args: dict[str, object] = {} - async def inner(_scope: object, _receive: object, send_fn: object) -> None: - nonlocal inner_invoked - inner_invoked = True + async def inner(scope: object, receive: object, send_fn: object) -> None: + captured_args["scope"] = scope + captured_args["receive"] = receive + captured_args["send_fn"] = send_fn await send_fn( # type: ignore[operator] {"type": "http.response.start", "status": 200, "headers": []} ) @@ -377,7 +505,10 @@ async def send(_message: dict[str, object]) -> None: asyncio.run(middleware(scope, receive, send)) - assert inner_invoked + # Verify health paths are exempt and pass through unmodified. + assert captured_args["scope"] is scope + assert captured_args["receive"] is receive + assert captured_args["send_fn"] is send def test_build_http_app_with_tenant_role_returns_403_for_unknown_role() -> None: @@ -386,6 +517,7 @@ def test_build_http_app_with_tenant_role_returns_403_for_unknown_role() -> None: "MCPG_DATABASE_URL": "postgresql://u:p@localhost/db", "MCPG_DEFAULT_ROLE": "tenant_a", "MCPG_ALLOWED_ROLES": "tenant_a,tenant_b", + "MCPG_HTTP_ALLOW_UNAUTHENTICATED": "true", } ) @@ -456,11 +588,56 @@ def streamable_http_app(self, *, host: str = "127.0.0.1") -> Starlette: # Wrong token → 401 (verification fails — discovery never reached). response = client.get("/", headers={"Authorization": "Bearer not.a.real.jwt"}) assert response.status_code == 401 - # /metrics and /healthz still bypass auth. + # /metrics, /healthz, and /readyz still bypass auth. response = client.get("/metrics") assert response.status_code == 200 response = client.get("/healthz") assert response.status_code == 200 + response = client.get("/readyz") + assert response.status_code == 200 + + +def test_build_http_app_in_oidc_mode_closes_the_verifier_client_on_lifespan_shutdown( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The OIDCVerifier is built inside build_http_app — after + mcpg.server.make_lifespan's closure already exists — so there's no + way to reach it from that closure's own `finally`. build_http_app + instead wraps the app's own `router.lifespan_context` so the + verifier's HTTP client still gets closed when the ASGI lifespan + (entered by TestClient's context manager here, uvicorn in + production) shuts down.""" + closed: list[bool] = [] + + class _TrackedVerifier(OIDCVerifier): + async def aclose(self) -> None: + closed.append(True) + await super().aclose() + + import mcpg.http_runtime as http_runtime + + monkeypatch.setattr(http_runtime, "OIDCVerifier", _TrackedVerifier) + + settings = load_settings( + { + "MCPG_DATABASE_URL": "postgresql://u:p@localhost/db", + "MCPG_AUTH_MODE": "oidc", + "MCPG_OIDC_ISSUER": "https://issuer.example", + "MCPG_OIDC_AUDIENCE": "mcpg", + } + ) + + class _Stub: + def streamable_http_app(self, *, host: str = "127.0.0.1") -> Starlette: + return _bare_app() + + wrapped = build_http_app(_Stub(), settings, kind="streamable-http") + assert closed == [] # not yet closed while the app is "running" + with TestClient(wrapped) as client: + client.get("/metrics") + assert closed == [] + + assert closed == [True] # --- HTTP hardening middlewares (Security headers, CORS, request size limit) --- @@ -508,6 +685,7 @@ def test_cors_middleware_integration() -> None: { "MCPG_DATABASE_URL": "postgresql://u:p@localhost/db", "MCPG_HTTP_ALLOWED_ORIGINS": "http://localhost:3000, https://app.example.com", + "MCPG_HTTP_ALLOW_UNAUTHENTICATED": "true", } ) @@ -597,6 +775,7 @@ def test_cors_middleware_negative_and_default_config() -> None: { "MCPG_DATABASE_URL": "postgresql://u:p@localhost/db", "MCPG_HTTP_ALLOWED_ORIGINS": "", + "MCPG_HTTP_ALLOW_UNAUTHENTICATED": "true", } ) @@ -619,6 +798,7 @@ def streamable_http_app(self, *, host: str = "127.0.0.1") -> Starlette: { "MCPG_DATABASE_URL": "postgresql://u:p@localhost/db", "MCPG_HTTP_ALLOWED_ORIGINS": "https://app.example.com", + "MCPG_HTTP_ALLOW_UNAUTHENTICATED": "true", } ) wrapped_allowlist = build_http_app(_Stub(), settings_allowlist, kind="streamable-http") @@ -817,6 +997,7 @@ def test_build_http_app_passes_configured_host_to_streamable_http_app() -> None: "MCPG_DATABASE_URL": "postgresql://u:p@localhost/db", "MCPG_HTTP_HOST": "0.0.0.0", "MCPG_HTTP_PORT": "9999", + "MCPG_HTTP_ALLOW_UNAUTHENTICATED": "true", } ) @@ -841,6 +1022,7 @@ def test_build_http_app_passes_configured_host_to_sse_app() -> None: "MCPG_DATABASE_URL": "postgresql://u:p@localhost/db", "MCPG_HTTP_HOST": "0.0.0.0", "MCPG_HTTP_PORT": "9999", + "MCPG_HTTP_ALLOW_UNAUTHENTICATED": "true", } ) @@ -857,7 +1039,12 @@ def sse_app(self, *, host: str) -> Starlette: def test_build_http_app_supports_sse_kind() -> None: - settings = load_settings({"MCPG_DATABASE_URL": "postgresql://u:p@localhost/db"}) + settings = load_settings( + { + "MCPG_DATABASE_URL": "postgresql://u:p@localhost/db", + "MCPG_HTTP_ALLOW_UNAUTHENTICATED": "true", + } + ) class _Stub: def sse_app(self, *, host: str = "127.0.0.1") -> Starlette: @@ -877,6 +1064,7 @@ def test_run_http_builds_app_and_serves_via_uvicorn(monkeypatch: pytest.MonkeyPa "MCPG_DATABASE_URL": "postgresql://u:p@localhost/db", "MCPG_HTTP_HOST": "0.0.0.0", "MCPG_HTTP_PORT": "9999", + "MCPG_HTTP_ALLOW_UNAUTHENTICATED": "true", } ) @@ -919,7 +1107,12 @@ def test_run_http_falls_back_to_private_policy_name_when_public_name_raises( from mcpg import http_runtime - settings = load_settings({"MCPG_DATABASE_URL": "postgresql://u:p@localhost/db"}) + settings = load_settings( + { + "MCPG_DATABASE_URL": "postgresql://u:p@localhost/db", + "MCPG_HTTP_ALLOW_UNAUTHENTICATED": "true", + } + ) captured_kwargs: dict[str, object] = {} @@ -964,7 +1157,12 @@ def test_run_http_pins_selector_loop_on_windows(monkeypatch: pytest.MonkeyPatch) from mcpg import http_runtime - settings = load_settings({"MCPG_DATABASE_URL": "postgresql://u:p@localhost/db"}) + settings = load_settings( + { + "MCPG_DATABASE_URL": "postgresql://u:p@localhost/db", + "MCPG_HTTP_ALLOW_UNAUTHENTICATED": "true", + } + ) captured_kwargs: dict[str, object] = {} @@ -1005,7 +1203,12 @@ def test_run_http_leaves_the_event_loop_alone_off_windows(monkeypatch: pytest.Mo from mcpg import http_runtime - settings = load_settings({"MCPG_DATABASE_URL": "postgresql://u:p@localhost/db"}) + settings = load_settings( + { + "MCPG_DATABASE_URL": "postgresql://u:p@localhost/db", + "MCPG_HTTP_ALLOW_UNAUTHENTICATED": "true", + } + ) captured_kwargs: dict[str, object] = {} @@ -1285,7 +1488,10 @@ def test_settings_rejects_nonexistent_cert_path() -> None: def test_build_http_app_installs_request_timeout_only_when_positive() -> None: - base = {"MCPG_DATABASE_URL": "postgresql://u:p@localhost/db"} + base = { + "MCPG_DATABASE_URL": "postgresql://u:p@localhost/db", + "MCPG_HTTP_ALLOW_UNAUTHENTICATED": "true", + } class _Stub: def streamable_http_app(self, *, host: str = "127.0.0.1") -> Starlette: @@ -1444,7 +1650,10 @@ async def _inner_app(scope: dict[str, object], _receive: object, _send: object) def test_build_http_app_installs_ip_allowlist_only_when_configured() -> None: - base = {"MCPG_DATABASE_URL": "postgresql://u:p@localhost/db"} + base = { + "MCPG_DATABASE_URL": "postgresql://u:p@localhost/db", + "MCPG_HTTP_ALLOW_UNAUTHENTICATED": "true", + } class _Stub: def streamable_http_app(self, *, host: str = "127.0.0.1") -> Starlette: @@ -1463,6 +1672,33 @@ def streamable_http_app(self, *, host: str = "127.0.0.1") -> Starlette: assert any(m.cls.__name__ == "_IPAllowlistMiddleware" for m in on.user_middleware) +def test_build_http_app_installs_trusted_host_middleware_only_when_configured() -> None: + from starlette.middleware.trustedhost import TrustedHostMiddleware + + base = { + "MCPG_DATABASE_URL": "postgresql://u:p@localhost/db", + "MCPG_HTTP_ALLOW_UNAUTHENTICATED": "true", + } + + class _Stub: + def streamable_http_app(self, *, host: str = "127.0.0.1") -> Starlette: + return _bare_app() + + # No trusted hosts configured -> middleware not installed. + off = build_http_app(_Stub(), load_settings(base), kind="streamable-http") + off_classes = [m.cls for m in off.user_middleware] + assert TrustedHostMiddleware not in off_classes + + # Trusted hosts configured -> middleware installed. + on = build_http_app( + _Stub(), + load_settings({**base, "MCPG_HTTP_TRUSTED_HOSTS": "api.example.com"}), + kind="streamable-http", + ) + on_classes = [m.cls for m in on.user_middleware] + assert TrustedHostMiddleware in on_classes + + def test_settings_rejects_invalid_ip_allowlist_entry() -> None: from mcpg.config import ConfigError diff --git a/tests/unit/test_introspection.py b/tests/unit/test_introspection.py index 599c0357..6b8ee8b1 100644 --- a/tests/unit/test_introspection.py +++ b/tests/unit/test_introspection.py @@ -465,7 +465,7 @@ def __init__(self) -> None: self.calls: list[tuple[str, list[Any] | None]] = [] async def execute_query( - self, query: str, params: list[Any] | None = None, force_readonly: bool = False + self, query: str, params: list[Any] | None = None, *, force_readonly: bool = False ) -> list[Any]: del force_readonly self.calls.append((query, params)) @@ -639,8 +639,8 @@ async def test_list_domains_maps_rows_including_constraints() -> None: ) assert await list_domains(driver, "app") == [ - DomainInfo("positive_int", "integer", False, "0", ["CHECK ((VALUE > 0))"]), - DomainInfo("free_text", "text", True, None, []), + DomainInfo("positive_int", "integer", nullable=False, default="0", constraints=["CHECK ((VALUE > 0))"]), + DomainInfo("free_text", "text", nullable=True, default=None, constraints=[]), ] @@ -782,8 +782,26 @@ async def test_list_publications_maps_rows_including_tables() -> None: ) assert await list_publications(driver) == [ - PublicationInfo("widget_pub", "app_owner", False, True, True, False, False, ["app.widget", "app.event"]), - PublicationInfo("everything_pub", "postgres", True, True, True, True, True, []), + PublicationInfo( + "widget_pub", + "app_owner", + all_tables=False, + publishes_insert=True, + publishes_update=True, + publishes_delete=False, + publishes_truncate=False, + tables=["app.widget", "app.event"], + ), + PublicationInfo( + "everything_pub", + "postgres", + all_tables=True, + publishes_insert=True, + publishes_update=True, + publishes_delete=True, + publishes_truncate=True, + tables=[], + ), ] @@ -801,7 +819,9 @@ async def test_list_subscriptions_maps_rows() -> None: ) assert await list_subscriptions(driver) == [ - SubscriptionInfo("widget_sub", "app_owner", True, "host=upstream dbname=app", ["widget_pub"]), + SubscriptionInfo( + "widget_sub", "app_owner", enabled=True, connection="host=upstream dbname=app", publications=["widget_pub"] + ), ] @@ -862,7 +882,7 @@ async def test_introspection_tools_are_registered() -> None: async with create_connected_server_and_client_session(server) as client: listed = {tool.name for tool in (await client.list_tools()).tools} - assert _INTROSPECTION_TOOLS <= listed + assert listed >= _INTROSPECTION_TOOLS # --- list_generated_columns (Phase 4.7) --------------------------------- diff --git a/tests/unit/test_listen.py b/tests/unit/test_listen.py index 5875798e..102832b3 100644 --- a/tests/unit/test_listen.py +++ b/tests/unit/test_listen.py @@ -94,6 +94,24 @@ async def factory() -> _FakeConn: return manager, conn +class _FailingCloseConn(_FakeConn): + """A connection whose ``close()`` raises — simulates a half-open TCP + socket or libpq quirk during shutdown.""" + + async def close(self) -> None: + raise RuntimeError("connection already reset by peer") + + +def _manager_with_failing_close_conn() -> tuple[ListenManager, _FailingCloseConn]: + conn = _FailingCloseConn() + + async def factory() -> _FailingCloseConn: + return conn + + manager = ListenManager(database_url="postgresql:///x", connection_factory=factory) + return manager, conn + + # --- subscribe / unsubscribe / poll lifecycle ---------------------------- @@ -302,6 +320,33 @@ async def test_close_is_idempotent_and_cancels_the_reader_task() -> None: assert conn.closed is True +async def test_close_logs_debug_when_connection_close_fails(caplog: pytest.LogCaptureFixture) -> None: + """A failed best-effort ``conn.close()`` during shutdown is swallowed + (close() must never raise) but now logs at debug rather than silently.""" + import logging + + manager, _conn = _manager_with_failing_close_conn() + await manager.subscribe("orders") + + root_logger = logging.getLogger("mcpg") + old_propagate = root_logger.propagate + root_logger.propagate = True + try: + caplog.set_level(logging.DEBUG, logger="mcpg.listen") + await manager.close() # must not raise despite conn.close() failing + finally: + root_logger.propagate = old_propagate + + matches = [ + r + for r in caplog.records + if r.name == "mcpg.listen" and "Best-effort connection close" in r.message and r.levelno == logging.DEBUG + ] + assert len(matches) == 1 + # exc_info=True must actually attach a traceback, not just the message. + assert matches[0].exc_info is not None + + async def test_subscribe_after_close_raises() -> None: manager, _ = _manager_with_fake_conn() await manager.close() diff --git a/tests/unit/test_migration_history.py b/tests/unit/test_migration_history.py index 902b341e..796033f4 100644 --- a/tests/unit/test_migration_history.py +++ b/tests/unit/test_migration_history.py @@ -211,11 +211,11 @@ async def test_read_migration_history_schema_filter() -> None: async def test_read_migration_history_resilient_to_errors() -> None: class FailingRoutingDriver(FakeRoutingDriver): async def execute_query( - self, query: str, params: list[Any] | None = None, force_readonly: bool = False + self, query: str, params: list[Any] | None = None, *, force_readonly: bool = False ) -> list[SqlDriver.RowResult]: if "alembic_version" in query and "version_num" in query: raise RuntimeError("database error") - return await super().execute_query(query, params, force_readonly) + return await super().execute_query(query, params, force_readonly=force_readonly) driver = FailingRoutingDriver( { diff --git a/tests/unit/test_migrations.py b/tests/unit/test_migrations.py index c81baa96..51b11397 100644 --- a/tests/unit/test_migrations.py +++ b/tests/unit/test_migrations.py @@ -3,6 +3,7 @@ from __future__ import annotations from datetime import UTC, datetime, timedelta +from typing import Any import pytest from _fakes import FakeDatabase, FakeDriver @@ -62,12 +63,15 @@ def test_shadow_name_starts_with_the_documented_prefix() -> None: def test_column_clause_builds_create_table_fragments() -> None: class _Col: - def __init__(self, name: str, data_type: str, nullable: bool, default: str | None) -> None: + def __init__(self, name: str, data_type: str, *, nullable: bool, default: str | None) -> None: self.name, self.data_type, self.nullable, self.default = name, data_type, nullable, default - assert _column_clause(_Col("id", "integer", False, None)) == '"id" integer NOT NULL' - assert _column_clause(_Col("name", "text", True, None)) == '"name" text' - assert _column_clause(_Col("flag", "boolean", False, "false")) == '"flag" boolean NOT NULL DEFAULT false' + assert _column_clause(_Col("id", "integer", nullable=False, default=None)) == '"id" integer NOT NULL' + assert _column_clause(_Col("name", "text", nullable=True, default=None)) == '"name" text' + assert ( + _column_clause(_Col("flag", "boolean", nullable=False, default="false")) + == '"flag" boolean NOT NULL DEFAULT false' + ) def test_rewrite_schema_reference_rewrites_only_the_target_schema_in_fk_defs() -> None: @@ -190,6 +194,65 @@ async def test_prepare_migration_rejects_unsafe_target_schema() -> None: ) +class _FailingDropSchemaDriver: + """FakeRoutingDriver-alike whose ``DROP SCHEMA`` statements always + raise — used to exercise the "shadow cleanup itself fails" branch of + ``prepare_migration``'s except handler, on top of an outer failure + (a non-transactional candidate statement) that triggers the cleanup + in the first place.""" + + def __init__(self) -> None: + self.calls: list[Any] = [] + + async def execute_query(self, query: str, params: Any = None, *, force_readonly: bool = False) -> Any: + self.calls.append((query, params, force_readonly)) + if "DROP SCHEMA" in query: + raise RuntimeError("cannot drop schema: schema is being accessed by other users") + return [] + + +async def test_prepare_migration_logs_debug_when_shadow_cleanup_also_fails( + caplog: pytest.LogCaptureFixture, +) -> None: + """When the candidate SQL fails to apply AND the shadow-schema cleanup + itself fails, the cleanup failure is now logged at debug (not + silently), and the original error still propagates via the trailing + ``raise``.""" + import logging + + from mcpg.migrations import prepare_migration + + driver = _FailingDropSchemaDriver() + + root_logger = logging.getLogger("mcpg") + old_propagate = root_logger.propagate + root_logger.propagate = True + try: + caplog.set_level(logging.DEBUG, logger="mcpg.migrations") + + # VACUUM cannot run inside a transaction block — _execute_in_schema + # rejects it immediately, entering prepare_migration's cleanup path + # without needing a real connection pool. + with pytest.raises(MigrationError, match="cannot run inside a transaction"): + await prepare_migration( + driver, # type: ignore[arg-type] + name="bad_vacuum", + target_schema="app", + candidate_sql="VACUUM;", + ) + finally: + root_logger.propagate = old_propagate + + matches = [ + r + for r in caplog.records + if r.name == "mcpg.migrations" and "shadow schema" in r.message.lower() and r.levelno == logging.DEBUG + ] + assert len(matches) == 1 + # exc_info=True must actually attach a traceback, not just the message. + assert matches[0].exc_info is not None + + # --- validate_migration (Phase 9.2) ------------------------------- diff --git a/tests/unit/test_multidb.py b/tests/unit/test_multidb.py index 7c257070..bf7e6081 100644 --- a/tests/unit/test_multidb.py +++ b/tests/unit/test_multidb.py @@ -13,6 +13,7 @@ from __future__ import annotations from dataclasses import FrozenInstanceError +from typing import Self import pytest from _fakes import FakePool @@ -74,7 +75,7 @@ def __init__(self, conn: _FakeConnection, *, rows: list[dict[str, object]]) -> N self._rows = rows self.description: object | None = None - async def __aenter__(self) -> _FakeCursor: + async def __aenter__(self) -> Self: return self async def __aexit__(self, *exc: object) -> None: diff --git a/tests/unit/test_multidb_cache.py b/tests/unit/test_multidb_cache.py index ea7a52b8..b24a2bc0 100644 --- a/tests/unit/test_multidb_cache.py +++ b/tests/unit/test_multidb_cache.py @@ -119,8 +119,12 @@ async def _run() -> str: calls.append(1) return "primary-value" - a = await _cached_call(ctx, "list_schemas", _run, True, database=None) - b = await _cached_call(ctx, "list_schemas", _run, True, database="primary") + # `include_system` here is a *key_args cache-key component (mirrors + # the real `include_system` call site), not a flag parameter -- + # _cached_call's *key_args is variadic-positional by design. + include_system = True + a = await _cached_call(ctx, "list_schemas", _run, include_system, database=None) + b = await _cached_call(ctx, "list_schemas", _run, include_system, database="primary") assert a == b == "primary-value" assert calls == [1] # None normalises to "primary" — one shared entry diff --git a/tests/unit/test_nl2sql.py b/tests/unit/test_nl2sql.py index 8daf8a94..0721ae03 100644 --- a/tests/unit/test_nl2sql.py +++ b/tests/unit/test_nl2sql.py @@ -2,8 +2,10 @@ from __future__ import annotations +from collections.abc import Iterator +from contextlib import contextmanager from dataclasses import dataclass -from typing import Any +from typing import Any, Self from unittest.mock import patch import httpx @@ -18,6 +20,7 @@ DEFAULT_SCHEMA_DENYLIST, HARD_MAX_BRIEF_CHARS, HARD_MAX_TOKENS, + NL2SQL_CIRCUIT_FAILURE_THRESHOLD, OPENAI_COMPATIBLE_BASE_URLS, VENDOR_ENV_VAR_HINT, VENDOR_KEY_ENV_VARS, @@ -31,6 +34,7 @@ _assert_single_statement, _parse_response, _reset_egress_notice_cache, + _reset_shared_http_client, _resolve_schema_policy, _sanitize_default_expr, _validate_schema_name, @@ -41,6 +45,28 @@ ) +@pytest.fixture(autouse=True) +def _reset_circuit_breakers(): + """Reset every registered circuit breaker before/after each test. + + ``@circuit`` decorates each provider's ``complete`` once at class + definition time, so the breaker's failure count is a single object + shared across every provider *instance* for the life of the test + process (production builds a fresh provider per call — see + ``build_provider`` — so per-instance state wouldn't accumulate + failures at all; the shared-at-class-level design is intentional). + Without this reset, a test that trips a breaker open would leave it + open for every unrelated test that runs afterward in the same session. + """ + from circuitbreaker import CircuitBreakerMonitor + + for cb in CircuitBreakerMonitor.get_circuits(): + cb.reset() + yield + for cb in CircuitBreakerMonitor.get_circuits(): + cb.reset() + + @dataclass class _StubProvider: """LLMProvider double — returns whatever ``response`` was given. @@ -64,7 +90,7 @@ async def complete( user_prompt: str, model: str, max_tokens: int, - timeout: float, + timeout: float, # noqa: ASYNC109 -- must match the LLMProvider Protocol's signature exactly ) -> ProviderCompletion: _ = timeout self.captured_system = system_prompt @@ -535,6 +561,24 @@ async def test_translate_nl_to_sql_records_provider_and_model_on_the_result() -> assert result.provider == "stub" +async def test_translation_result_records_the_schema_context_it_saw() -> None: + """A caller can trace generated SQL back to the schema evidence the model was given.""" + provider = _StubProvider(response='{"sql": "SELECT count(*) FROM public.widget", "explanation": "row count"}') + + result = await translate_nl_to_sql( + FakeRoutingDriver(_routes_for_simple_schema()), # type: ignore[arg-type] + provider=provider, # type: ignore[arg-type] + model="m", + question="how many widgets are there?", + schema="public", + ) + + assert result.schema_context # non-empty + assert "widget" in result.schema_context # the table the question is actually about was included + # It's the exact brief sent to the model, not a re-derived copy. + assert result.schema_context in provider.captured_user + + async def test_translate_nl_to_sql_reports_the_providers_token_usage() -> None: """The internal provider call's tokens never go through a caller's own model loop — TranslationResult is the only place they're visible, so @@ -1106,7 +1150,7 @@ async def test_translate_nl_to_sql_emits_egress_warning_once_per_provider() -> N # Exactly one provider — "stub" — is in the cache; if the warning # fired twice, the set membership doesn't change but the test # below would catch any logic that bypassed the cache. - assert nl2sql_mod._EGRESS_NOTICE_LOGGED == {"stub"} + assert {"stub"} == nl2sql_mod._EGRESS_NOTICE_LOGGED async def test_translate_nl_to_sql_egress_warning_fires_per_distinct_provider() -> None: @@ -1127,7 +1171,7 @@ async def test_translate_nl_to_sql_egress_warning_fires_per_distinct_provider() question="x", schema="public", ) - assert nl2sql_mod._EGRESS_NOTICE_LOGGED == {"anthropic_x", "openai_x", "gemini_x"} + assert {"anthropic_x", "openai_x", "gemini_x"} == nl2sql_mod._EGRESS_NOTICE_LOGGED # --- P2 #6 — QueryError redaction ---------------------------------------- @@ -1144,13 +1188,13 @@ async def test_query_error_message_is_redacted_in_translation_result() -> None: # exercise the QueryError branch via the safety stack. class _RaisingDriver(FakeRoutingDriver): - async def execute_query(self, query, params=None, force_readonly=False): # type: ignore[override] + async def execute_query(self, query, params=None, *, force_readonly=False): # type: ignore[override] if "SELECT count(*)" in query and "public.widget" in query: # Simulate a libpq error with an embedded credential. from mcpg.query import QueryError raise QueryError("could not connect to postgres://alice:hunter2@db/x") - return await super().execute_query(query, params, force_readonly) + return await super().execute_query(query, params, force_readonly=force_readonly) provider = _StubProvider(response='{"sql": "SELECT count(*) FROM public.widget", "explanation": "x"}') result = await translate_nl_to_sql( @@ -1256,8 +1300,18 @@ def test_parse_response_extracts_fence_body_over_outer_garbage() -> None: # provider classes' usage-block parsing is actually exercised. -def _mock_post_response(body: dict[str, Any]): - """Patch ``mcpg.nl2sql.httpx`` so any POST returns ``body`` as JSON.""" +@contextmanager +def _mock_post_response(body: dict[str, Any]) -> Iterator[None]: + """Patch ``mcpg.nl2sql.httpx`` so any POST returns ``body`` as JSON. + + Also resets the module-level shared httpx client + (``mcpg.nl2sql._get_shared_http_client``) around the patch. The + providers now share one lazily-constructed client at module scope + instead of opening one per call — without the reset, a client + cached by an earlier test would still be the real (or differently- + mocked) instance from that test, and this test's mock would never + actually be hit. + """ class _AsyncResponse: def __init__(self, body: dict[str, Any]) -> None: @@ -1274,7 +1328,7 @@ class _AsyncClient: def __init__(self, *args: Any, **kwargs: Any) -> None: pass - async def __aenter__(self) -> _AsyncClient: + async def __aenter__(self) -> Self: return self async def __aexit__(self, *exc_info: object) -> None: @@ -1283,10 +1337,15 @@ async def __aexit__(self, *exc_info: object) -> None: async def post(self, url: str, **_kwargs: Any) -> _AsyncResponse: return _AsyncResponse(body) - return patch.multiple( - "mcpg.nl2sql", - httpx=type("S", (), {"AsyncClient": _AsyncClient, "HTTPError": httpx.HTTPError}), - ) + _reset_shared_http_client() + try: + with patch.multiple( + "mcpg.nl2sql", + httpx=type("S", (), {"AsyncClient": _AsyncClient, "HTTPError": httpx.HTTPError}), + ): + yield + finally: + _reset_shared_http_client() async def test_anthropic_provider_parses_usage_from_the_real_response_shape() -> None: @@ -1341,3 +1400,215 @@ async def test_provider_usage_missing_or_malformed_defaults_to_zero_not_a_crash( ) assert result.tokens_in == 0 assert result.tokens_out == 0 + + +# --- shared httpx.AsyncClient reuse (perf audit remediation, Task 9) ------ +# +# NOTE on test shape: `build_provider` is called fresh on EVERY +# `translate_nl_to_sql` tool invocation in production (see +# `mcpg.tools._register_nl2sql`) — a new provider instance every call. +# A test that instead calls `.complete()` twice on one hand-held +# provider instance would go green the moment `__init__` held a client, +# even though production still built a fresh provider (and thus a +# fresh client) on every call — the defect this task fixes would +# survive untouched. This test instead calls `build_provider()` twice, +# matching the real per-call construction pattern, and asserts the two +# resulting (distinct) provider instances still share one underlying +# `httpx.AsyncClient`. + + +async def test_providers_reuse_one_shared_httpx_client_across_build_provider_calls( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Two separate `build_provider()` calls share one `httpx.AsyncClient`. + + Providers hold no client of their own — `complete()` reaches into + `mcpg.nl2sql`'s module-level, lazily-constructed shared client + (`_get_shared_http_client`) instead of opening `async with + httpx.AsyncClient(...)` per call. + """ + _reset_shared_http_client() + constructed: list[object] = [] + + class _FakeResponse: + def raise_for_status(self) -> None: + return None + + def json(self) -> dict[str, Any]: + return {"content": [{"type": "text", "text": "{}"}], "usage": {}} + + class _FakeAsyncClient: + def __init__(self, *args: Any, **kwargs: Any) -> None: + constructed.append(self) + + async def post(self, *args: Any, **kwargs: Any) -> _FakeResponse: + return _FakeResponse() + + # Patch the real `httpx.AsyncClient` attribute (not + # `mcpg.nl2sql.httpx` wholesale) so `_get_shared_http_client`'s + # `httpx.AsyncClient()` call resolves to the fake at call time, + # tracking every construction regardless of which provider/call + # triggers it. + monkeypatch.setattr(httpx, "AsyncClient", _FakeAsyncClient) + try: + provider_one = build_provider("anthropic", "key-one") + result_one = await provider_one.complete( + system_prompt="sys", user_prompt="user", model="m", max_tokens=10, timeout=5 + ) + provider_two = build_provider("anthropic", "key-two") + result_two = await provider_two.complete( + system_prompt="sys", user_prompt="user", model="m", max_tokens=10, timeout=5 + ) + finally: + _reset_shared_http_client() + + assert provider_one is not provider_two # two distinct instances, matching production + assert result_one.text == "{}" + assert result_two.text == "{}" + assert len(constructed) == 1, ( + f"expected exactly one httpx.AsyncClient construction across two build_provider() calls; got {len(constructed)}" + ) + + +# --- circuit breaker on provider HTTP calls (audit remediation, Task 15) -- + + +async def test_anthropic_complete_opens_circuit_after_repeated_failures( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """After enough consecutive HTTP failures, the breaker opens and further + calls fail fast without re-invoking the underlying HTTP request. + + Loops only just past the failure threshold (6, not 10) to keep this + fast — the invariant only needs one call past the point the breaker + opens. Uses a *fresh provider instance* each iteration (mirroring + production's ``build_provider``-per-call pattern) to prove the breaker's + state lives at the class level, not on any one instance. + """ + _reset_shared_http_client() + call_count = 0 + + class _FailingAsyncClient: + def __init__(self, *args: Any, **kwargs: Any) -> None: + pass + + async def post(self, *args: Any, **kwargs: Any) -> Any: + nonlocal call_count + call_count += 1 + raise httpx.ConnectError("simulated outage", request=None) + + monkeypatch.setattr(httpx, "AsyncClient", _FailingAsyncClient) + try: + for _ in range(6): + with pytest.raises(Exception): # noqa: B017 - either httpx.HTTPError or CircuitBreakerError + await AnthropicProvider(api_key="k").complete( + system_prompt="sys", user_prompt="user", model="m", max_tokens=10, timeout=5 + ) + + calls_before_open = call_count + with pytest.raises(Exception): # noqa: B017 + await AnthropicProvider(api_key="k").complete( + system_prompt="sys", user_prompt="user", model="m", max_tokens=10, timeout=5 + ) + assert call_count == calls_before_open, "breaker should short-circuit without re-invoking the HTTP request" + finally: + _reset_shared_http_client() + + +# --- retry with backoff on provider HTTP calls (audit remediation, Task 16) + + +async def test_anthropic_complete_retries_transient_failures_before_giving_up( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A provider call that fails twice then succeeds is retried + transparently — not immediately surfaced as an error.""" + _reset_shared_http_client() + attempts = 0 + + class _FakeResponse: + def raise_for_status(self) -> None: + return None + + def json(self) -> dict[str, Any]: + return {"content": [{"type": "text", "text": "ok"}], "usage": {}} + + class _FlakyAsyncClient: + def __init__(self, *args: Any, **kwargs: Any) -> None: + pass + + async def post(self, *args: Any, **kwargs: Any) -> Any: + nonlocal attempts + attempts += 1 + if attempts < 3: + raise httpx.ConnectError("transient", request=None) + return _FakeResponse() + + monkeypatch.setattr(httpx, "AsyncClient", _FlakyAsyncClient) + try: + result = await AnthropicProvider(api_key="k").complete( + system_prompt="sys", user_prompt="user", model="m", max_tokens=10, timeout=5 + ) + finally: + _reset_shared_http_client() + + assert attempts == 3 + assert result.text == "ok" + + +# --- CircuitBreakerError translation at the translate_nl_to_sql level ----- +# +# The two tests above call AnthropicProvider(...).complete(...) directly, +# bypassing translate_nl_to_sql entirely — they never exercise its +# `except CircuitBreakerError` branch (nl2sql.py's sibling to the existing +# `except httpx.HTTPError`). This test drives a tripped breaker through +# translate_nl_to_sql itself, matching what the CHANGELOG entry advertises: +# a tripped breaker surfaces as NL2SQLError, never a bare CircuitBreakerError. + + +async def test_translate_nl_to_sql_surfaces_open_circuit_as_nl2sqlerror( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Once the provider's breaker is open, translate_nl_to_sql must still + raise its own NL2SQLError — not let a bare CircuitBreakerError escape + to the caller.""" + _reset_shared_http_client() + + class _FailingAsyncClient: + def __init__(self, *args: Any, **kwargs: Any) -> None: + pass + + async def post(self, *args: Any, **kwargs: Any) -> Any: + raise httpx.ConnectError("simulated outage", request=None) + + monkeypatch.setattr(httpx, "AsyncClient", _FailingAsyncClient) + driver = FakeRoutingDriver(_routes_for_simple_schema()) + try: + # Trip the breaker via translate_nl_to_sql itself (not + # provider.complete() directly) — each failing call here already + # raises NL2SQLError via the existing `except httpx.HTTPError` + # branch, since the breaker is still closed for these attempts. + for _ in range(NL2SQL_CIRCUIT_FAILURE_THRESHOLD): + with pytest.raises(NL2SQLError): + await translate_nl_to_sql( + driver, # type: ignore[arg-type] + provider=AnthropicProvider(api_key="k"), + model="m", + question="how many widgets?", + schema="public", + ) + + # The breaker should now be open. The next call must still surface + # NL2SQLError (via the `except CircuitBreakerError` branch) — a + # bare CircuitBreakerError leaking out here would be a regression. + with pytest.raises(NL2SQLError, match="circuit open") as excinfo: + await translate_nl_to_sql( + driver, # type: ignore[arg-type] + provider=AnthropicProvider(api_key="k"), + model="m", + question="how many widgets?", + schema="public", + ) + assert excinfo.type is NL2SQLError + finally: + _reset_shared_http_client() diff --git a/tests/unit/test_obs_logging.py b/tests/unit/test_obs_logging.py index bb2e706c..20d308a6 100644 --- a/tests/unit/test_obs_logging.py +++ b/tests/unit/test_obs_logging.py @@ -5,8 +5,10 @@ import json import logging +import pytest + from mcpg.config import load_settings -from mcpg.obs_logging import JSONFormatter, setup_logging +from mcpg.obs_logging import JSONFormatter, RedactionFilter, setup_logging def test_json_formatter_formats_standard_record() -> None: @@ -144,3 +146,120 @@ def test_setup_logging_synchronizes_audit_format() -> None: from mcpg.audit import _log_format as current_format assert current_format == "json" + + +def test_redaction_filter_scrubs_a_connection_string_even_when_a_call_site_forgot( + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The centralized filter catches a password-bearing log line even without obfuscate_password.""" + # Other tests in this module (e.g. test_setup_logging_configures_logger) mutate the shared + # "mcpg" logger's propagate flag; force it True here so caplog (attached to the root logger) + # reliably observes records regardless of test order. + monkeypatch.setattr(logging.getLogger("mcpg"), "propagate", True) + logger = logging.getLogger("mcpg.test_redaction") + logger.addFilter(RedactionFilter()) + with caplog.at_level(logging.INFO, logger="mcpg.test_redaction"): + logger.info("connecting to postgresql://user:hunter2@host/db") + assert "hunter2" not in caplog.text + assert "****" in caplog.text + + +def test_redaction_filter_correctly_renders_percent_style_args( + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A %-style lazy-formatted log call must render its args exactly once, not double-substitute. + + RedactionFilter renders record.msg % record.args via getMessage(), then clears record.args + so a later formatter call (which also invokes getMessage()) doesn't re-apply substitution. + """ + monkeypatch.setattr(logging.getLogger("mcpg"), "propagate", True) + logger = logging.getLogger("mcpg.test_redaction_args") + logger.addFilter(RedactionFilter()) + with caplog.at_level(logging.INFO, logger="mcpg.test_redaction_args"): + logger.info("connecting to %s as %s", "postgresql://user:hunter2@host/db", "app_user") + record = caplog.records[0] + + # The password must be redacted in the fully rendered message. + assert "hunter2" not in record.getMessage() + assert "****" in record.getMessage() + # The user value substituted correctly (no leftover %s placeholders, no literal tuple repr). + assert "app_user" in record.getMessage() + assert "%s" not in record.getMessage() + # args must be cleared so a second getMessage() call (as JSONFormatter performs) does not + # attempt to re-substitute against the (now redacted, %-containing-free) rendered string. + assert record.args == () + assert record.getMessage() == record.getMessage() + + +def test_redaction_filter_passes_through_already_obfuscated_message( + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A message that already went through obfuscate_password() at the call site is unchanged.""" + from mcpg.sql import obfuscate_password + + monkeypatch.setattr(logging.getLogger("mcpg"), "propagate", True) + logger = logging.getLogger("mcpg.test_redaction_preobfuscated") + logger.addFilter(RedactionFilter()) + already_safe = obfuscate_password("connecting to postgresql://user:hunter2@host/db") + with caplog.at_level(logging.INFO, logger="mcpg.test_redaction_preobfuscated"): + logger.info(already_safe) + assert "hunter2" not in caplog.text + assert caplog.records[0].getMessage() == already_safe + + +def test_setup_logging_redacts_end_to_end_through_json_formatter( + capsys: pytest.CaptureFixture[str], +) -> None: + """A DSN logged without calling obfuscate_password() is redacted in the actual stderr output + emitted through the real setup_logging() -> handler -> JSONFormatter chain. + + This exercises the full pipeline together (not RedactionFilter or JSONFormatter each tested + in isolation), so a future refactor of either one — e.g. JSONFormatter reading record.msg + directly instead of via getMessage() — can't silently reintroduce a leak that the isolated + unit tests above wouldn't catch. + """ + settings = load_settings( + { + "MCPG_DATABASE_URL": "postgresql://u:p@localhost/db", + "MCPG_LOG_LEVEL": "INFO", + "MCPG_LOG_FORMAT": "json", + } + ) + + logger = logging.getLogger("mcpg") + logger.handlers.clear() + logger.propagate = True + + setup_logging(settings) + capsys.readouterr() # discard anything emitted by setup_logging itself + + logging.getLogger("mcpg.test_e2e_redaction").info("connecting to %s", "postgresql://user:hunter2@host/db") + + emitted = capsys.readouterr().err.strip() + assert emitted, "expected a JSON log line on stderr" + data = json.loads(emitted.splitlines()[-1]) # real JSONFormatter output; must parse cleanly + assert "hunter2" not in data["message"] + assert "****" in data["message"] + + +def test_setup_logging_attaches_redaction_filter_to_its_handler() -> None: + settings = load_settings( + { + "MCPG_DATABASE_URL": "postgresql://u:p@localhost/db", + "MCPG_LOG_LEVEL": "DEBUG", + "MCPG_LOG_FORMAT": "json", + } + ) + + logger = logging.getLogger("mcpg") + logger.handlers.clear() + logger.propagate = True + + setup_logging(settings) + + assert len(logger.handlers) == 1 + handler = logger.handlers[0] + assert any(isinstance(f, RedactionFilter) for f in handler.filters) diff --git a/tests/unit/test_oidc.py b/tests/unit/test_oidc.py index fd135d99..cc48f390 100644 --- a/tests/unit/test_oidc.py +++ b/tests/unit/test_oidc.py @@ -4,7 +4,7 @@ import json import time -from typing import Any +from typing import Any, Self from unittest.mock import patch import httpx @@ -19,6 +19,30 @@ VerifiedToken, ) +# --- fixtures -------------------------------------------------------------- + + +@pytest.fixture(autouse=True) +def _reset_circuit_breakers(): + """Reset every registered circuit breaker before/after each test. + + ``@circuit`` decorates ``OIDCVerifier._resolve_jwks_url`` once at class + definition time, so the breaker's failure count is a single object + shared across *every* ``OIDCVerifier`` instance for the life of the test + process — not per-instance state. Without this reset, a test that trips + the breaker open would leave it open for every unrelated test that runs + afterward in the same session (including tests in test_nl2sql.py, which + registers its own separately-named breakers alongside this module's). + """ + from circuitbreaker import CircuitBreakerMonitor + + for cb in CircuitBreakerMonitor.get_circuits(): + cb.reset() + yield + for cb in CircuitBreakerMonitor.get_circuits(): + cb.reset() + + # --- helpers ------------------------------------------------------------- @@ -56,7 +80,10 @@ def _make_jwt( return jwt.encode(payload, private_key, algorithm="RS256", headers={"kid": kid}) -def _mock_httpx_responses(*, discovery: dict[str, Any], jwks: dict[str, Any]): +# C901 rationale: test-only mock-response builder patching both httpx (our +# code) and urllib.request (PyJWKClient's internal transport) with several +# small nested fake classes -- test infrastructure, not production logic. +def _mock_httpx_responses(*, discovery: dict[str, Any], jwks: dict[str, Any]): # noqa: C901 """Patch httpx.AsyncClient.get to return either the discovery or JWKS doc. The PyJWKClient uses ``urllib.request`` rather than httpx for the @@ -78,7 +105,7 @@ class _AsyncClient: def __init__(self, *args: Any, **kwargs: Any) -> None: pass - async def __aenter__(self) -> _AsyncClient: + async def __aenter__(self) -> Self: return self async def __aexit__(self, *exc_info: object) -> None: @@ -98,7 +125,7 @@ def __init__(self, body: dict[str, Any]) -> None: def read(self) -> bytes: return self._body - def __enter__(self) -> _UrllibResponse: + def __enter__(self) -> Self: return self def __exit__(self, *exc_info: object) -> None: @@ -327,7 +354,7 @@ class _BrokenClient: def __init__(self, *args: Any, **kwargs: Any) -> None: pass - async def __aenter__(self) -> _BrokenClient: + async def __aenter__(self) -> Self: return self async def __aexit__(self, *exc_info: object) -> None: @@ -359,7 +386,7 @@ class _AsyncClient: def __init__(self, *args: Any, **kwargs: Any) -> None: pass - async def __aenter__(self) -> _AsyncClient: + async def __aenter__(self) -> Self: return self async def __aexit__(self, *exc_info: object) -> None: @@ -373,7 +400,7 @@ class _UrllibResponse: def read(self) -> bytes: return json.dumps({"keys": [jwk]}).encode() - def __enter__(self) -> _UrllibResponse: + def __enter__(self) -> Self: return self def __exit__(self, *exc_info: object) -> None: @@ -427,3 +454,133 @@ async def _spy_to_thread(fn, *args, **kwargs): # type: ignore[no-untyped-def] assert verified.claims["sub"] == "user-42" # PyJWKClient.get_signing_key_from_jwt is the method we offloaded. assert "get_signing_key_from_jwt" in observed + + +# --- shared httpx.AsyncClient reuse (perf audit remediation, Task 9) ------ + + +async def test_verifier_reuses_one_httpx_client_across_discovery_fetches() -> None: + """The discovery-document fetch reuses one ``httpx.AsyncClient`` held + for the verifier's whole lifetime, rather than opening ``async with + httpx.AsyncClient(...)`` fresh on every fetch. ``jwks_cache_seconds=0`` + forces the cache to be treated as expired immediately, so two direct + ``_resolve_jwks_url()`` calls both actually hit the network path.""" + issuer = "https://issuer.example" + discovery = {"issuer": issuer, "jwks_uri": f"{issuer}/.well-known/jwks.json"} + constructed: list[object] = [] + + class _AsyncResponse: + def raise_for_status(self) -> None: + return None + + def json(self) -> dict[str, Any]: + return discovery + + class _AsyncClient: + def __init__(self, *args: Any, **kwargs: Any) -> None: + constructed.append(self) + + async def get(self, url: str, **_kwargs: Any) -> _AsyncResponse: + return _AsyncResponse() + + with patch( + "mcpg.oidc.httpx", + type("S", (), {"AsyncClient": _AsyncClient, "HTTPError": httpx.HTTPError}), + ): + verifier = OIDCVerifier(issuer=issuer, audience="mcpg", jwks_cache_seconds=0.0) + url_one = await verifier._resolve_jwks_url() + url_two = await verifier._resolve_jwks_url() + + assert url_one == url_two == discovery["jwks_uri"] + assert len(constructed) == 1, f"expected exactly one httpx.AsyncClient construction; got {len(constructed)}" + + +async def test_verifier_aclose_closes_the_underlying_client() -> None: + """``aclose()`` closes the verifier's held HTTP client.""" + closed: list[bool] = [] + + class _AsyncClient: + def __init__(self, *args: Any, **kwargs: Any) -> None: + pass + + async def aclose(self) -> None: + closed.append(True) + + with patch( + "mcpg.oidc.httpx", + type("S", (), {"AsyncClient": _AsyncClient, "HTTPError": httpx.HTTPError}), + ): + verifier = OIDCVerifier(issuer="https://issuer.example", audience="mcpg") + await verifier.aclose() + + assert closed == [True] + + +# --- circuit breaker on JWKS discovery (audit remediation, Task 15) ------- + + +async def test_ensure_jwks_client_opens_circuit_after_repeated_discovery_failures( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """After enough consecutive discovery-fetch failures, the breaker opens + and further calls fail fast (as OIDCError, not a bare CircuitBreakerError) + without hitting the network again. + + Loops only just past the failure threshold (6, not 10) to keep this fast + — the invariant only needs one call past the point the breaker opens. + """ + verifier = OIDCVerifier(issuer="https://idp.example", audience="mcpg") + call_count = 0 + + async def _always_fails(_url: str, **_kwargs: Any) -> Any: + nonlocal call_count + call_count += 1 + raise httpx.ConnectError("simulated outage", request=None) + + monkeypatch.setattr(verifier._client, "get", _always_fails) + + for _ in range(6): + with pytest.raises(OIDCError): + await verifier._ensure_jwks_client() + + # The breaker should have opened well before the 6th iteration — assert + # relatively (not against an absolute call count), since Task 16 layers + # retry *inside* the breaker and changes how many real network calls + # happen per logical failure. + calls_before_open = call_count + with pytest.raises(OIDCError): + await verifier._ensure_jwks_client() + assert call_count == calls_before_open, "breaker should short-circuit without re-invoking the discovery fetch" + + +# --- retry with backoff on JWKS discovery (audit remediation, Task 16) ---- + + +async def test_resolve_jwks_url_retries_transient_failures_before_giving_up( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A discovery fetch that fails twice then succeeds is retried + transparently — not immediately surfaced as an error.""" + attempts = 0 + + class _FakeResponse: + def raise_for_status(self) -> None: + return None + + def json(self) -> dict[str, Any]: + return {"jwks_uri": "https://idp.example/jwks"} + + async def _flaky_get(_url: str, **_kwargs: Any) -> Any: + nonlocal attempts + attempts += 1 + if attempts < 3: + raise httpx.ConnectError("transient", request=None) + return _FakeResponse() + + verifier = OIDCVerifier(issuer="https://idp.example", audience="mcpg") + monkeypatch.setattr(verifier._client, "get", _flaky_get) + + url = await verifier._resolve_jwks_url() + + assert attempts == 3 + assert url == "https://idp.example/jwks" diff --git a/tests/unit/test_optimizer.py b/tests/unit/test_optimizer.py index f74e7d5b..9fe0345b 100644 --- a/tests/unit/test_optimizer.py +++ b/tests/unit/test_optimizer.py @@ -1,10 +1,13 @@ """Tests for the Query Syntax Optimizer (optimize_query) tool.""" import json +import logging +import pytest from _fakes import FakeDatabase, FakeDriver, FakeRoutingDriver from _mcp_test_helpers import create_connected_server_and_client_session +from mcpg import advisors from mcpg.advisors import optimize_query from mcpg.config import load_settings from mcpg.server import create_server @@ -51,6 +54,69 @@ async def test_optimize_query_detects_all_anti_patterns() -> None: assert "Seq Scan" in res.rationale +class _FlakyPlan: + """A plan double whose ``sequential_scans`` raises on its third access. + + ``optimize_query`` reads ``plan.sequential_scans`` twice while building + the EXPLAIN summary (truthy check, then ``", ".join(...)``) inside a + ``try``/``except QueryError`` block, and once more later while + composing the rationale, inside a separate ``try``/``except + Exception`` block. Raising only on the third access exercises that + second, best-effort block without tripping the first. + """ + + total_cost = 12.0 + estimated_rows = 5 + node_types = ("Seq Scan",) + + def __init__(self) -> None: + self._accesses = 0 + + @property + def sequential_scans(self) -> list[str]: + self._accesses += 1 + if self._accesses <= 2: + return ["large_table"] + raise RuntimeError("plan inspection blew up") + + +async def test_optimize_query_logs_debug_when_sequential_scan_advisory_fails( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """A failure while composing the seq-scan advisory line logs at debug, not silently.""" + + async def _flaky_analyze_query_plan(driver: object, sql: str) -> _FlakyPlan: + return _FlakyPlan() + + monkeypatch.setattr(advisors, "analyze_query_plan", _flaky_analyze_query_plan) + + # setup_logging() (invoked by create_server in other tests in this + # session) disables propagation on the "mcpg" logger to avoid + # double-logging in production; restore it here so caplog (which + # attaches to the root logger) can see records from "mcpg.advisors". + root_logger = logging.getLogger("mcpg") + old_propagate = root_logger.propagate + root_logger.propagate = True + try: + caplog.set_level(logging.DEBUG, logger="mcpg.advisors") + + driver = FakeRoutingDriver({}) + res = await optimize_query(driver, "SELECT * FROM large_table;") # type: ignore[arg-type] + + # The function completes normally — the failure is swallowed, not raised. + assert res.original_sql == "SELECT * FROM large_table;" + matches = [ + record + for record in caplog.records + if "sequential-scan advisory" in record.message and record.levelno == logging.DEBUG + ] + assert len(matches) == 1 + # exc_info=True must actually attach a traceback, not just the message. + assert matches[0].exc_info is not None + finally: + root_logger.propagate = old_propagate + + async def test_optimize_query_tool_registered() -> None: server = create_server(_SETTINGS, database=FakeDatabase(FakeDriver())) # type: ignore[arg-type] async with create_connected_server_and_client_session(server) as client: diff --git a/tests/unit/test_otel_tracing.py b/tests/unit/test_otel_tracing.py index 482af8cd..3b6847ce 100644 --- a/tests/unit/test_otel_tracing.py +++ b/tests/unit/test_otel_tracing.py @@ -265,9 +265,8 @@ def test_tool_span_records_error_attributes_on_exception() -> None: assert handle is not None try: exporter = _capture_spans(handle) - with pytest.raises(RuntimeError, match="boom"): - with tool_span(handle, "run_select", {"sql": "SELECT 1"}): - raise RuntimeError("boom") + with pytest.raises(RuntimeError, match="boom"), tool_span(handle, "run_select", {"sql": "SELECT 1"}): + raise RuntimeError("boom") spans = exporter.get_finished_spans() assert len(spans) == 1 @@ -291,9 +290,8 @@ def test_tool_span_truncates_long_error_messages() -> None: try: exporter = _capture_spans(handle) long_message = "x" * 500 - with pytest.raises(RuntimeError): - with tool_span(handle, "huge_query", {}): - raise RuntimeError(long_message) + with pytest.raises(RuntimeError), tool_span(handle, "huge_query", {}): + raise RuntimeError(long_message) spans = exporter.get_finished_spans() assert spans[0].attributes is not None @@ -314,9 +312,8 @@ def test_tool_span_redacts_dsn_in_error_message() -> None: assert handle is not None try: exporter = _capture_spans(handle) - with pytest.raises(RuntimeError): - with tool_span(handle, "run_select", {}): - raise RuntimeError("connection failed: postgresql://alice:hunter2@db.example.com:5432/app") + with pytest.raises(RuntimeError), tool_span(handle, "run_select", {}): + raise RuntimeError("connection failed: postgresql://alice:hunter2@db.example.com:5432/app") spans = exporter.get_finished_spans() attrs = spans[0].attributes @@ -343,9 +340,8 @@ def test_tool_span_redacts_dsn_before_the_200_char_cap_applies() -> None: # Place the DSN at position ~190 (just inside the 200-char cap) # and make sure it gets redacted. padding = "x" * 180 - with pytest.raises(RuntimeError): - with tool_span(handle, "run_select", {}): - raise RuntimeError(f"{padding} postgresql://u:secret_pw@db/app rest") + with pytest.raises(RuntimeError), tool_span(handle, "run_select", {}): + raise RuntimeError(f"{padding} postgresql://u:secret_pw@db/app rest") spans = exporter.get_finished_spans() attrs = spans[0].attributes diff --git a/tests/unit/test_packaging.py b/tests/unit/test_packaging.py new file mode 100644 index 00000000..0fd5d857 --- /dev/null +++ b/tests/unit/test_packaging.py @@ -0,0 +1,12 @@ +"""Packaging-correctness checks: py.typed presence and wheel include rules.""" + +from __future__ import annotations + +from pathlib import Path + + +def test_py_typed_marker_present() -> None: + """PEP 561: the py.typed marker must exist in the package directory.""" + marker = Path(__file__).resolve().parents[2] / "src" / "mcpg" / "py.typed" + assert marker.is_file() + assert marker.read_text(encoding="utf-8") == "" diff --git a/tests/unit/test_partman.py b/tests/unit/test_partman.py index d09cdd82..3ac7681a 100644 --- a/tests/unit/test_partman.py +++ b/tests/unit/test_partman.py @@ -168,4 +168,4 @@ async def test_partman_tools_registered_in_unrestricted_mode_with_allow_ddl() -> server = create_server(_UNRESTRICTED_DDL, database=FakeDatabase(FakeDriver())) # type: ignore[arg-type] async with create_connected_server_and_client_session(server) as client: listed = {tool.name for tool in (await client.list_tools()).tools} - assert _PARTMAN_TOOLS <= listed + assert listed >= _PARTMAN_TOOLS diff --git a/tests/unit/test_pg19_ddl.py b/tests/unit/test_pg19_ddl.py index 174ade90..7e914b60 100644 --- a/tests/unit/test_pg19_ddl.py +++ b/tests/unit/test_pg19_ddl.py @@ -80,7 +80,7 @@ def __init__(self, *, convalidated: bool | None, alter_fails: bool = False) -> N self.executed: list[str] = [] self.calls: list[tuple[str, object, bool]] = [] - async def execute_query(self, query, params=None, force_readonly=False): # type: ignore[no-untyped-def] + async def execute_query(self, query, params=None, *, force_readonly=False): # type: ignore[no-untyped-def] from mcpg.sql import SqlDriver self.calls.append((query, params, force_readonly)) diff --git a/tests/unit/test_pg19_runtime.py b/tests/unit/test_pg19_runtime.py index bc06094e..c8a00540 100644 --- a/tests/unit/test_pg19_runtime.py +++ b/tests/unit/test_pg19_runtime.py @@ -70,7 +70,7 @@ async def test_checksums_status_never_raises_on_driver_failure() -> None: # --- enable_data_checksums / disable_data_checksums ----------------------- -def _pg19_database_with_checksums(enabled: bool) -> FakeDatabase: +def _pg19_database_with_checksums(*, enabled: bool) -> FakeDatabase: """Wire a FakeDatabase that reports PG 19 + a given checksum state.""" driver = FakeDriver() # Two consecutive read queries: version + setting. The fake returns diff --git a/tests/unit/test_pg19_skip_scan.py b/tests/unit/test_pg19_skip_scan.py index 62a76d2e..2e75cf48 100644 --- a/tests/unit/test_pg19_skip_scan.py +++ b/tests/unit/test_pg19_skip_scan.py @@ -98,7 +98,7 @@ class _CatalogFailingDriver: def __init__(self) -> None: self.calls: list[str] = [] - async def execute_query(self, query, params=None, force_readonly=False): # type: ignore[no-untyped-def] + async def execute_query(self, query, params=None, *, force_readonly=False): # type: ignore[no-untyped-def] from mcpg.sql import SqlDriver self.calls.append(query) diff --git a/tests/unit/test_query.py b/tests/unit/test_query.py index e1a2a885..abc38490 100644 --- a/tests/unit/test_query.py +++ b/tests/unit/test_query.py @@ -20,6 +20,7 @@ run_select_tuned, ) from mcpg.server import create_server +from mcpg.sql import SqlDriver _SETTINGS = load_settings({"MCPG_DATABASE_URL": "postgresql://u:p@localhost/db"}) @@ -66,6 +67,46 @@ async def test_run_select_rejects_non_positive_max_rows() -> None: await run_select(FakeDriver(), "SELECT 1", max_rows=0) +async def test_run_select_does_not_fetch_beyond_max_rows_plus_one() -> None: + """A query matching far more rows than max_rows only pulls max_rows+1 from the driver. + + A test that only checks ``len(result.rows) <= max_rows`` would already + pass today (the pre-fix code slices *after* fetching everything) and + wouldn't catch the bug. This test instead observes the ``row_limit`` + the driver was actually asked for — proving the fetch itself, not just + the returned slice, is bounded. + """ + + class _CountingFakeDriver(SqlDriver): + def __init__(self) -> None: + self.requested_row_limits: list[int | None] = [] + + async def execute_query( + self, + query: str, + params: list[Any] | None = None, + *, + force_readonly: bool = True, + row_limit: int | None = None, + ) -> list[SqlDriver.RowResult]: + self.requested_row_limits.append(row_limit) + # A real driver bounded by row_limit never materializes the + # full table -- simulate that by only ever producing up to + # row_limit rows (or the full 1,000,000-row table if no bound + # was passed, which is what the pre-fix code did). + n = row_limit if row_limit is not None else 1_000_000 + return [SqlDriver.RowResult(cells={"n": i}) for i in range(n)] + + driver = _CountingFakeDriver() + + result = await run_select(driver, "SELECT * FROM huge_table", max_rows=5) + + assert result.truncated is True + assert result.row_count == 5 + # max_rows + 1 = 6, not the full 1,000,000-row table. + assert driver.requested_row_limits == [6] + + @pytest.mark.parametrize( "unsafe_sql", [ @@ -163,6 +204,34 @@ async def test_run_select_tuned_accepts_the_2gb_boundary() -> None: assert result.row_count == 1 +async def test_run_select_tuned_does_not_fetch_beyond_max_rows_plus_one() -> None: + """Same bound as run_select, for run_select_tuned's own execute_query call.""" + + class _CountingFakeDriver(SqlDriver): + def __init__(self) -> None: + self.requested_row_limits: list[int | None] = [] + + async def execute_query( + self, + query: str, + params: list[Any] | None = None, + *, + force_readonly: bool = True, + row_limit: int | None = None, + ) -> list[SqlDriver.RowResult]: + self.requested_row_limits.append(row_limit) + n = row_limit if row_limit is not None else 1_000_000 + return [SqlDriver.RowResult(cells={"n": i}) for i in range(n)] + + driver = _CountingFakeDriver() + + result = await run_select_tuned(driver, "SELECT * FROM huge_table", work_mem="64MB", max_rows=5) + + assert result.truncated is True + assert result.row_count == 5 + assert driver.requested_row_limits == [6] + + async def test_run_select_tuned_validates_maintenance_work_mem_too() -> None: with pytest.raises(QueryError, match="maintenance_work_mem"): await run_select_tuned(FakeDriver(), "SELECT 1", work_mem="64MB", maintenance_work_mem="9GB") @@ -438,7 +507,7 @@ class _StallingDriver: """ async def execute_query( - self, query: str, params: list[Any] | None = None, force_readonly: bool = False + self, query: str, params: list[Any] | None = None, *, force_readonly: bool = False ) -> list[Any]: del query, params, force_readonly import asyncio diff --git a/tests/unit/test_replicas.py b/tests/unit/test_replicas.py index 70711c65..b7466b25 100644 --- a/tests/unit/test_replicas.py +++ b/tests/unit/test_replicas.py @@ -44,7 +44,9 @@ async def execute_query( self, query: str, params: list[Any] | None = None, + *, force_readonly: bool = False, + row_limit: int | None = None, ) -> list[Any]: self.calls.append((query, params, force_readonly)) if self.raises is not None: diff --git a/tests/unit/test_review_fixes.py b/tests/unit/test_review_fixes.py index 4a9b94b9..81e7d81b 100644 --- a/tests/unit/test_review_fixes.py +++ b/tests/unit/test_review_fixes.py @@ -27,7 +27,7 @@ async def test_replay_does_not_rewrite_schema_in_check_constraint_string_literal class _RoutingDriver: async def execute_query( - self, query: str, params: list[object] | None = None, force_readonly: bool = False + self, query: str, params: list[object] | None = None, *, force_readonly: bool = False ) -> list[object]: captured_sqls.append(query) # information_schema.tables → one BASE TABLE. @@ -205,7 +205,12 @@ async def test_execute_in_schema_raises_migration_error_for_concurrently() -> No # --- Fix 2: ListenManager recovers after reader-loop death --- -async def test_listen_manager_reopens_connection_and_relistens_after_reader_death() -> None: +# C901 rationale: test builds a fake LISTEN connection that dies mid-stream +# to exercise ListenManager's reconnect-and-relisten path -- the fake's +# async generator branching (die-on-first-iteration vs. live, timeout vs. +# no-timeout, close-sentinel handling) is test-fixture plumbing, not +# production logic. +async def test_listen_manager_reopens_connection_and_relistens_after_reader_death() -> None: # noqa: C901 """When the reader loop dies (PG restart, network blip), the manager must clear the dead conn AND re-issue LISTEN on every active channel against the fresh connection — not silently stop delivering.""" @@ -329,7 +334,10 @@ async def fake_run(binary: str, *argv: str, **kwargs: object) -> SubprocessResul # --- Fix 9: shell._write_stdin closes stdin in finally --- -async def test_write_stdin_closes_pipe_even_when_drain_raises_a_non_pipe_exception() -> None: +# C901 rationale: test builds a fake stdin/process pair to exercise +# shell._write_stdin's finally-closes-stdin guarantee against a non-pipe +# exception -- fake-object plumbing, not production logic. +async def test_write_stdin_closes_pipe_even_when_drain_raises_a_non_pipe_exception() -> None: # noqa: C901 """A non-BrokenPipeError (e.g. OSError, RuntimeError) on write/drain must still close stdin so the child sees EOF.""" import asyncio diff --git a/tests/unit/test_schema_diff.py b/tests/unit/test_schema_diff.py index 5edf1f09..2af07497 100644 --- a/tests/unit/test_schema_diff.py +++ b/tests/unit/test_schema_diff.py @@ -237,7 +237,7 @@ def test_table_diff_is_empty_recognises_an_unchanged_table() -> None: ) populated = TableDiff( table="x", - columns_added=[ColumnInfo("c", "int", False, None, None)], + columns_added=[ColumnInfo("c", "int", nullable=False, default=None, vector_dimension=None)], columns_removed=[], columns_changed=[], indexes_added=[], @@ -287,8 +287,8 @@ def test_column_change_dataclass_shape() -> None: # this catches accidental field renames the wiring depends on. change = ColumnChange( name="id", - before=ColumnInfo("id", "integer", False, None, None), - after=ColumnInfo("id", "bigint", False, None, None), + before=ColumnInfo("id", "integer", nullable=False, default=None, vector_dimension=None), + after=ColumnInfo("id", "bigint", nullable=False, default=None, vector_dimension=None), fields_changed=["data_type"], ) assert change.fields_changed == ["data_type"] diff --git a/tests/unit/test_server.py b/tests/unit/test_server.py index 9d93f88c..27a780d6 100644 --- a/tests/unit/test_server.py +++ b/tests/unit/test_server.py @@ -1,6 +1,7 @@ """Tests for the MCP server bootstrap.""" from typing import Any +from unittest.mock import patch import pytest from _fakes import FakePool @@ -78,6 +79,33 @@ async def test_lifespan_connects_database_and_yields_app_context() -> None: assert db.is_connected is False +async def test_lifespan_closes_the_shared_nl2sql_http_client() -> None: + """NL→SQL providers share one process-wide httpx.AsyncClient + (mcpg.nl2sql._get_shared_http_client) rather than each opening one + per call — make_lifespan's shutdown finally must close it out + symmetrically so it doesn't leak past the server's lifetime.""" + import mcpg.nl2sql as nl2sql + + pool = FakePool() + db = Database(_SETTINGS, pool=pool) # type: ignore[arg-type] + lm = ListenManager(database_url=_SETTINGS.database_url) + cm = CursorManager(database_url=_SETTINGS.database_url) + lifespan = make_lifespan(_SETTINGS, db, lm, cm) + + closed: list[bool] = [] + real_aclose = nl2sql.aclose_shared_client + + async def _tracking_aclose() -> None: + closed.append(True) + await real_aclose() + + with patch.object(nl2sql, "aclose_shared_client", _tracking_aclose): + async with lifespan(create_server(_SETTINGS)): + pass + + assert closed == [True] + + def test_run_dispatches_stdio_transport(monkeypatch: pytest.MonkeyPatch) -> None: seen: list[str] = [] monkeypatch.setattr(MCPServer, "run", lambda self, transport: seen.append(transport)) diff --git a/tests/unit/test_session_advisor.py b/tests/unit/test_session_advisor.py index 0fa86405..9c7df565 100644 --- a/tests/unit/test_session_advisor.py +++ b/tests/unit/test_session_advisor.py @@ -16,7 +16,7 @@ ) -def _audit_present(present: bool) -> dict[str, list[dict[str, object]]]: +def _audit_present(*, present: bool) -> dict[str, list[dict[str, object]]]: return {"to_regclass('mcpg_audit.events')": [{"present": present}]} @@ -53,7 +53,7 @@ async def test_rejects_zero_threshold() -> None: async def test_returns_diagnostic_when_audit_table_missing() -> None: - driver = FakeRoutingDriver(_audit_present(False)) + driver = FakeRoutingDriver(_audit_present(present=False)) result = await analyze_session_cost(driver) # type: ignore[arg-type] assert isinstance(result, SessionCostAnalysis) assert result.audit_table_present is False @@ -64,7 +64,7 @@ async def test_returns_diagnostic_when_audit_table_missing() -> None: async def test_returns_idle_finding_when_no_events_in_window() -> None: routes: dict[str, list[dict[str, object]]] = {} - routes.update(_audit_present(True)) + routes.update(_audit_present(present=True)) routes.update(_events_route([])) driver = FakeRoutingDriver(routes) result = await analyze_session_cost(driver, lookback_minutes=15) # type: ignore[arg-type] @@ -82,7 +82,7 @@ async def test_returns_idle_finding_when_no_events_in_window() -> None: async def test_redundant_listing_classified_for_catalogue_tool() -> None: routes: dict[str, list[dict[str, object]]] = {} - routes.update(_audit_present(True)) + routes.update(_audit_present(present=True)) routes.update(_events_route([{"tool": "list_tables", "call_count": 47}])) driver = FakeRoutingDriver(routes) result = await analyze_session_cost(driver, hot_threshold=10) # type: ignore[arg-type] @@ -96,7 +96,7 @@ async def test_redundant_listing_classified_for_catalogue_tool() -> None: async def test_hot_repeated_call_classified_for_non_catalogue_tool() -> None: routes: dict[str, list[dict[str, object]]] = {} - routes.update(_audit_present(True)) + routes.update(_audit_present(present=True)) routes.update(_events_route([{"tool": "explain_query", "call_count": 25}])) driver = FakeRoutingDriver(routes) result = await analyze_session_cost(driver, hot_threshold=10) # type: ignore[arg-type] @@ -109,7 +109,7 @@ async def test_hot_repeated_call_classified_for_non_catalogue_tool() -> None: async def test_under_threshold_emits_no_finding() -> None: routes: dict[str, list[dict[str, object]]] = {} - routes.update(_audit_present(True)) + routes.update(_audit_present(present=True)) routes.update(_events_route([{"tool": "list_tables", "call_count": 3}, {"tool": "explain_query", "call_count": 2}])) driver = FakeRoutingDriver(routes) result = await analyze_session_cost(driver, hot_threshold=10) # type: ignore[arg-type] @@ -121,7 +121,7 @@ async def test_under_threshold_emits_no_finding() -> None: async def test_threshold_inclusive_lower_bound() -> None: """Equal-to-threshold doesn't flag — only strictly above does.""" routes: dict[str, list[dict[str, object]]] = {} - routes.update(_audit_present(True)) + routes.update(_audit_present(present=True)) routes.update(_events_route([{"tool": "list_tables", "call_count": 10}])) driver = FakeRoutingDriver(routes) result = await analyze_session_cost(driver, hot_threshold=10) # type: ignore[arg-type] @@ -135,7 +135,7 @@ async def test_threshold_inclusive_lower_bound() -> None: async def test_examines_all_events_even_when_only_some_flag() -> None: routes: dict[str, list[dict[str, object]]] = {} - routes.update(_audit_present(True)) + routes.update(_audit_present(present=True)) routes.update( _events_route( [ @@ -161,7 +161,7 @@ async def test_examines_all_events_even_when_only_some_flag() -> None: async def test_lookback_lands_as_bound_parameter() -> None: routes: dict[str, list[dict[str, object]]] = {} - routes.update(_audit_present(True)) + routes.update(_audit_present(present=True)) routes.update(_events_route([])) driver = FakeRoutingDriver(routes) await analyze_session_cost(driver, lookback_minutes=42) # type: ignore[arg-type] diff --git a/tests/unit/test_session_intent.py b/tests/unit/test_session_intent.py index 4f5871c0..60e2b262 100644 --- a/tests/unit/test_session_intent.py +++ b/tests/unit/test_session_intent.py @@ -170,12 +170,12 @@ def test_admin_preset_is_empty_set_sentinel() -> None: def test_always_keep_includes_the_dynamic_meta_tools() -> None: from mcpg.session_intent import ALWAYS_KEEP - assert ALWAYS_KEEP == { + assert { "describe_self", "describe_tool", "list_session_intents", "enable_session_intent", - } + } == ALWAYS_KEEP # --------------------------------------------------------------------------- diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 3bfb3b4a..427477bd 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -1,6 +1,6 @@ """Tests for the subprocess execution policy (ADR-0004).""" -import os +from pathlib import Path from typing import Any import pytest @@ -313,10 +313,12 @@ async def test_run_pg_binary_spawns_in_a_temp_cwd_with_no_preexec_by_default( # A throwaway working directory is always passed; no rlimit preexec # unless limits are configured. cwd = record["kwargs"]["cwd"] - assert os.path.isabs(cwd) + assert Path(cwd).is_absolute() assert record["kwargs"]["preexec_fn"] is None # The temp cwd must be cleaned up by the time run_pg_binary returns. - assert not os.path.isdir(cwd) + # ASYNC240 rationale: test-only assertion, single fast local stat after the + # call under test already completed; not a hot path. + assert not Path(cwd).is_dir() # noqa: ASYNC240 async def test_run_pg_binary_passes_a_preexec_fn_when_limits_are_set( @@ -360,7 +362,8 @@ async def fake_exec(*args: Any, **kwargs: Any) -> _FakeProcess: await run_pg_binary("pg_dump", "--version", timeout_sec=10, max_output_bytes=1024) assert captured["cwd"] # spawn got far enough to record it - assert not os.path.isdir(captured["cwd"]) + # ASYNC240 rationale: test-only assertion, single fast local stat; not a hot path. + assert not Path(captured["cwd"]).is_dir() # noqa: ASYNC240 async def test_run_pg_binary_enforces_the_bin_allowlist(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/unit/test_slow_call.py b/tests/unit/test_slow_call.py index e15b6688..25579fa2 100644 --- a/tests/unit/test_slow_call.py +++ b/tests/unit/test_slow_call.py @@ -44,6 +44,10 @@ async def test_slow_call_warning_emitted_when_exceeding_threshold(caplog) -> Non { "MCPG_DATABASE_URL": "postgresql://u:p@localhost/db", "MCPG_SLOW_CALL_THRESHOLD_MS": "100", # 0.1 seconds + # Rate limiting isn't under test here; it defaults to enabled and + # would consume extra time.monotonic() calls from the fixed-length + # side_effect list below, sized only for the slow-call timing path. + "MCPG_RATE_LIMIT_ENABLED": "false", } ) @@ -78,6 +82,8 @@ async def test_slow_call_warning_not_emitted_when_under_threshold(caplog) -> Non { "MCPG_DATABASE_URL": "postgresql://u:p@localhost/db", "MCPG_SLOW_CALL_THRESHOLD_MS": "100", # 0.1 seconds + # See comment in test_slow_call_warning_emitted_when_exceeding_threshold. + "MCPG_RATE_LIMIT_ENABLED": "false", } ) @@ -106,6 +112,8 @@ async def test_slow_call_warning_not_emitted_when_disabled(caplog) -> None: { "MCPG_DATABASE_URL": "postgresql://u:p@localhost/db", "MCPG_SLOW_CALL_THRESHOLD_MS": "0", # Disabled + # See comment in test_slow_call_warning_emitted_when_exceeding_threshold. + "MCPG_RATE_LIMIT_ENABLED": "false", } ) @@ -134,6 +142,8 @@ async def test_slow_call_warning_emitted_on_error_path(caplog) -> None: { "MCPG_DATABASE_URL": "postgresql://u:p@localhost/db", "MCPG_SLOW_CALL_THRESHOLD_MS": "100", # 0.1 seconds + # See comment in test_slow_call_warning_emitted_when_exceeding_threshold. + "MCPG_RATE_LIMIT_ENABLED": "false", } ) diff --git a/tests/unit/test_sql_kernel_driver.py b/tests/unit/test_sql_kernel_driver.py index 2728a0a7..77bebf98 100644 --- a/tests/unit/test_sql_kernel_driver.py +++ b/tests/unit/test_sql_kernel_driver.py @@ -268,9 +268,8 @@ async def test_execute_query_error_log_redacts_sql_literal_secret(caplog): driver_logger = logging.getLogger("mcpg.sql.driver") driver_logger.addHandler(caplog.handler) try: - with caplog.at_level(logging.ERROR, logger="mcpg.sql.driver"): - with pytest.raises(Exception, match="boom"): - await driver._execute_with_connection(connection, query, None, force_readonly=False) + with caplog.at_level(logging.ERROR, logger="mcpg.sql.driver"), pytest.raises(Exception, match="boom"): + await driver._execute_with_connection(connection, query, None, force_readonly=False) finally: driver_logger.removeHandler(caplog.handler) diff --git a/tests/unit/test_sql_kernel_internals.py b/tests/unit/test_sql_kernel_internals.py index 9369b75a..8ea0b18b 100644 --- a/tests/unit/test_sql_kernel_internals.py +++ b/tests/unit/test_sql_kernel_internals.py @@ -62,9 +62,11 @@ async def test_pool_connect_returns_cached_when_valid() -> None: async def test_pool_connect_wraps_construction_failure() -> None: pool = DbConnPool("postgresql://u:p@localhost/db") - with patch("mcpg.sql.driver.AsyncConnectionPool", side_effect=RuntimeError("boom")): - with pytest.raises(ValueError, match="Connection attempt failed"): - await pool.pool_connect() + with ( + patch("mcpg.sql.driver.AsyncConnectionPool", side_effect=RuntimeError("boom")), + pytest.raises(ValueError, match="Connection attempt failed"), + ): + await pool.pool_connect() assert pool.is_valid is False assert pool.last_error is not None @@ -110,9 +112,11 @@ async def test_execute_query_lazily_connects_from_engine_url() -> None: # then fail cleanly when the pool can't open — exercising the lazy-connect # branch without a live database. driver = SqlDriver(engine_url="postgresql://u:p@localhost/db") - with patch("mcpg.sql.driver.AsyncConnectionPool", side_effect=RuntimeError("no db")): - with pytest.raises(ValueError, match="Connection attempt failed"): - await driver.execute_query("SELECT 1") + with ( + patch("mcpg.sql.driver.AsyncConnectionPool", side_effect=RuntimeError("no db")), + pytest.raises(ValueError, match="Connection attempt failed"), + ): + await driver.execute_query("SELECT 1") assert driver.is_pool is True @@ -127,7 +131,9 @@ async def test_safe_execute_prefixes_marker_and_forces_readonly() -> None: safe = SafeSqlDriver(inner) await safe.execute_query("SELECT 1") - inner.execute_query.assert_awaited_once_with("/* crystaldba */ SELECT 1", params=None, force_readonly=True) + inner.execute_query.assert_awaited_once_with( + "/* crystaldba */ SELECT 1", params=None, force_readonly=True, row_limit=None + ) async def test_safe_execute_with_timeout_success() -> None: diff --git a/tests/unit/test_sql_kernel_safety.py b/tests/unit/test_sql_kernel_safety.py index bb142813..466c6a19 100644 --- a/tests/unit/test_sql_kernel_safety.py +++ b/tests/unit/test_sql_kernel_safety.py @@ -33,7 +33,7 @@ async def test_select_statement(safe_driver, mock_sql_driver): query = "SELECT * FROM users WHERE age > 18" await safe_driver.execute_query(query) mock_sql_driver.execute_query.assert_awaited_once_with( - "/* crystaldba */ " + query, params=None, force_readonly=True + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None ) @@ -59,7 +59,7 @@ async def test_select_with_join(safe_driver, mock_sql_driver): """ await safe_driver.execute_query(query) mock_sql_driver.execute_query.assert_awaited_once_with( - "/* crystaldba */ " + query, params=None, force_readonly=True + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None ) @@ -69,7 +69,7 @@ async def test_show_variable(safe_driver, mock_sql_driver): query = "SHOW search_path" await safe_driver.execute_query(query) mock_sql_driver.execute_query.assert_awaited_once_with( - "/* crystaldba */ " + query, params=None, force_readonly=True + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None ) @@ -90,7 +90,7 @@ async def test_select_with_arithmetic(safe_driver, mock_sql_driver): query = "SELECT id, price * quantity as total FROM orders" await safe_driver.execute_query(query) mock_sql_driver.execute_query.assert_awaited_once_with( - "/* crystaldba */ " + query, params=None, force_readonly=True + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None ) @@ -100,7 +100,7 @@ async def test_select_current_user(safe_driver, mock_sql_driver): query = "SELECT current_user" await safe_driver.execute_query(query) mock_sql_driver.execute_query.assert_awaited_once_with( - "/* crystaldba */ " + query, params=None, force_readonly=True + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None ) @@ -135,7 +135,7 @@ async def test_select_with_subquery(safe_driver, mock_sql_driver): """ await safe_driver.execute_query(query) mock_sql_driver.execute_query.assert_awaited_once_with( - "/* crystaldba */ " + query, params=None, force_readonly=True + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None ) @@ -162,7 +162,7 @@ async def test_select_with_union(safe_driver, mock_sql_driver): """ await safe_driver.execute_query(query) mock_sql_driver.execute_query.assert_awaited_once_with( - "/* crystaldba */ " + query, params=None, force_readonly=True + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None ) @@ -228,7 +228,7 @@ async def test_explain_plan(safe_driver, mock_sql_driver): """ await safe_driver.execute_query(query) mock_sql_driver.execute_query.assert_awaited_once_with( - "/* crystaldba */ " + query, params=None, force_readonly=True + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None ) @@ -366,7 +366,7 @@ async def test_complex_index_metadata_select(safe_driver, mock_sql_driver): HAVING COUNT(array_agg(attname)) > 1""" await safe_driver.execute_query(query) mock_sql_driver.execute_query.assert_awaited_once_with( - "/* crystaldba */ " + query, params=None, force_readonly=True + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None ) @@ -399,7 +399,7 @@ async def test_session_info_functions(safe_driver, mock_sql_driver): query = "SELECT current_user, current_database(), version()" await safe_driver.execute_query(query) mock_sql_driver.execute_query.assert_awaited_once_with( - "/* crystaldba */ " + query, params=None, force_readonly=True + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None ) @@ -409,7 +409,7 @@ async def test_blocking_pids_functions(safe_driver, mock_sql_driver): query = "SELECT pg_blocking_pids(1234)" await safe_driver.execute_query(query) mock_sql_driver.execute_query.assert_awaited_once_with( - "/* crystaldba */ " + query, params=None, force_readonly=True + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None ) @@ -419,7 +419,7 @@ async def test_logfile_functions(safe_driver, mock_sql_driver): query = "SELECT pg_current_logfile()" await safe_driver.execute_query(query) mock_sql_driver.execute_query.assert_awaited_once_with( - "/* crystaldba */ " + query, params=None, force_readonly=True + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None ) @@ -432,7 +432,7 @@ async def test_complex_session_info_queries(safe_driver, mock_sql_driver): """ await safe_driver.execute_query(query) mock_sql_driver.execute_query.assert_awaited_once_with( - "/* crystaldba */ " + query, params=None, force_readonly=True + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None ) @@ -442,7 +442,7 @@ async def test_security_privilege_functions(safe_driver, mock_sql_driver): query = "SELECT has_table_privilege('user', 'table', 'SELECT')" await safe_driver.execute_query(query) mock_sql_driver.execute_query.assert_awaited_once_with( - "/* crystaldba */ " + query, params=None, force_readonly=True + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None ) @@ -478,7 +478,9 @@ async def test_complex_security_privilege_queries(safe_driver, mock_sql_driver): for query in queries: await safe_driver.execute_query(query) - mock_sql_driver.execute_query.assert_awaited_with("/* crystaldba */ " + query, params=None, force_readonly=True) + mock_sql_driver.execute_query.assert_awaited_with( + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None + ) @pytest.mark.asyncio @@ -507,7 +509,9 @@ async def test_security_privilege_functions_with_subqueries(safe_driver, mock_sq for query in queries: await safe_driver.execute_query(query) - mock_sql_driver.execute_query.assert_awaited_with("/* crystaldba */ " + query, params=None, force_readonly=True) + mock_sql_driver.execute_query.assert_awaited_with( + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None + ) @pytest.mark.parametrize("operator", ["LIKE", "ILIKE"]) @@ -524,7 +528,9 @@ async def test_like_patterns(safe_driver, mock_sql_driver, operator): for query in queries: await safe_driver.execute_query(query) - mock_sql_driver.execute_query.assert_awaited_with("/* crystaldba */ " + query, params=None, force_readonly=True) + mock_sql_driver.execute_query.assert_awaited_with( + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None + ) @pytest.mark.asyncio @@ -558,7 +564,9 @@ async def test_datetime_functions(safe_driver, mock_sql_driver): for query in queries: await safe_driver.execute_query(query) - mock_sql_driver.execute_query.assert_awaited_with("/* crystaldba */ " + query, params=None, force_readonly=True) + mock_sql_driver.execute_query.assert_awaited_with( + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None + ) @pytest.mark.asyncio @@ -585,7 +593,9 @@ async def test_type_conversion_functions(safe_driver, mock_sql_driver): for query in queries: await safe_driver.execute_query(query) - mock_sql_driver.execute_query.assert_awaited_with("/* crystaldba */ " + query, params=None, force_readonly=True) + mock_sql_driver.execute_query.assert_awaited_with( + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None + ) @pytest.mark.asyncio @@ -594,7 +604,7 @@ async def test_regexp_functions(safe_driver, mock_sql_driver): query = "SELECT regexp_replace('Hello World', 'World', 'PostgreSQL')" await safe_driver.execute_query(query) mock_sql_driver.execute_query.assert_awaited_once_with( - "/* crystaldba */ " + query, params=None, force_readonly=True + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None ) @@ -609,7 +619,7 @@ async def test_complex_type_conversion_queries(safe_driver, mock_sql_driver): """ await safe_driver.execute_query(query) mock_sql_driver.execute_query.assert_awaited_once_with( - "/* crystaldba */ " + query, params=None, force_readonly=True + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None ) @@ -619,7 +629,7 @@ async def test_network_functions(safe_driver, mock_sql_driver): query = "SELECT inet_client_addr(), inet_client_port()" await safe_driver.execute_query(query) mock_sql_driver.execute_query.assert_awaited_once_with( - "/* crystaldba */ " + query, params=None, force_readonly=True + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None ) @@ -634,7 +644,7 @@ async def test_network_functions_in_complex_queries(safe_driver, mock_sql_driver """ await safe_driver.execute_query(query) mock_sql_driver.execute_query.assert_awaited_once_with( - "/* crystaldba */ " + query, params=None, force_readonly=True + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None ) @@ -644,7 +654,7 @@ async def test_notification_and_server_functions(safe_driver, mock_sql_driver): query = "SELECT pg_listening_channels(), pg_postmaster_start_time()" await safe_driver.execute_query(query) mock_sql_driver.execute_query.assert_awaited_once_with( - "/* crystaldba */ " + query, params=None, force_readonly=True + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None ) @@ -654,7 +664,7 @@ async def test_minmax_expressions(safe_driver, mock_sql_driver): query = "SELECT GREATEST(1, 2, 3), LEAST(1, 2, 3)" await safe_driver.execute_query(query) mock_sql_driver.execute_query.assert_awaited_once_with( - "/* crystaldba */ " + query, params=None, force_readonly=True + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None ) @@ -664,7 +674,7 @@ async def test_row_expressions(safe_driver, mock_sql_driver): query = "SELECT ROW(1, 2, 3) = ROW(1, 2, 3)" await safe_driver.execute_query(query) mock_sql_driver.execute_query.assert_awaited_once_with( - "/* crystaldba */ " + query, params=None, force_readonly=True + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None ) @@ -674,7 +684,7 @@ async def test_extension_check_query(safe_driver, mock_sql_driver): query = "SELECT extname, extversion FROM pg_extension WHERE extname = 'hypopg'" await safe_driver.execute_query(query) mock_sql_driver.execute_query.assert_awaited_once_with( - "/* crystaldba */ " + query, params=None, force_readonly=True + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None ) @@ -684,7 +694,7 @@ async def test_create_extension_query(safe_driver, mock_sql_driver): query = "CREATE EXTENSION IF NOT EXISTS hypopg" await safe_driver.execute_query(query) mock_sql_driver.execute_query.assert_awaited_once_with( - "/* crystaldba */ " + query, params=None, force_readonly=True + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None ) @@ -694,7 +704,7 @@ async def test_hypopg_create_index_query(safe_driver, mock_sql_driver): query = "SELECT * FROM hypopg_create_index('CREATE INDEX idx ON users(id)')" await safe_driver.execute_query(query) mock_sql_driver.execute_query.assert_awaited_once_with( - "/* crystaldba */ " + query, params=None, force_readonly=True + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None ) @@ -704,7 +714,7 @@ async def test_hypopg_reset_query(safe_driver, mock_sql_driver): query = "SELECT * FROM hypopg_reset()" await safe_driver.execute_query(query) mock_sql_driver.execute_query.assert_awaited_once_with( - "/* crystaldba */ " + query, params=None, force_readonly=True + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None ) @@ -714,7 +724,7 @@ async def test_hypopg_list_indexes_query(safe_driver, mock_sql_driver): query = "SELECT * FROM hypopg_list_indexes()" await safe_driver.execute_query(query) mock_sql_driver.execute_query.assert_awaited_once_with( - "/* crystaldba */ " + query, params=None, force_readonly=True + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None ) @@ -724,7 +734,7 @@ async def test_pg_stat_statements_query(safe_driver, mock_sql_driver): query = "SELECT * FROM pg_stat_statements ORDER BY calls DESC LIMIT 10" await safe_driver.execute_query(query) mock_sql_driver.execute_query.assert_awaited_once_with( - "/* crystaldba */ " + query, params=None, force_readonly=True + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None ) @@ -734,7 +744,7 @@ async def test_pg_indexes_query(safe_driver, mock_sql_driver): query = "SELECT * FROM pg_indexes WHERE schemaname = 'public'" await safe_driver.execute_query(query) mock_sql_driver.execute_query.assert_awaited_once_with( - "/* crystaldba */ " + query, params=None, force_readonly=True + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None ) @@ -744,7 +754,7 @@ async def test_pg_stats_query(safe_driver, mock_sql_driver): query = "SELECT * FROM pg_stats WHERE schemaname = 'public'" await safe_driver.execute_query(query) mock_sql_driver.execute_query.assert_awaited_once_with( - "/* crystaldba */ " + query, params=None, force_readonly=True + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None ) @@ -754,7 +764,7 @@ async def test_explain_query(safe_driver, mock_sql_driver): query = "EXPLAIN SELECT * FROM users" await safe_driver.execute_query(query) mock_sql_driver.execute_query.assert_awaited_once_with( - "/* crystaldba */ " + query, params=None, force_readonly=True + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None ) @@ -778,7 +788,7 @@ async def test_sql_driver_parameter_format(safe_driver, mock_sql_driver): await safe_driver.execute_query(formatted_query) mock_sql_driver.execute_query.assert_awaited_with( - "/* crystaldba */ " + formatted_query, params=None, force_readonly=True + "/* crystaldba */ " + formatted_query, params=None, force_readonly=True, row_limit=None ) @@ -791,8 +801,8 @@ async def test_multiple_queries(safe_driver, mock_sql_driver): await safe_driver.execute_query(query2) mock_sql_driver.execute_query.assert_has_awaits( [ - call("/* crystaldba */ " + query1, params=None, force_readonly=True), - call("/* crystaldba */ " + query2, params=None, force_readonly=True), + call("/* crystaldba */ " + query1, params=None, force_readonly=True, row_limit=None), + call("/* crystaldba */ " + query2, params=None, force_readonly=True, row_limit=None), ] ) @@ -809,7 +819,7 @@ async def test_query_with_comments(safe_driver, mock_sql_driver): """ await safe_driver.execute_query(query) mock_sql_driver.execute_query.assert_awaited_once_with( - "/* crystaldba */ " + query, params=None, force_readonly=True + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None ) @@ -826,5 +836,5 @@ async def test_query_with_whitespace(safe_driver, mock_sql_driver): """ await safe_driver.execute_query(query) mock_sql_driver.execute_query.assert_awaited_once_with( - "/* crystaldba */ " + query, params=None, force_readonly=True + "/* crystaldba */ " + query, params=None, force_readonly=True, row_limit=None ) diff --git a/tests/unit/test_test_data.py b/tests/unit/test_test_data.py index 0a1bc8cc..0e3b76b0 100644 --- a/tests/unit/test_test_data.py +++ b/tests/unit/test_test_data.py @@ -16,8 +16,8 @@ def test_quote_literal_handles_strings_and_special_values() -> None: assert _quote_literal(None) == "NULL" - assert _quote_literal(True) == "TRUE" - assert _quote_literal(False) == "FALSE" + assert _quote_literal(value=True) == "TRUE" + assert _quote_literal(value=False) == "FALSE" assert _quote_literal(42) == "42" assert _quote_literal(3.14) == "3.14" assert _quote_literal("hello") == "'hello'" diff --git a/tests/unit/test_tools.py b/tests/unit/test_tools.py index eedf158e..1673221f 100644 --- a/tests/unit/test_tools.py +++ b/tests/unit/test_tools.py @@ -343,7 +343,7 @@ async def test_read_tools_are_exposed_in_every_access_mode(access_mode: AccessMo async with create_connected_server_and_client_session(_server_for(access_mode)) as client: names = {tool.name for tool in (await client.list_tools()).tools} - assert _READ_TOOLS <= names + assert names >= _READ_TOOLS @pytest.mark.parametrize("access_mode", list(AccessMode)) @@ -371,7 +371,7 @@ async def test_write_tools_are_exposed_in_restricted_and_unrestricted_modes(acce ], ) async def test_run_ddl_requires_unrestricted_mode_and_the_allow_ddl_opt_in( - access_mode: str, allow_ddl: bool, expected: bool + access_mode: str, *, allow_ddl: bool, expected: bool ) -> None: env = {"MCPG_DATABASE_URL": "postgresql://u:p@localhost/db", "MCPG_ACCESS_MODE": access_mode} if allow_ddl: diff --git a/tests/unit/test_vector_ops.py b/tests/unit/test_vector_ops.py index 128aea96..df7401a5 100644 --- a/tests/unit/test_vector_ops.py +++ b/tests/unit/test_vector_ops.py @@ -1636,6 +1636,7 @@ async def execute_query( self, sql: str, params: list[object] | None = None, + *, force_readonly: bool = False, ) -> list[object]: from mcpg.sql import SqlDriver @@ -1690,6 +1691,7 @@ async def execute_query( self, sql: str, params: list[object] | None = None, + *, force_readonly: bool = False, ) -> list[object]: from mcpg.sql import SqlDriver @@ -1753,6 +1755,7 @@ async def execute_query( self, sql: str, params: list[object] | None = None, + *, force_readonly: bool = False, ) -> list[object]: from mcpg.sql import SqlDriver diff --git a/tests/unit/test_vector_recall_direction.py b/tests/unit/test_vector_recall_direction.py index be63e300..ef0f3f8b 100644 --- a/tests/unit/test_vector_recall_direction.py +++ b/tests/unit/test_vector_recall_direction.py @@ -32,7 +32,7 @@ class _CaptureDriver: def __init__(self) -> None: self.sqls: list[str] = [] - async def execute_query(self, sql: Any, params: Any = None, force_readonly: bool = False) -> list[Any]: + async def execute_query(self, sql: Any, params: Any = None, *, force_readonly: bool = False) -> list[Any]: self.sqls.append(str(sql)) return [ SqlDriver.RowResult(cells={"id": 1, "vec": "[1,0]"}), diff --git a/tests/unit/test_vector_tuning.py b/tests/unit/test_vector_tuning.py index 119425f7..0709aeb9 100644 --- a/tests/unit/test_vector_tuning.py +++ b/tests/unit/test_vector_tuning.py @@ -88,7 +88,7 @@ def test_recommend_hnsw_boundary_at_exactly_one_million_rows() -> None: # --- tune_vector_index ----------------------------------------------------- -def _vector_column_row(name: str, dimension: int, nullable: bool = True) -> dict[str, object]: +def _vector_column_row(name: str, dimension: int, *, nullable: bool = True) -> dict[str, object]: return { "column_name": name, "data_type": f"vector({dimension})", diff --git a/tests/unit/test_wait_for_lsn.py b/tests/unit/test_wait_for_lsn.py index 2f201068..9c76de49 100644 --- a/tests/unit/test_wait_for_lsn.py +++ b/tests/unit/test_wait_for_lsn.py @@ -104,7 +104,7 @@ def __init__(self, *, ver_num: int = 190001, ver: str = "19beta1", fail_with: st self.executed: list[str] = [] self.calls: list[tuple[str, object, bool]] = [] - async def execute_query(self, query, params=None, force_readonly=False): # type: ignore[no-untyped-def] + async def execute_query(self, query, params=None, *, force_readonly=False): # type: ignore[no-untyped-def] from mcpg.sql import SqlDriver self.calls.append((query, params, force_readonly)) @@ -158,7 +158,7 @@ async def test_wait_for_lsn_timeout_detected_via_sqlstate_57014() -> None: # Override the driver's failure with a SQLSTATE-bearing exception. driver._fail_with = None - async def fail_with_sqlstate(query, params=None, force_readonly=False): # type: ignore[no-untyped-def] + async def fail_with_sqlstate(query, params=None, *, force_readonly=False): # type: ignore[no-untyped-def] from mcpg.sql import SqlDriver driver.calls.append((query, params, force_readonly)) diff --git a/tests/unit/test_warehousepg.py b/tests/unit/test_warehousepg.py index 41a988cd..85ae311d 100644 --- a/tests/unit/test_warehousepg.py +++ b/tests/unit/test_warehousepg.py @@ -124,7 +124,7 @@ async def test_version_probe_driver_failure_surfaces_as_available_false() -> Non class _FailingDriver: async def execute_query( - self, query: str, params: list[Any] | None = None, force_readonly: bool = False + self, query: str, params: list[Any] | None = None, *, force_readonly: bool = False ) -> list[Any]: del query, params, force_readonly raise RuntimeError("connection lost") @@ -145,7 +145,7 @@ def __init__(self) -> None: self.call_index = 0 async def execute_query( - self, query: str, params: list[Any] | None = None, force_readonly: bool = False + self, query: str, params: list[Any] | None = None, *, force_readonly: bool = False ) -> list[Any]: del params, force_readonly self.call_index += 1 @@ -175,7 +175,7 @@ def __init__(self) -> None: self.call_index = 0 async def execute_query( - self, query: str, params: list[Any] | None = None, force_readonly: bool = False + self, query: str, params: list[Any] | None = None, *, force_readonly: bool = False ) -> list[Any]: del params, force_readonly self.call_index += 1 @@ -205,7 +205,7 @@ async def execute_query( ("PostgreSQL only", False), ], ) -async def test_version_marker_recognition_is_case_insensitive(version_substring: str, expected_match: bool) -> None: +async def test_version_marker_recognition_is_case_insensitive(version_substring: str, *, expected_match: bool) -> None: """The version string scan must be case-insensitive — operators have shipped releases with both 'WarehousePG' and 'warehousepg' spellings across the years.""" diff --git a/tests/unit/test_warehousepg_advisors.py b/tests/unit/test_warehousepg_advisors.py index 246b4f4a..840d9d75 100644 --- a/tests/unit/test_warehousepg_advisors.py +++ b/tests/unit/test_warehousepg_advisors.py @@ -152,7 +152,7 @@ def __init__(self) -> None: self.call_index = 0 async def execute_query( - self, query: str, params: list[Any] | None = None, force_readonly: bool = False + self, query: str, params: list[Any] | None = None, *, force_readonly: bool = False ) -> list[Any]: del params, force_readonly from mcpg.sql import SqlDriver diff --git a/tests/unit/test_warehousepg_reads.py b/tests/unit/test_warehousepg_reads.py index 1e42ce1b..ef16ed15 100644 --- a/tests/unit/test_warehousepg_reads.py +++ b/tests/unit/test_warehousepg_reads.py @@ -298,7 +298,9 @@ async def test_describe_ao_table_passes_schema_and_table_as_parameters() -> None driver = FakeRoutingDriver(routes) await describe_ao_table(driver, "warehouse", "fact_events") ao_calls = [c for c in driver.calls if "pg_appendonly" in c[0]] - assert ao_calls + # Verify exactly one pg_appendonly query was issued, not multiple. + # This catches bugs where the query is issued twice by accident. + assert len(ao_calls) == 1 assert ao_calls[0][1] == ["warehouse", "fact_events"] @@ -374,7 +376,7 @@ def __init__(self) -> None: self.call_index = 0 async def execute_query( - self, query: str, params: list[Any] | None = None, force_readonly: bool = False + self, query: str, params: list[Any] | None = None, *, force_readonly: bool = False ) -> list[Any]: del force_readonly self.call_index += 1 diff --git a/tools/analyse_tool_overlap.py b/tools/analyse_tool_overlap.py index fb06dd88..7bf22a14 100644 --- a/tools/analyse_tool_overlap.py +++ b/tools/analyse_tool_overlap.py @@ -211,7 +211,7 @@ def _input_param_set(schema: dict[str, Any]) -> frozenset[str]: def _classify_pair( - name_sim: float, jaccard: float, containment: float, shared_verb_noun: bool, same_required: bool + name_sim: float, jaccard: float, containment: float, *, shared_verb_noun: bool, same_required: bool ) -> str: """One-line label summarising why the pair was flagged.""" bits: list[str] = [] @@ -229,7 +229,7 @@ def _classify_pair( def _pair_score( - name_sim: float, jaccard: float, containment: float, shared_verb_noun: bool, same_required: bool + name_sim: float, jaccard: float, containment: float, *, shared_verb_noun: bool, same_required: bool ) -> float: """Combined score for ranking. Hand-tuned weights. @@ -250,7 +250,12 @@ def _pair_score( ) -def main() -> int: +# C901 rationale: one-off dev-analysis CLI (not part of the shipped +# package) -- pairwise O(n^2) tool-overlap scan with several independent +# similarity signals (name, description Jaccard/containment, shared +# verb+noun) combined into one flag decision; the branching is the +# heuristic itself. +def main() -> int: # noqa: C901 snapshot = json.loads(_SNAPSHOT_PATH.read_text(encoding="utf-8")) tools: list[dict[str, Any]] = snapshot["tools"] @@ -299,7 +304,13 @@ def main() -> int: "containment": containment, "shared_vn": shared_vn, "same_required": same_required, - "score": _pair_score(name_sim, jaccard, containment, shared_vn, same_required), + "score": _pair_score( + name_sim, + jaccard, + containment, + shared_verb_noun=shared_vn, + same_required=same_required, + ), "a_desc": a["desc"], "b_desc": b["desc"], "required": a["required"] if same_required else None, @@ -361,8 +372,8 @@ def main() -> int: pair["name_sim"], pair["jaccard"], pair["containment"], - pair["shared_vn"], - pair["same_required"], + shared_verb_noun=pair["shared_vn"], + same_required=pair["same_required"], ) out.append(f"_{classification}_") out.append("") diff --git a/tools/generate_doc_tables.py b/tools/generate_doc_tables.py index fb295647..b11de711 100755 --- a/tools/generate_doc_tables.py +++ b/tools/generate_doc_tables.py @@ -291,7 +291,7 @@ def module_descriptions() -> dict[str, str]: except SyntaxError: doc = "" desc = _first_sentence(doc) or _MODULE_FALLBACK.get(name, "") - out[name] = _MODULE_FALLBACK.get(name, desc) if not desc else desc + out[name] = desc if desc else _MODULE_FALLBACK.get(name, desc) return out diff --git a/tools/observe_llm_behaviour.py b/tools/observe_llm_behaviour.py index ce1a2bbd..91696aac 100644 --- a/tools/observe_llm_behaviour.py +++ b/tools/observe_llm_behaviour.py @@ -141,7 +141,7 @@ async def _call_anthropic( user_prompt: str, tools: list[dict[str, Any]], max_tokens: int, - timeout: float, + timeout: float, # noqa: ASYNC109 -- forwarded to httpx's per-request timeout, not a manual reimplementation ) -> tuple[dict[str, Any], int]: """Single Messages API call. Returns ``(response_json, latency_ms)``.""" started = time.monotonic() @@ -218,7 +218,7 @@ async def _run( category_filter: str | None, limit: int | None, max_tokens: int, - timeout: float, + timeout: float, # noqa: ASYNC109 -- forwarded to _call_anthropic's httpx client timeout ) -> int: corpus = json.loads(_CORPUS_PATH.read_text(encoding="utf-8")) prompts = corpus["prompts"] diff --git a/tools/static_tool_facts.py b/tools/static_tool_facts.py index 76f797d4..0e155867 100644 --- a/tools/static_tool_facts.py +++ b/tools/static_tool_facts.py @@ -287,7 +287,12 @@ def _hist_bins(values: list[int], edges: list[int]) -> list[tuple[str, int]]: return list(zip(labels, counts, strict=True)) -def main() -> int: +# C901 rationale: one-off dev-analysis CLI (not part of the shipped +# package) computing several independent report sections (catalog shape, +# description quality, token cost) over the tool snapshot -- each section +# is self-contained; the count is report-section volume, not entangled +# logic. +def main() -> int: # noqa: C901 snapshot = json.loads(_SNAPSHOT_PATH.read_text(encoding="utf-8")) tools: list[dict[str, Any]] = snapshot["tools"] n = len(tools) diff --git a/tools/summarise_llm_observations.py b/tools/summarise_llm_observations.py index 37b24ab1..fc1a5f88 100644 --- a/tools/summarise_llm_observations.py +++ b/tools/summarise_llm_observations.py @@ -66,7 +66,11 @@ def _truncate(text: str, limit: int = 280) -> str: return text if len(text) <= limit else text[: limit - 1] + "…" -def main() -> int: +# C901 rationale: one-off dev-report CLI (not part of the shipped package) +# assembling a multi-section Markdown report line-by-line from an +# observation dataset -- straight-line report-building with conditional +# sections, not entangled logic. +def main() -> int: # noqa: C901 rows = _load_observations() if not rows: print("ERROR: observation file is empty.", file=sys.stderr) diff --git a/uv.lock b/uv.lock index ea081a0f..5adf388d 100644 --- a/uv.lock +++ b/uv.lock @@ -361,6 +361,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/db/8f/61959034484a4a7c527811f4721e75d02d653a35afb0b6054474d8185d4c/charset_normalizer-3.4.7-py3-none-any.whl", hash = "sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d", size = 61958, upload-time = "2026-04-02T09:28:37.794Z" }, ] +[[package]] +name = "circuitbreaker" +version = "2.1.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/df/ac/de7a92c4ed39cba31fe5ad9203b76a25ca67c530797f6bb420fff5f65ccb/circuitbreaker-2.1.3.tar.gz", hash = "sha256:1a4baee510f7bea3c91b194dcce7c07805fe96c4423ed5594b75af438531d084", size = 10787, upload-time = "2025-03-31T08:12:08.963Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/34/15f08edd4628f65217de1fc3c1a27c82e46fe357d60c217fc9881e12ebcc/circuitbreaker-2.1.3-py3-none-any.whl", hash = "sha256:87ba6a3ed03fdc7032bc175561c2b04d52ade9d5faf94ca2b035fbdc5e6b1dd1", size = 7737, upload-time = "2025-03-31T08:12:07.802Z" }, +] + [[package]] name = "click" version = "8.4.1" @@ -1334,6 +1343,7 @@ wheels = [ name = "mcpg" source = { editable = "." } dependencies = [ + { name = "circuitbreaker" }, { name = "httpx" }, { name = "mcp", extra = ["cli"] }, { name = "mcp-types" }, @@ -1341,6 +1351,7 @@ dependencies = [ { name = "psycopg", extra = ["binary"] }, { name = "psycopg-pool" }, { name = "pyjwt", extra = ["crypto"] }, + { name = "tenacity" }, { name = "typing-extensions" }, ] @@ -1376,17 +1387,24 @@ dev = [ { name = "opentelemetry-exporter-otlp-proto-http" }, { name = "opentelemetry-sdk" }, { name = "pip-audit" }, + { name = "pip-licenses" }, { name = "pre-commit" }, { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-cov" }, + { name = "pytest-mock" }, + { name = "pytest-randomly" }, + { name = "pytest-rerunfailures" }, + { name = "pytest-socket" }, { name = "ruff" }, + { name = "time-machine" }, { name = "twine" }, ] [package.metadata] requires-dist = [ { name = "boto3", marker = "extra == 'aws'", specifier = ">=1.28" }, + { name = "circuitbreaker", specifier = ">=2.1.3" }, { name = "google-cloud-secret-manager", marker = "extra == 'gcp'", specifier = ">=2.16" }, { name = "httpx", specifier = ">=0.27" }, { name = "hvac", marker = "extra == 'vault'", specifier = ">=1.0" }, @@ -1399,6 +1417,7 @@ requires-dist = [ { name = "psycopg", extras = ["binary"], specifier = ">=3.3.2" }, { name = "psycopg-pool", specifier = ">=3.3.0" }, { name = "pyjwt", extras = ["crypto"], specifier = ">=2.8" }, + { name = "tenacity", specifier = ">=9.1.4" }, { name = "typing-extensions", specifier = ">=4.12" }, ] provides-extras = ["aws", "gcp", "otel", "vault"] @@ -1419,11 +1438,17 @@ dev = [ { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.27" }, { name = "opentelemetry-sdk", specifier = ">=1.27" }, { name = "pip-audit", specifier = ">=2.7" }, + { name = "pip-licenses", specifier = ">=5.5.5" }, { name = "pre-commit", specifier = ">=4.0.0" }, { name = "pytest", specifier = ">=9.0.2" }, { name = "pytest-asyncio", specifier = ">=1.3.0" }, { name = "pytest-cov", specifier = ">=6.0.0" }, + { name = "pytest-mock", specifier = ">=3.15.1" }, + { name = "pytest-randomly", specifier = ">=4.1.0" }, + { name = "pytest-rerunfailures", specifier = ">=16.6" }, + { name = "pytest-socket", specifier = ">=0.8.1" }, { name = "ruff", specifier = ">=0.14.0" }, + { name = "time-machine", specifier = ">=3.5.0" }, { name = "twine", specifier = ">=5.1" }, ] @@ -1799,6 +1824,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/be/f3/4888f895c02afa085630a3a3329d1b18b998874642ad4c530e9a4d7851fe/pip_audit-2.10.0-py3-none-any.whl", hash = "sha256:16e02093872fac97580303f0848fa3ad64f7ecf600736ea7835a2b24de49613f", size = 61518, upload-time = "2025-12-01T23:42:39.193Z" }, ] +[[package]] +name = "pip-licenses" +version = "5.5.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "prettytable" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7d/18/ddd93af610a04f56a51a27095ddfe55238e1ec236f6758730a0d2c0b49f2/pip_licenses-5.5.5.tar.gz", hash = "sha256:60750c006adf7a0910347b726e8ee9fee3bc8d2e7c8307a5c4ec0776c8e2a276", size = 54955, upload-time = "2026-03-28T22:12:56.48Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/9a/6acfdb8d463eac7cdae7534d35d72237eca63f5fbafe797289d8a5fae447/pip_licenses-5.5.5-py3-none-any.whl", hash = "sha256:f4c4c6d9e6a03612cf59f29f19dc8ab54904d82e055b8e191498f2279a224e14", size = 23247, upload-time = "2026-03-28T22:12:54.89Z" }, +] + [[package]] name = "pip-requirements-parser" version = "32.0.1" @@ -1846,6 +1883,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/80/6e/4b28b62ecb6aae56769c34a8ff1d661473ec1e9519e2d5f8b2c150086b26/pre_commit-4.6.0-py2.py3-none-any.whl", hash = "sha256:e2cf246f7299edcabcf15f9b0571fdce06058527f0a06535068a86d38089f29b", size = 226472, upload-time = "2026-04-21T20:31:40.092Z" }, ] +[[package]] +name = "prettytable" +version = "3.18.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wcwidth" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/81/74/ba08d81e668ccfe8658d7520a307e63c19862c08eb4ccb26f356c5239a7a/prettytable-3.18.0.tar.gz", hash = "sha256:439217116152244369caf3d9f1caf2f9fe29b03bd79e88d2928c8e718c95d680", size = 76373, upload-time = "2026-06-22T16:07:50.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/be/2e6798ace5cc036f5d05d36b7b2fd85346f1a708c87060890b070d0ec607/prettytable-3.18.0-py3-none-any.whl", hash = "sha256:b3346e0e6f79180833aebaac088ae926340586cf6d7d991b9eb125b65f72313a", size = 37357, upload-time = "2026-06-22T16:07:48.595Z" }, +] + [[package]] name = "proto-plus" version = "1.28.0" @@ -2159,6 +2208,55 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, ] +[[package]] +name = "pytest-mock" +version = "3.15.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/68/14/eb014d26be205d38ad5ad20d9a80f7d201472e08167f0bb4361e251084a9/pytest_mock-3.15.1.tar.gz", hash = "sha256:1849a238f6f396da19762269de72cb1814ab44416fa73a8686deac10b0d87a0f", size = 34036, upload-time = "2025-09-16T16:37:27.081Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/cc/06253936f4a7fa2e0f48dfe6d851d9c56df896a9ab09ac019d70b760619c/pytest_mock-3.15.1-py3-none-any.whl", hash = "sha256:0a25e2eb88fe5168d535041d09a4529a188176ae608a6d249ee65abc0949630d", size = 10095, upload-time = "2025-09-16T16:37:25.734Z" }, +] + +[[package]] +name = "pytest-randomly" +version = "4.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/27/b3/36192dacc0f470ac2cc516f73e01739c9a48a8224f76beada4f85e1c8a89/pytest_randomly-4.1.0.tar.gz", hash = "sha256:47f1d9746c3bc3efabd53ae1ebfb8bb385cf3d4df4b505b6d58d9c97a3dfe70f", size = 14302, upload-time = "2026-04-20T13:01:51.831Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/db/2df9a1fca597a273f957a559c20c2d95d629928384507b2afa43ba6909d1/pytest_randomly-4.1.0-py3-none-any.whl", hash = "sha256:f55e89e53367b090c0c053697d7f9d77595543d0e0516c93978b50c0f6b252f9", size = 8353, upload-time = "2026-04-20T13:01:50.382Z" }, +] + +[[package]] +name = "pytest-rerunfailures" +version = "16.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ed/63/0114e45d4b2fcd5f6297dac655c067b47de28be4d33e088f200f0f2c4c28/pytest_rerunfailures-16.6.tar.gz", hash = "sha256:29dbfee46f542073c888e0ed4e81c51e15b9096f49a299eb1a759629c601684a", size = 42806, upload-time = "2026-08-17T07:11:00.447Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/5e/1e994889673d7a0da11651f17ef789b6c83bfe349f29f871873dd3802445/pytest_rerunfailures-16.6-py3-none-any.whl", hash = "sha256:6af2d1ebd6e5cb79666ac408942cd6a0672a49fefd8523664770718560795e13", size = 19137, upload-time = "2026-08-17T07:10:59.121Z" }, +] + +[[package]] +name = "pytest-socket" +version = "0.8.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ba/ce/4ef7b049852c95a8727b4a7e6496f762df1ac0b47bc0320d10293f5e95ec/pytest_socket-0.8.1.tar.gz", hash = "sha256:2f57787914ad2e1308d09ce141b95c3e55741fbb4fb7b7556593a6b063e0c9c7", size = 17313, upload-time = "2026-08-19T15:16:25.653Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/ef/ab507f117b3d19b54e3c9c632a99c28c3b284562ec6e02e274581d530d92/pytest_socket-0.8.1-py3-none-any.whl", hash = "sha256:f9846bed1dcd96eed459e5e14795bbaf96715cf4e827891fe70773817ecb8ed4", size = 8751, upload-time = "2026-08-19T15:16:24.426Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -2702,6 +2800,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f5/ac/19f9941c74add59d17694930ec8105d5eddeee4ce56dd8632b765ca16d6c/stevedore-5.8.0-py3-none-any.whl", hash = "sha256:88eede9e66ca80e34085b9174e2327da2c61ac91f24f70e41c3ad76e4bb4872b", size = 54553, upload-time = "2026-05-18T09:15:25.82Z" }, ] +[[package]] +name = "tenacity" +version = "9.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/c6/ee486fd809e357697ee8a44d3d69222b344920433d3b6666ccd9b374630c/tenacity-9.1.4.tar.gz", hash = "sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a", size = 49413, upload-time = "2026-02-07T10:45:33.841Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" }, +] + [[package]] name = "tiktoken" version = "0.13.0" @@ -2749,6 +2856,62 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/16/5b/f2aa703a4fc5d2dff73460a7d46cc2f3f44aa0f3dd8eeb20d2a0ecf68862/tiktoken-0.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:85b78cc3a2c3d48723ca751fa981f1fedccd54194ca0471b957364353a898b07", size = 918110, upload-time = "2026-05-15T04:51:17.237Z" }, ] +[[package]] +name = "time-machine" +version = "3.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/1a/d9c82e780ac6c41102d50b30c3fb3c25a3b8dd46f807147462d1f4429171/time_machine-3.5.0.tar.gz", hash = "sha256:bc193985b43f15394cfded976efaf9068c3078a2135f42f81c874ba684720eae", size = 25692, upload-time = "2026-08-25T00:18:32.594Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ba/fe/7719438b331df5a4b44740da046ad4919f2d19cf6ec8872425823368b404/time_machine-3.5.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:60baf021c396bc07b3403f96bbc06a2ed977dd42e1d942fc29edd6575ff6c601", size = 25708, upload-time = "2026-08-25T00:17:41.725Z" }, + { url = "https://files.pythonhosted.org/packages/68/e6/8acacd366d94a19f02011ce2af4821315c59af8b868220543096c1a70375/time_machine-3.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cdfd042a4b71fd2687eb16070db3d24e652c147d132a66e1756ac932f2f2bfdb", size = 25888, upload-time = "2026-08-25T00:17:42.783Z" }, + { url = "https://files.pythonhosted.org/packages/62/20/1cbce3bc36438730c945d6a3990f9de0ab07af1ad1cf9b6a399d0887e8ed/time_machine-3.5.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2b0ec43f3f8eb0edb92637684cdc4309fa9ea09f8f78e5daec20c7d5c4bc0b2e", size = 57769, upload-time = "2026-08-25T00:17:43.712Z" }, + { url = "https://files.pythonhosted.org/packages/17/7a/a3dd6e0b8543cb559705af629fb73bbef753a8db31f7abe891d9c95ed7db/time_machine-3.5.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1962b3f01d979e3fe7d8ce1883bb841ef7f39be227776755e2f1f38d6d50b1dc", size = 58524, upload-time = "2026-08-25T00:17:44.724Z" }, + { url = "https://files.pythonhosted.org/packages/e0/5d/0e6dc3fc6c055d35e140a862e1e214833edd1ae565ea63e1249e7ed841b0/time_machine-3.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7523bc19dfd77243ff20361685dd8b1fe319389f7bce758cadc72fd51e4f9471", size = 57199, upload-time = "2026-08-25T00:17:45.802Z" }, + { url = "https://files.pythonhosted.org/packages/09/ac/79a13f1b16a40597e52214160ee79fbef2db00f39c8fb16dd3f9842d2ebf/time_machine-3.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8dd66efe1fe4740d1e789a3dc4121b593e0eaa1c3b9b75d6047033f9cc2263bc", size = 56917, upload-time = "2026-08-25T00:17:46.863Z" }, + { url = "https://files.pythonhosted.org/packages/55/fb/47c16256f63f654ba90fe3a757b251820cf335a47c4bc6031dc029a4fc1c/time_machine-3.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:2e65738b7574877b45890f48d9abcf9c7e4f8100eee3dc58681015cb6e29f4f2", size = 27745, upload-time = "2026-08-25T00:17:47.881Z" }, + { url = "https://files.pythonhosted.org/packages/51/52/51125f0a27469ae55d2ad3529ab0213df92c13b21d2901da572967ad8c67/time_machine-3.5.0-cp312-cp312-win_arm64.whl", hash = "sha256:8e2ebbcb081269848617b65f2ea97e25532e44c9bc62190b5545196c34652d22", size = 26992, upload-time = "2026-08-25T00:17:48.877Z" }, + { url = "https://files.pythonhosted.org/packages/8a/dc/61669d9a3c95291de10486d36b24ca8af50dce17e8d624f9887208f484d1/time_machine-3.5.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f01f0e0fa67ea501c4ea3bf5983234612fb5036ac32ddab092b09ead37715c7c", size = 25707, upload-time = "2026-08-25T00:17:49.791Z" }, + { url = "https://files.pythonhosted.org/packages/2c/a4/3f5a4286768dff89e2de8c9ebcb8765962f2ce00196865954f48132435ea/time_machine-3.5.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:479abe218c4bc03e1acd1914f095a2351ca658db17e69e56b194f5fe2337b216", size = 25878, upload-time = "2026-08-25T00:17:50.881Z" }, + { url = "https://files.pythonhosted.org/packages/b3/ee/1cf35afb752519cd38a1aaf74617cde56c8417a6d235d26c9a650c74138f/time_machine-3.5.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a625e2fd1e3b0dcc1479f559866a068a8c48c3ebe7a2c2093daef8e924ff1a7e", size = 57713, upload-time = "2026-08-25T00:17:52.036Z" }, + { url = "https://files.pythonhosted.org/packages/67/57/d9e3a30cad31fbbe17b81971984ae83121cf968e7583e36b8cf92f31e534/time_machine-3.5.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c5624a6c3256182aaf1d15defcff782d6b0a12c8b2554191981bb59fe00954a", size = 58443, upload-time = "2026-08-25T00:17:53.074Z" }, + { url = "https://files.pythonhosted.org/packages/63/37/82035b221f3bd657f8ca1eb3e28a5a7865338d35331ccc06a0569b87c39a/time_machine-3.5.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:27a2a399d500f03fafe185b4313d372098901bb66b839abf9fb1b981ffb85f44", size = 57122, upload-time = "2026-08-25T00:17:54.116Z" }, + { url = "https://files.pythonhosted.org/packages/93/ae/196922dd5af6b0b4ef650c02343f1276e4be5d98b2f83dfada5f79047cab/time_machine-3.5.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e5e0c69a871a6a03daf491421590671652fd299803c218a37af9551c15dc3e68", size = 56842, upload-time = "2026-08-25T00:17:55.219Z" }, + { url = "https://files.pythonhosted.org/packages/ad/d2/e6e055de97a1b4137bce620a44bd696c95d85f7621959e80e4a6a68c8265/time_machine-3.5.0-cp313-cp313-win_amd64.whl", hash = "sha256:e31e50fc71059e9669da03fcf84a7431c1b731203e27625e4852e35a72bfd16e", size = 27753, upload-time = "2026-08-25T00:17:56.275Z" }, + { url = "https://files.pythonhosted.org/packages/66/74/1fd21def4e95506209b1a10d756d3ca038828347730d4eaef2609d3813dc/time_machine-3.5.0-cp313-cp313-win_arm64.whl", hash = "sha256:ca4a49ef0c28bb8a31e6cb9ff4b6a392e515b8642b07f561369d5023e820c583", size = 26987, upload-time = "2026-08-25T00:17:57.225Z" }, + { url = "https://files.pythonhosted.org/packages/97/e4/f100f18682c0a4c2cdff7d90dbad96fc91d6629a2459502c2781e451c6f6/time_machine-3.5.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:408f1cd20c5e5546ad71c25398b4b8ca2cc698e9efa52cbf1a21160ca5607505", size = 25765, upload-time = "2026-08-25T00:17:58.187Z" }, + { url = "https://files.pythonhosted.org/packages/1e/0a/8a0c8cf3414b11f3c195df9c020bf393fab2e26f9a9973f16ae94e73d509/time_machine-3.5.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:92726ff3240d30d67b054f307dbe9257026dd092ce742d2a59a1c74317d194f4", size = 25889, upload-time = "2026-08-25T00:17:59.142Z" }, + { url = "https://files.pythonhosted.org/packages/43/cd/84c4c7f51d0881182e348ea0e30a329bc284bb24d680d29e0123bbd35a16/time_machine-3.5.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5dfab29d9d54bb073d5a7c4ed155d56964f6afb7726ead00003aee9f77d5e561", size = 57752, upload-time = "2026-08-25T00:18:00.111Z" }, + { url = "https://files.pythonhosted.org/packages/09/b6/a72ab775f6656cc746b6f2cdbbf7740772f7805fc3d99acde6be3112cd63/time_machine-3.5.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ac0ef9d1dd64de23ba8b716c2321137c18d4b3d4229b33b494a889a47d82b35", size = 58547, upload-time = "2026-08-25T00:18:01.154Z" }, + { url = "https://files.pythonhosted.org/packages/0f/5c/bf25aab5688a29bfde2682d44d2a6ba6627d7ff9b26acb6893f1f71f4dfa/time_machine-3.5.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6ba9a85d7b2a25e758337e3aebbd489f1578baad6ef3ad7b50e2e6ef47c52d26", size = 57183, upload-time = "2026-08-25T00:18:02.169Z" }, + { url = "https://files.pythonhosted.org/packages/97/3c/e4ae4520b49d684bac5a0404d7ae072e46d71f0a859e7abe43141db4eb65/time_machine-3.5.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d50d894be50d74fbcee13e53e20ced6c35336519efb1d20fa96ec97dd25e8063", size = 56895, upload-time = "2026-08-25T00:18:03.288Z" }, + { url = "https://files.pythonhosted.org/packages/89/2e/50940fb6a5d9f70f408b502fb4a492436924cceb283080ec3e1a9a531961/time_machine-3.5.0-cp314-cp314-win_amd64.whl", hash = "sha256:ea3b1a6189385ea65e2b68a96dc0599e2add900f8149900cc818eaddbe33ae32", size = 27995, upload-time = "2026-08-25T00:18:04.269Z" }, + { url = "https://files.pythonhosted.org/packages/ce/08/0e5fd740dbbd007d96196e6914c0768724ed4f68550c7692e18ce7b9d819/time_machine-3.5.0-cp314-cp314-win_arm64.whl", hash = "sha256:12747ede9dac2e67b3e707b7a55d8b532be26beebf534ae2370344fafedf2747", size = 27178, upload-time = "2026-08-25T00:18:05.246Z" }, + { url = "https://files.pythonhosted.org/packages/72/60/344e0174a1d923c2b8611a03546c432e0c52d785a6fc780a32f18e619353/time_machine-3.5.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d7f4145552b9e9c32548a5f56cb0e57cf760fb189651812d42dcddd9ea38d7c5", size = 26468, upload-time = "2026-08-25T00:18:06.281Z" }, + { url = "https://files.pythonhosted.org/packages/97/31/428d23f87a53a00d15cf0079756e74ca675bc43f318987b288e0edc0baa7/time_machine-3.5.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2f2234fd57263fd60cd47e87afff24b4b41ff64bcf62a88fbdff9df96884e10f", size = 26709, upload-time = "2026-08-25T00:18:07.41Z" }, + { url = "https://files.pythonhosted.org/packages/0d/61/b88cc24cb437fca0971ca738841440148a841e37bf46c51f236de8ba4fb7/time_machine-3.5.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b9c01bdf00862e5f13554b45e75f9103a0e0ec9cc43c80d3e6283b23dfccbe79", size = 69081, upload-time = "2026-08-25T00:18:08.552Z" }, + { url = "https://files.pythonhosted.org/packages/64/7d/a4f6b7121f16cd0668f549c8896c2033361909431ddca5b504866809aa59/time_machine-3.5.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ad1ca1aba7f0b52958646ef9f261123c04526218fcc85a041cc00caa7a0c510c", size = 71319, upload-time = "2026-08-25T00:18:09.799Z" }, + { url = "https://files.pythonhosted.org/packages/70/cc/96b68c16638936390f3afd939faee61d0e9a565813874e912b426c24d91c/time_machine-3.5.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:94c4baf87208eb5c1161dd6dbae6d1770c412c8a5a91c079f433912e050272d7", size = 69637, upload-time = "2026-08-25T00:18:10.836Z" }, + { url = "https://files.pythonhosted.org/packages/97/b3/5da9984647c59d50128b0e7d5f03f73c273645eef9fed9dd42531da4183c/time_machine-3.5.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a610fa8487da78d4769dc77d04395cb7eaa8863a6907d58d0f142adba1c6b70f", size = 67643, upload-time = "2026-08-25T00:18:11.853Z" }, + { url = "https://files.pythonhosted.org/packages/23/67/344f3de8420ca7d8cbb2f67f3954ed1cbec41845c419a6895f5b406f6687/time_machine-3.5.0-cp314-cp314t-win_amd64.whl", hash = "sha256:605f7c050b05c41dc7c59db26c0df031d40cc5e1d16e411efb1bd3849f9b5ca6", size = 29114, upload-time = "2026-08-25T00:18:12.891Z" }, + { url = "https://files.pythonhosted.org/packages/91/94/a0bb05794e5ebd0fc2b66aaf92c503365b19a63e09a9cb40de8cf42af8f1/time_machine-3.5.0-cp314-cp314t-win_arm64.whl", hash = "sha256:f9bde363fb659e478d0ce95d5ef7e5b91b4fd77a542ee1ba6e611bfd44ba0bd8", size = 27530, upload-time = "2026-08-25T00:18:13.873Z" }, + { url = "https://files.pythonhosted.org/packages/67/99/dfe5115f8e26838c8e7f3cbe89ef1a1af58df3fb44a6603dcaaca1cc686c/time_machine-3.5.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:ed8ad24ea16263f2aa5fd65e172489e31dd18c27da9f9c5aacf0f13a7b1f33ae", size = 25767, upload-time = "2026-08-25T00:18:15.429Z" }, + { url = "https://files.pythonhosted.org/packages/7e/79/efc28c06a78bdaffb165fd6e6389dc9afe6b5edfdec4f415e7fd32b252e3/time_machine-3.5.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:618a3690c5d30c0c815f589c3f2625825d62b4e94f67b390c6f434dd489eae6d", size = 25893, upload-time = "2026-08-25T00:18:16.514Z" }, + { url = "https://files.pythonhosted.org/packages/7e/ad/9c35bdde5bc64d0c73c2dee8edd89cfc799d1edf6498afa4acfdf171624c/time_machine-3.5.0-cp315-cp315-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:435549f1010a1344f032d842d51d2e1d62f7d24bf4108590590b5989ca336da6", size = 57938, upload-time = "2026-08-25T00:18:17.54Z" }, + { url = "https://files.pythonhosted.org/packages/93/99/0f6d87fdd9f63a5caf29b0e0b51b29160f148ce08663730e5436453aa309/time_machine-3.5.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0f137ca62841c1b13d333dbda3e1c20d29c598aa1c0efb051a3301af3281902c", size = 58723, upload-time = "2026-08-25T00:18:18.577Z" }, + { url = "https://files.pythonhosted.org/packages/5d/5b/e911dc4fb4313a0eb99f1ddc9f453adb7951a34d988abc5589426a91174e/time_machine-3.5.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:e9488dc2a733c6b433f4a73ed09aa855512680713a8148580a9471aad1df100d", size = 57398, upload-time = "2026-08-25T00:18:19.661Z" }, + { url = "https://files.pythonhosted.org/packages/0c/37/2c349d5f93c0d35133ff392c0922588b1db55a29aa48cedeb168f7384b4c/time_machine-3.5.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e9d89725cd7cc1ef6dc0560397fa0e820aecdb9ceb7741b0c60108ea3cc7fc1a", size = 57091, upload-time = "2026-08-25T00:18:20.813Z" }, + { url = "https://files.pythonhosted.org/packages/b3/83/ad06a9ab6bd91cf592a2847aab11d84d8da92f7a441fadb444e951dbb6f5/time_machine-3.5.0-cp315-cp315-win_amd64.whl", hash = "sha256:0bc3328e8b56528284719cab10658ea2d9bd44acbae2dc7f1440348b4a2f1703", size = 27991, upload-time = "2026-08-25T00:18:22.028Z" }, + { url = "https://files.pythonhosted.org/packages/bd/4d/666f6c166b89787792a13c8622f0d22bd7df89de0d3ec1abe304a46657b2/time_machine-3.5.0-cp315-cp315-win_arm64.whl", hash = "sha256:de746c044184d7a00e1af7257d9ff25273cebb6c6c833b1994e55632a2c19276", size = 27175, upload-time = "2026-08-25T00:18:23.057Z" }, + { url = "https://files.pythonhosted.org/packages/3f/94/c319fdd99f9cf9f6a4ed3026bdaf0cc0617de0eca46aed7dc3b6abe2e087/time_machine-3.5.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:0e1a72dcf0acc644209ddf048502229c404ad4e44a4b41ce85045902dc2becbc", size = 26471, upload-time = "2026-08-25T00:18:24.09Z" }, + { url = "https://files.pythonhosted.org/packages/30/14/12acfc2661560d0c5ec134331571b4835cf23a16f2848d2627191303913f/time_machine-3.5.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:98c28f9a2f7bd63ade6eeb4e0c2f1bee85862747f9ea5bf59d81584689db19d1", size = 26717, upload-time = "2026-08-25T00:18:25.101Z" }, + { url = "https://files.pythonhosted.org/packages/cd/95/23668168666e6b835caa979848b4addb2176b031ffbe1be3e42401ec699d/time_machine-3.5.0-cp315-cp315t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3dce82621d4f8782f2b8893b4152b58dc43254cee1ac69229c36621709b11036", size = 69104, upload-time = "2026-08-25T00:18:26.154Z" }, + { url = "https://files.pythonhosted.org/packages/a7/ef/cec9cf58e5c6b91789b1d6c053fb230391896761fdfed4bd1f0690efe22a/time_machine-3.5.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bba5d2f7f34aa491521c1d1c13f66802aedc58b10826166753f4b824f8a9c8a3", size = 71339, upload-time = "2026-08-25T00:18:27.391Z" }, + { url = "https://files.pythonhosted.org/packages/7c/d3/41391aa07e6f519b44a5df0c6cb4a0bba05ab22151a1dcbc7dc9c18a028b/time_machine-3.5.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:b4810f9be2dd8c9484a57e949c3ad23cbb6edf4c0e75ca6a10f5935b7d7c2cd0", size = 69674, upload-time = "2026-08-25T00:18:28.51Z" }, + { url = "https://files.pythonhosted.org/packages/df/e3/f948ca31e142cebbe1efd81d361bcac7e9db4de441a6705fc0e109856f34/time_machine-3.5.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:2a48f9ff297700a9036d6fc2a597d97ea2a2bbbd735b45a22fee9ebd3b6b7a9f", size = 67650, upload-time = "2026-08-25T00:18:29.54Z" }, + { url = "https://files.pythonhosted.org/packages/a1/83/5ce829078bc46cf521f11d2a4ccefd833788bdeea1991608ae6b935d7ea7/time_machine-3.5.0-cp315-cp315t-win_amd64.whl", hash = "sha256:1c85556427fe4b1b9680492882c681bb3ab99b43543a593e4def7d1bbd9bb53b", size = 29099, upload-time = "2026-08-25T00:18:30.608Z" }, + { url = "https://files.pythonhosted.org/packages/9e/e1/0fda7534a0b3af4f19fc70bcd4b421924427bc0645b7c1255617ae98de69/time_machine-3.5.0-cp315-cp315t-win_arm64.whl", hash = "sha256:a93195067e2ea6d64e17a0ce0b1eed1fa9bd056ace20c6338fe9be532917ca75", size = 27531, upload-time = "2026-08-25T00:18:31.616Z" }, +] + [[package]] name = "tomli" version = "2.4.1" @@ -2923,6 +3086,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ff/dc/ac4f3a987a87e1a18556896f257c4e15c95ed157b7975347ec6b313b75ce/virtualenv-21.4.1-py3-none-any.whl", hash = "sha256:caf4ff72d1b4039057f41d8e8466e859513d67c0400d9c6b62c02c9d1ebc3e12", size = 7594078, upload-time = "2026-05-28T04:12:47.686Z" }, ] +[[package]] +name = "wcwidth" +version = "0.8.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/34/74/c6428f875774288bec1396f5bfcbc2d925700a4dad61727fd5f2b12f249d/wcwidth-0.8.2.tar.gz", hash = "sha256:91fbef97204b96a3d4d421609b80340b760cf33e26da123ff243d76b1fda8dda", size = 1466253, upload-time = "2026-06-29T18:11:11.601Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/42/3e5985a0a7e57de470b320c6d6a1a67c844f6737a587f3d44dd13d1819e7/wcwidth-0.8.2-py3-none-any.whl", hash = "sha256:d63947694a0539a1d51e01eda7caf800c291020e6cdd7e28ad7b14dd33ad4f85", size = 323166, upload-time = "2026-06-29T18:11:09.888Z" }, +] + [[package]] name = "webcolors" version = "25.10.0"