From bd3447fe9ea98f10be48ae33b78af14585e87f2e Mon Sep 17 00:00:00 2001 From: NickLeko Date: Thu, 9 Apr 2026 08:40:25 -0700 Subject: [PATCH 1/2] Finalize prior auth readiness copilot overhaul and freeze polish --- .env.example | 9 + .github/workflows/ci.yml | 11 +- DESIGN_DECISIONS.md | 68 + EXTRACTION_CONTRACT.md | 13 +- FINAL_FREEZE_AUDIT.md | 75 + IMPLEMENTATION_PLAN.md | 113 ++ INTERVIEW_TALKING_POINTS.md | 94 + LIMITATIONS.md | 32 + MARATHON_PASS_PLAN.md | 101 ++ Makefile | 39 +- NEXT_STEPS.md | 26 + PRODUCT_OVERVIEW.md | 40 + PUSH_SUMMARY.md | 73 + README.md | 198 ++- RELEASE_NOTES_LAST_3_SPRINTS.md | 144 ++ SECOND_PASS_PLAN.md | 98 ++ WHY_THIS_EXISTS.md | 13 + WORKLOG.md | 319 ++++ api.py | 78 + app.py | 1558 ++++++----------- cli.py | 187 ++ docs/DEMO_WALKTHROUGH.md | 38 - docs/api.md | 142 ++ docs/architecture.md | 114 ++ docs/artifacts/CPAP-02-borderline.json | 448 +++++ docs/artifacts/MRI-01-complete.json | 508 ++++++ .../MRI-08-edge-below-threshold.json | 553 ++++++ docs/artifacts/MRI-CERV-01-ready.json | 508 ++++++ docs/artifacts/MRI-KNEE-01-ready.json | 507 ++++++ docs/artifacts/demo_cases.json | 518 ++++++ docs/artifacts/drift_report.md | 17 + docs/artifacts/drift_status.json | 29 + docs/artifacts/featured_demo_cases.json | 158 ++ .../rulebook_diff_reviewed_vs_active.json | 25 + .../rulebook_diff_reviewed_vs_active.md | 12 + docs/artifacts/rulebook_status.json | 62 + docs/artifacts/status.json | 10 + docs/artifacts/supported_procedures.json | 300 ++++ docs/demo_walkthrough.md | 127 ++ docs/notes/PRD.md | 2 +- docs/safety_and_scope.md | 84 + docs/testing.md | 98 ++ engine/acceptance.py | 58 + engine/config.py | 103 ++ engine/demo_cases.py | 47 + engine/evaluate.py | 44 +- engine/extract.py | 124 +- engine/letter_draft.py | 38 +- engine/logging_utils.py | 51 + engine/policy_monitor.py | 40 +- engine/provenance.py | 24 + engine/rendering.py | 154 ++ engine/rulebook.py | 258 +++ engine/rules_loader.py | 55 +- engine/schemas.py | 361 +++- engine/score.py | 1 - engine/service.py | 531 ++++++ engine/test_suites.py | 8 +- inputs/synthetic_cases.json | 126 +- pyproject.toml | 16 + requirements.txt | 4 + rulebook/manifest.yaml | 55 + .../2026-04-09-active-v0.5/payer_rules.yaml | 132 ++ .../policy_sources.yaml | 12 + .../2026-04-09-active-v0.5/provenance.yaml | 34 + .../2026-04-09-reviewed-v0.4/payer_rules.yaml | 99 ++ .../policy_sources.yaml | 12 + .../2026-04-09-reviewed-v0.4/provenance.yaml | 27 + rules/payer_rules.yaml | 87 +- rules/policy_sources.yaml | 1 + rules/provenance.yaml | 28 +- scripts/__init__.py | 1 + scripts/generate_artifacts.py | 77 + scripts/generate_golden_outputs.py | 34 + test/conftest.py | 8 + .../evaluations/CPAP-02-borderline.json | 431 +++++ test/golden/evaluations/MRI-01-complete.json | 493 ++++++ .../MRI-08-edge-below-threshold.json | 538 ++++++ .../golden/evaluations/MRI-KNEE-01-ready.json | 492 ++++++ test/golden/governance/drift_status.json | 29 + .../rulebook_diff_reviewed_vs_active.json | 25 + test/golden/governance/rulebook_status.json | 62 + test/test_acceptance_snapshots.py | 34 + test/test_api.py | 94 + test/test_artifact_generation.py | 29 + test/test_cli.py | 91 + test/test_config_contracts.py | 38 + test/test_extract_contracts.py | 50 +- test/test_letter_draft.py | 4 +- test/test_policy_monitor.py | 1 + test/test_rulebook.py | 27 + test/test_service.py | 176 ++ test/test_streamlit_app.py | 24 + 93 files changed, 11678 insertions(+), 1159 deletions(-) create mode 100644 .env.example create mode 100644 DESIGN_DECISIONS.md create mode 100644 FINAL_FREEZE_AUDIT.md create mode 100644 IMPLEMENTATION_PLAN.md create mode 100644 INTERVIEW_TALKING_POINTS.md create mode 100644 LIMITATIONS.md create mode 100644 MARATHON_PASS_PLAN.md create mode 100644 NEXT_STEPS.md create mode 100644 PRODUCT_OVERVIEW.md create mode 100644 PUSH_SUMMARY.md create mode 100644 RELEASE_NOTES_LAST_3_SPRINTS.md create mode 100644 SECOND_PASS_PLAN.md create mode 100644 WHY_THIS_EXISTS.md create mode 100644 WORKLOG.md create mode 100644 api.py create mode 100644 cli.py delete mode 100644 docs/DEMO_WALKTHROUGH.md create mode 100644 docs/api.md create mode 100644 docs/architecture.md create mode 100644 docs/artifacts/CPAP-02-borderline.json create mode 100644 docs/artifacts/MRI-01-complete.json create mode 100644 docs/artifacts/MRI-08-edge-below-threshold.json create mode 100644 docs/artifacts/MRI-CERV-01-ready.json create mode 100644 docs/artifacts/MRI-KNEE-01-ready.json create mode 100644 docs/artifacts/demo_cases.json create mode 100644 docs/artifacts/drift_report.md create mode 100644 docs/artifacts/drift_status.json create mode 100644 docs/artifacts/featured_demo_cases.json create mode 100644 docs/artifacts/rulebook_diff_reviewed_vs_active.json create mode 100644 docs/artifacts/rulebook_diff_reviewed_vs_active.md create mode 100644 docs/artifacts/rulebook_status.json create mode 100644 docs/artifacts/status.json create mode 100644 docs/artifacts/supported_procedures.json create mode 100644 docs/demo_walkthrough.md create mode 100644 docs/safety_and_scope.md create mode 100644 docs/testing.md create mode 100644 engine/acceptance.py create mode 100644 engine/config.py create mode 100644 engine/demo_cases.py create mode 100644 engine/logging_utils.py create mode 100644 engine/rendering.py create mode 100644 engine/rulebook.py delete mode 100644 engine/score.py create mode 100644 engine/service.py create mode 100644 pyproject.toml create mode 100644 rulebook/manifest.yaml create mode 100644 rulebook/releases/2026-04-09-active-v0.5/payer_rules.yaml create mode 100644 rulebook/releases/2026-04-09-active-v0.5/policy_sources.yaml create mode 100644 rulebook/releases/2026-04-09-active-v0.5/provenance.yaml create mode 100644 rulebook/releases/2026-04-09-reviewed-v0.4/payer_rules.yaml create mode 100644 rulebook/releases/2026-04-09-reviewed-v0.4/policy_sources.yaml create mode 100644 rulebook/releases/2026-04-09-reviewed-v0.4/provenance.yaml create mode 100644 scripts/__init__.py create mode 100644 scripts/generate_artifacts.py create mode 100644 scripts/generate_golden_outputs.py create mode 100644 test/conftest.py create mode 100644 test/golden/evaluations/CPAP-02-borderline.json create mode 100644 test/golden/evaluations/MRI-01-complete.json create mode 100644 test/golden/evaluations/MRI-08-edge-below-threshold.json create mode 100644 test/golden/evaluations/MRI-KNEE-01-ready.json create mode 100644 test/golden/governance/drift_status.json create mode 100644 test/golden/governance/rulebook_diff_reviewed_vs_active.json create mode 100644 test/golden/governance/rulebook_status.json create mode 100644 test/test_acceptance_snapshots.py create mode 100644 test/test_api.py create mode 100644 test/test_artifact_generation.py create mode 100644 test/test_cli.py create mode 100644 test/test_rulebook.py create mode 100644 test/test_service.py create mode 100644 test/test_streamlit_app.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..0604c56 --- /dev/null +++ b/.env.example @@ -0,0 +1,9 @@ +PA_COPILOT_LOG_LEVEL=WARNING +PA_COPILOT_RULES_PATH=rules/payer_rules.yaml +PA_COPILOT_PROVENANCE_PATH=rules/provenance.yaml +PA_COPILOT_POLICY_SOURCES_PATH=rules/policy_sources.yaml +PA_COPILOT_SNAPSHOT_ROOT=policy_snapshots +PA_COPILOT_SYNTHETIC_CASES_PATH=inputs/synthetic_cases.json +PA_COPILOT_ARTIFACTS_DIR=docs/artifacts +PA_COPILOT_API_HOST=127.0.0.1 +PA_COPILOT_API_PORT=8000 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7ceb062..7927e8d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,7 +5,7 @@ on: pull_request: jobs: - pytest: + quality: runs-on: ubuntu-latest steps: @@ -22,5 +22,14 @@ jobs: python -m pip install --upgrade pip python -m pip install -r requirements.txt + - name: Run Ruff + run: ruff check . + - name: Run pytest run: pytest -q + + - name: Generate demo artifacts + run: python -m scripts.generate_artifacts + + - name: Generate golden outputs + run: python -m scripts.generate_golden_outputs diff --git a/DESIGN_DECISIONS.md b/DESIGN_DECISIONS.md new file mode 100644 index 0000000..9af9910 --- /dev/null +++ b/DESIGN_DECISIONS.md @@ -0,0 +1,68 @@ +# Design Decisions + +## 1. Deterministic Before LLM + +Reason: + +- easier to defend in interviews +- easier to audit +- easier to regression test + +Tradeoff: + +- narrower extraction coverage +- less tolerance for messy phrasing + +## 2. `CANNOT_DETERMINE` As A First-Class Result + +Reason: + +- missing documentation should not be silently inferred away + +Tradeoff: + +- more refusals +- less superficially impressive throughput + +## 3. Shared Service Layer Instead Of Rewriting The Engine + +Reason: + +- the original deterministic core was already credible +- the real weakness was orchestration inside the UI + +Tradeoff: + +- some engine modules still use dictionary-shaped internals under the service boundary + +## 4. FastAPI And CLI Added, No Database Added + +Reason: + +- interview-friendly product shape +- no need for persistence in the current scope + +Tradeoff: + +- no multi-user state +- no historical run store + +## 5. Governance-Only Drift Monitoring + +Reason: + +- useful enterprise signal without pretending to solve policy lifecycle management + +Tradeoff: + +- humans must still update rules and tests after drift + +## 6. Synthetic Fixtures Reused Everywhere + +Reason: + +- one source of truth across demo, tests, CLI, API, and artifact generation + +Tradeoff: + +- realism is intentionally bounded diff --git a/EXTRACTION_CONTRACT.md b/EXTRACTION_CONTRACT.md index 9a438a9..24c3826 100644 --- a/EXTRACTION_CONTRACT.md +++ b/EXTRACTION_CONTRACT.md @@ -5,7 +5,7 @@ Scope: Deterministic extraction only Current repo status: implemented and deterministic; no LLM is used anywhere in the extraction path Output: `(facts, evidence_map)` -This document describes the extraction behavior implemented in [engine/extract.py](/Users/nicholasleko/projects/PriorAuthorizationCopilot/engine/extract.py). +This document describes the extraction behavior implemented in [engine/extract.py](engine/extract.py). ## 1. Core Rules @@ -55,6 +55,16 @@ Current behavior: - `inconclusive` when imaging is mentioned without a usable result, or when findings are normal, unclear, unknown, or otherwise non-blocking. - Imaging mention without a result is treated as documented `inconclusive`, not `null`. +### `mechanical_symptoms_documented` + +Type: `bool | null` + +Current behavior: +- Returns `True` when supported mechanical symptom phrasing such as `locking`, `catching`, `buckling`, `giving way`, or `instability` is explicitly present. +- Returns `False` when those symptoms are explicitly denied with supported negation phrasing. +- Returns `null` when the note does not explicitly address the supported symptom phrases. +- Positive phrasing takes precedence if the note contains both denial and later affirmative mechanical-symptom language. + ### `osa_diagnosis` Type: `bool | null` @@ -100,6 +110,7 @@ Current behavior: - `Denies weakness. No saddle anesthesia.` -> `neuro_red_flags_documented = True` - `Prior MRI reviewed` -> `prior_imaging_result = "inconclusive"` - `No prior imaging yet` -> `prior_imaging_result = "none"` +- `Denies locking or instability` -> `mechanical_symptoms_documented = false` - `Sleep study completed 2024-05-18` -> `sleep_study_date = True` - `AHI 22 documented` -> `ahi_documented = True` - `AHI not stated` -> `ahi_documented = null` diff --git a/FINAL_FREEZE_AUDIT.md b/FINAL_FREEZE_AUDIT.md new file mode 100644 index 0000000..8c221fc --- /dev/null +++ b/FINAL_FREEZE_AUDIT.md @@ -0,0 +1,75 @@ +# FINAL FREEZE AUDIT + +## Freeze Recommendation + +Yes. + +## Issues Found + +1. Governance outputs exposed machine-specific absolute paths. +2. `cli.py list-demo-cases` used a single `expected=` label for both true overall-status expectations and coarse fixture labels. +3. Streamlit featured-case wording was ambiguous about whether a value represented a real overall status or only a fixture label. +4. The synthetic eval suite explanation in the UI and docs did not clearly distinguish coarse fixture checks from exact golden acceptance snapshots. +5. Several docs still used machine-specific absolute repo links. +6. Small low-value complexity remained: + - empty dead file `engine/score.py` + - duplicated featured-demo filtering in `app.py` + - repeated service calls in artifact generation +7. Drift reporting exposed a governance-summary inconsistency: + - stale monitored sources were surfaced + - but `any_review_required` still remained `false` + - Streamlit gating and governance JSON were therefore slightly out of sync + +## Fixes Applied In This Final Pass + +- Normalized governance output paths to repo-relative values in drift and rulebook surfaces. +- Clarified CLI demo-case output by splitting `expected_status` from `fixture_label`. +- Clarified Streamlit featured-case wording and synthetic-eval explanation. +- Removed `engine/score.py`. +- Reused shared featured-demo helpers in the app and reduced duplicate work in artifact generation. +- Converted remaining machine-specific doc links to relative links. +- Added regression coverage for portable governance paths and clearer CLI output labels. +- Aligned drift summary behavior so stale or missing monitored baselines now set `any_review_required=true`. +- Updated the Streamlit smoke test to follow the real governance-acknowledgement path before running a featured case. + +## Risks That Remain + +- Scope is still intentionally narrow: one payer, four procedures, and one monitored policy source. +- Drift monitoring remains governance-only and currently reflects a stale monitored source by design. +- Extraction remains regex-based and conservative; unsupported language stays unsupported. +- Rulebook promotion remains manual and human-reviewed, not automated. +- The synthetic eval suite is intentionally coarse and should be explained alongside the stricter golden acceptance tests. + +## Exact Verification Commands + +```bash +ruff check . +pytest -q +make acceptance +make verify +python3 cli.py list-demo-cases +python3 cli.py list-procedures +python3 cli.py drift-status --json +python3 cli.py rulebook-status --json +python3 cli.py evaluate --demo-case MRI-KNEE-01-ready +python3 -m scripts.generate_artifacts +python3 -m scripts.generate_golden_outputs +``` + +## Verification Results + +- `ruff check .` -> passed +- `pytest -q` -> `59 passed in 1.31s` +- `make acceptance` -> `2 passed in 0.19s` +- `make verify` -> passed +- `python3 cli.py list-demo-cases` -> passed +- `python3 cli.py list-procedures` -> passed +- `python3 cli.py drift-status --json` -> passed +- `python3 cli.py rulebook-status --json` -> passed +- `python3 cli.py evaluate --demo-case MRI-KNEE-01-ready` -> passed +- `python3 -m scripts.generate_artifacts` -> passed +- `python3 -m scripts.generate_golden_outputs` -> passed + +## Freeze Readiness Rationale + +The repo is now freeze-ready. The remaining limitations are intentional scope limits, not unfinished engineering. The final pass removed the last portability problems, clarified reviewer-facing wording, trimmed low-value complexity, and aligned the governance summary with the actual stale-source state. README, docs, app behavior, CLI behavior, generated artifacts, golden outputs, and tests now tell one coherent story: deterministic administrative readiness review, synthetic-only data, narrow auditable scope, governance-only drift monitoring, and manual rule promotion. diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md new file mode 100644 index 0000000..50c3a6f --- /dev/null +++ b/IMPLEMENTATION_PLAN.md @@ -0,0 +1,113 @@ +# IMPLEMENTATION PLAN + +## Current State Summary + +The repo already has a credible deterministic core: + +- deterministic extraction in `engine/extract.py` +- deterministic requirement evaluation in `engine/evaluate.py` +- write-only administrative letter drafting in `engine/letter_draft.py` +- versioned payer rules in `rules/payer_rules.yaml` +- partial policy drift monitoring in `engine/policy_monitor.py` +- a Streamlit demo surface in `app.py` +- a small but solid pytest suite + +The repo is compact, explainable, and honest about scope. That should be preserved. + +## Biggest Weaknesses + +1. `app.py` currently owns too much orchestration logic. + - Evaluation assembly, audit construction, metrics, and UI presentation are tightly coupled. + - That makes API and CLI additions harder than they need to be. + +2. Typed contracts exist, but they are not broad enough. + - Major internal objects such as extracted facts, evidence spans, audit summaries, and supported-procedure metadata are still mostly loose dictionaries. + +3. There is no interview-friendly programmatic surface. + - The repo has a strong Streamlit demo, but no lightweight API or CLI for clean demos, curl examples, or reusable artifact generation. + +4. Error handling and unsupported-scope behavior are under-surfaced. + - The core logic is narrow by design, but unsupported procedures, malformed requests, and partial inputs should fail more explicitly and consistently. + +5. Docs are useful but fragmented. + - The repo needs a recruiter-proof top-level narrative, architecture doc, API doc, demo walkthrough, safety/scope doc, and interview talking points. + +6. Testing is good for the current core but not broad enough for a v2 artifact. + - There is little protection yet for API, CLI, structured outputs, unsupported-scope handling, or reproducible demo artifacts. + +## Highest-Leverage Improvements + +1. Add a small shared application layer. + - Create a reusable evaluation service that wraps deterministic extraction, rule lookup, scoring, audit trace creation, and output shaping. + - This should become the single source of truth used by Streamlit, API, CLI, and artifact generation. + +2. Expand typed models with Pydantic. + - Add schemas for intake requests, supported procedures, evidence spans, blockers, audit traces, status metadata, drift summaries, and exportable reports. + +3. Add central config and structured logging. + - Keep it lightweight: one config module, one logger setup, explicit paths, and deterministic defaults. + +4. Add a lightweight FastAPI layer. + - Expose health, supported procedures, readiness evaluation, demo case listing, and drift status. + - Keep the API honest: synthetic-only, deterministic, no persistence, no autonomous action. + +5. Add a simple CLI. + - Support listing procedures, listing demo cases, evaluating a demo case, exporting a report, validating synthetic input, and checking drift artifacts. + +6. Upgrade the Streamlit demo without changing the product identity. + - Keep the current strengths: blockers, evidence mapping, auditability, and refusal-first behavior. + - Improve the flow, labels, operator framing, and scope/limitations presentation. + +7. Strengthen reusable synthetic fixtures and artifacts. + - Add clearer demo case helpers, stable sample outputs, and a reproducible artifact-generation script. + +8. Expand tests and CI only where signal is high. + - Add regression tests for the shared service layer, API, CLI, unsupported scope, artifact generation, and key deterministic outputs. + +## What Will Not Be Changed + +- The repo will remain deterministic-first. +- No LLM-first architecture will be introduced. +- The core readiness semantics will stay frozen: + - `READY` + - `NOT_READY` + - `CANNOT_DETERMINE` +- The repo will remain synthetic-only. +- No real payer integrations, no EHR connectivity, no claims adjudication, no medical-necessity review, no approval prediction, and no autonomous action will be added. +- No database will be added. +- The existing extraction and evaluation logic will not be rewritten unless a concrete defect appears during refactoring. +- Policy drift monitoring will remain governance-only and will not auto-edit rules. + +## Smallest Structural Change With Highest ROI + +The smallest high-ROI structural change is: + +1. introduce a shared, typed evaluation service +2. route Streamlit through that service +3. expose the same service through FastAPI and a CLI + +That yields cleaner architecture, stronger testability, better docs, and a much stronger interview story without replacing the deterministic core. + +## Planned Execution Order + +1. Audit and baseline verification +2. Add worklog and implementation plan +3. Add shared models, config, logging, and evaluation orchestration +4. Refactor Streamlit to consume shared outputs instead of building them inline +5. Add FastAPI endpoints with typed request/response schemas +6. Add CLI commands for core demo workflows +7. Add reusable demo fixture helpers and artifact generation +8. Expand tests for service, API, CLI, unsupported scope, and artifacts +9. Overhaul docs and interview-facing materials +10. Tighten CI and local developer commands +11. Final verification pass, simplification pass, and worklog closeout + +## Scope Discipline + +If a candidate improvement starts forcing broad rewrites across the repo, the narrower fallback is: + +- keep the deterministic engine intact +- add adapters around it +- document limitations honestly + +That is the bar for every change in this overnight pass. diff --git a/INTERVIEW_TALKING_POINTS.md b/INTERVIEW_TALKING_POINTS.md new file mode 100644 index 0000000..8755394 --- /dev/null +++ b/INTERVIEW_TALKING_POINTS.md @@ -0,0 +1,94 @@ +# Interview Talking Points + +## 30-Second Summary + +This repo is a deterministic prior authorization readiness copilot for synthetic demo cases. It checks administrative readiness against narrow, versioned payer rules, returns blocker-level reasoning and evidence mapping, exposes the same workflow through Streamlit, FastAPI, and a CLI, and now includes a lightweight rulebook and governance layer. It does not make clinical judgments, predict approval, or act autonomously. + +## 2-Minute Walkthrough + +1. A synthetic request enters through the UI, API, or CLI. +2. The shared service validates scope and normalizes the request. +3. Deterministic extraction pulls a narrow fact set and evidence spans from note text. +4. Deterministic evaluation applies versioned payer requirements and returns `READY`, `NOT_READY`, or `CANNOT_DETERMINE`. +5. The result includes blockers, requirement-level reasoning, extracted facts, audit metadata, and a deterministic administrative letter option. +6. Separate governance surfaces track policy drift and rulebook promotion without mutating runtime logic automatically. +7. The third pass added a non-spine knee MRI pathway, versioned rulebook snapshots, stale-source reporting, and golden acceptance checks. + +## Why This Matters In Healthcare Admin Workflows + +- many prior auth delays are administrative documentation failures, not deep clinical disagreements +- deterministic readiness checks can reduce preventable back-and-forth before submission +- refusal-first behavior is safer than pretending certainty when documentation is incomplete +- auditable outputs matter because reviewers need to know exactly why a request is blocked + +## Why Deterministic Before LLM Here + +- the supported scope is intentionally narrow +- requirement semantics matter more than broad language flexibility +- deterministic outputs are easier to test, explain, diff, and govern +- the repo is meant to show disciplined product framing, not prompt theater + +## Safety And Governance Rationale + +- synthetic-only inputs avoid PHI and production-readiness theater +- `CANNOT_DETERMINE` is an explicit safety feature +- unsupported scope is rejected instead of guessed through +- rulebook promotion is human-driven +- drift monitoring is governance-only and never auto-updates runtime rules + +## What Changed Across The Three Passes + +### v1 + +- pulled orchestration out of the Streamlit app +- added typed service, API, CLI, artifacts, and stronger docs + +### v2 + +- added cervical MRI +- strengthened provenance metadata and registry surfaces +- added Streamlit sanity coverage and richer artifacts + +### v3 + +- added a non-spine knee MRI pathway with one new extractor field +- introduced reviewed vs active rulebook snapshots and release diffs +- surfaced stale drift baselines and review reasons +- added golden acceptance snapshots for representative product outputs + +## Limitations + +- one payer only +- four supported procedures only +- pattern-based extraction only +- one monitored policy source only +- no persistence, auth, or deployment stack +- not production-ready for real healthcare operations + +## Next Real Product Steps + +- add a second monitored source only with a clean offline baseline +- expand procedure coverage only when a new pathway can stay equally deterministic +- tighten the human review workflow around rulebook promotion +- add structured intake adapters before adding any storage layer + +## Tradeoffs Intentionally Made + +- chose explainability over broad coverage +- chose one narrow non-spine pathway over a larger procedure list +- chose a lightweight rulebook over a full workflow platform +- chose acceptance snapshots over more speculative feature work +- chose to skip Docker, auth, and databases because they would add explanation burden faster than credibility + +## Top 10 Talking Points + +1. The repo solves administrative readiness, not approval prediction. +2. `CANNOT_DETERMINE` is a deliberate refusal mode, not a failure. +3. Deterministic logic was chosen because the supported scope is narrow and fully defensible. +4. The same workflow powers Streamlit, FastAPI, CLI, exported artifacts, and golden acceptance snapshots. +5. The third pass added a non-spine knee MRI pathway without turning the repo into a generic imaging engine. +6. Evidence spans and structured provenance make outputs auditable instead of opaque. +7. The rulebook shows reviewed vs active rule snapshots and release diffs without pretending to be a platform. +8. Drift monitoring exists, but it is governance-only and never auto-promotes rule changes. +9. Synthetic-only data keeps the repo safe to share and easy to test. +10. The architecture stays intentionally compact: no database, no auth, no LLM layer, no fake enterprise complexity. diff --git a/LIMITATIONS.md b/LIMITATIONS.md new file mode 100644 index 0000000..31803a3 --- /dev/null +++ b/LIMITATIONS.md @@ -0,0 +1,32 @@ +# Limitations + +## Scope Limits + +- only a small number of procedures are supported +- payer coverage is intentionally narrow +- supported sites of care are constrained + +## Extraction Limits + +- extraction is pattern-based, not language-model-based +- unusual phrasing can remain unparsed +- the system prefers missingness over aggressive inference + +## Governance Limits + +- drift monitoring is partial +- only configured sources are monitored +- rules are still curated offline + +## Product Limits + +- no persistence +- no authentication +- no user management +- no deployment packaging beyond local/demo use + +## Healthcare Limits + +- not validated for real-world clinical or administrative operations +- not suitable for real PHI workflows as currently packaged +- not a substitute for payer policy review or human chart review diff --git a/MARATHON_PASS_PLAN.md b/MARATHON_PASS_PLAN.md new file mode 100644 index 0000000..581fee1 --- /dev/null +++ b/MARATHON_PASS_PLAN.md @@ -0,0 +1,101 @@ +# MARATHON PASS PLAN + +## Current Starting Point + +The repo already has a solid deterministic backbone and shared product surface: + +- deterministic extraction, evaluation, blocker detection, and letter drafting +- FastAPI, CLI, and Streamlit surfaces over a shared service layer +- typed schemas, central config, and structured logging +- synthetic demo cases, reproducible artifacts, and growing regression coverage +- provenance metadata and a governance-only drift surface +- three supported procedures: + - `MRI_LUMBAR` + - `MRI_CERVICAL` + - `CPAP_DEVICE` + +The repo does not need a redesign. The remaining gains are mostly product depth, governance clarity, and behavior lock-in. + +## Chosen Work Items + +### 1. Add one narrow non-spine deterministic pathway if it stays defensible + +Target candidate: +- `MRI_KNEE` + +Acceptance bar: +- no fuzzy medical-necessity logic +- no broad orthopedic abstraction +- one additional extractor field at most +- clear documented blocker and cannot-determine behavior +- synthetic fixtures, tests, artifacts, and docs all updated + +Fallback: +- if the rule contract becomes vague, skip the new procedure and deepen scenario coverage for the existing three procedures instead + +### 2. Introduce a lightweight versioned rulebook and promotion workflow + +Planned shape: +- a small governance artifact for `draft`, `reviewed`, and `active` +- validation and diff utilities +- rulebook version metadata surfaced in outputs where useful +- explicit human-review promotion steps in docs + +### 3. Deepen offline drift reporting instead of widening to weak new sources + +Planned direction: +- richer source metadata and stale-review visibility +- markdown and JSON drift report artifacts +- comparison of monitored snapshots against active rule assumptions when practical + +### 4. Add golden outputs and acceptance checks + +Planned coverage: +- representative ready, not-ready, and cannot-determine cases +- rulebook and drift artifact integrity +- cheap deterministic acceptance command wired into local workflow + +### 5. Tighten local ergonomics and recruiter-facing docs + +Planned direction: +- one-command high-signal verify and acceptance flows +- refreshed docs tied to the new procedure, rulebook, and governance story +- sharper interview narrative for the v1, v2, v3 progression + +## Skipped Or Deprioritized Work Items + +- no LLM layer +- no database +- no auth +- no Docker or deployment stack +- no broad second wave of procedures unless the first new one is clearly rigorous +- no live payer scraping requirement +- no automatic rule promotion or rule mutation from drift monitoring +- no large UI redesign that mainly changes aesthetics without improving clarity + +## Dependency Order + +1. Confirm current repo state and add this plan and worklog section +2. Implement the non-spine procedure only if the rule contract survives review +3. Build the rulebook/promotion workflow around the updated rule set +4. Improve drift artifacts and governance reporting +5. Add golden outputs and acceptance-style tests +6. Tighten ergonomics, docs, and interview materials +7. Run Ruff, full tests, artifact generation, smoke checks, and a simplification pass + +## Stop Criteria + +Stop when one of these becomes true: + +- the backlog above is substantially complete +- the next remaining work is mostly cosmetic or platform theater +- the next remaining work would widen claims faster than it improves rigor +- the repo reaches a point where additional changes add more explanation burden than credibility + +## Decision Rules For The Marathon Pass + +- Prefer one rigorous non-spine procedure over multiple shallow ones. +- Prefer governance depth over procedure count once product scope becomes harder to defend. +- Prefer offline, inspectable artifacts over network-dependent demos. +- Prefer regression lock-in over additional optional features. +- If an idea feels impressive but hard to explain in two minutes, skip it. diff --git a/Makefile b/Makefile index cad8489..6a0cf34 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,9 @@ -PYTHON ?= python3.12 +PYTHON ?= python3 VENV_PYTHON ?= .venv/bin/python +CASE ?= MRI-01-complete +RUN_PYTHON := $(if $(wildcard $(VENV_PYTHON)),$(VENV_PYTHON),$(PYTHON)) -.PHONY: install run test +.PHONY: install run api test lint format artifacts cli-status verify smoke-ui evaluate-case acceptance goldens install: $(PYTHON) -m venv .venv @@ -9,7 +11,36 @@ install: $(VENV_PYTHON) -m pip install -r requirements.txt run: - $(VENV_PYTHON) -m streamlit run app.py + $(RUN_PYTHON) -m streamlit run app.py + +api: + $(RUN_PYTHON) -m uvicorn api:app --reload + +cli-status: + $(RUN_PYTHON) cli.py status + +evaluate-case: + $(RUN_PYTHON) cli.py evaluate --demo-case $(CASE) test: - $(VENV_PYTHON) -m pytest -q + $(RUN_PYTHON) -m pytest -q + +acceptance: + $(RUN_PYTHON) -m pytest -q test/test_acceptance_snapshots.py + +smoke-ui: + $(RUN_PYTHON) -m pytest -q test/test_streamlit_app.py + +lint: + $(RUN_PYTHON) -m ruff check . + +format: + $(RUN_PYTHON) -m ruff format . + +artifacts: + $(RUN_PYTHON) -m scripts.generate_artifacts + +goldens: + $(RUN_PYTHON) -m scripts.generate_golden_outputs + +verify: lint test artifacts goldens diff --git a/NEXT_STEPS.md b/NEXT_STEPS.md new file mode 100644 index 0000000..f1f5fcb --- /dev/null +++ b/NEXT_STEPS.md @@ -0,0 +1,26 @@ +# Next Steps + +## Near-Term Realistic Next Steps + +1. Add a second monitored policy source only if it has a clean offline snapshot baseline and an equally honest governance story. +2. Add one more supported procedure only if it can stay as deterministic and interview-defensible as the current four. +3. Tighten the rulebook workflow with a real draft snapshot only when there is a concrete candidate change worth reviewing. +4. Add a lightweight screenshot capture path only if it stays reproducible and dependency-light. + +## Intentionally Deferred + +1. Structured upstream intake adapters. +2. Human analyst annotation and review queues outside the deterministic core. +3. More explicit governance dashboards or operational telemetry. +4. Additional browser-level smoke coverage beyond Streamlit AppTest. + +## Low-ROI Ideas Rejected On Purpose + +- LLM-first extraction +- approval prediction +- autonomous action +- a database-backed platform rewrite +- auth and deployment machinery +- broad procedure expansion without a defendable rule contract + +Those ideas would widen the product story faster than they would improve credibility. diff --git a/PRODUCT_OVERVIEW.md b/PRODUCT_OVERVIEW.md new file mode 100644 index 0000000..90b9279 --- /dev/null +++ b/PRODUCT_OVERVIEW.md @@ -0,0 +1,40 @@ +# Product Overview + +Prior Authorization Readiness Copilot is a deterministic internal-product demo for administrative prior auth readiness review. + +## What It Does + +- checks whether a synthetic request is administratively ready under narrow, versioned payer rules +- extracts only a small supported fact set from note text +- returns requirement-level reasoning, blockers, evidence spans, and audit metadata +- exposes the same logic through Streamlit, FastAPI, CLI, exported artifacts, and acceptance snapshots + +## What It Does Not Do + +- no clinical judgment +- no medical-necessity review +- no approval prediction +- no autonomous action +- no real patient data + +## Current Scope + +- payer: `Aetna` +- supported procedures: + - `MRI_LUMBAR` + - `MRI_CERVICAL` + - `MRI_KNEE` + - `CPAP_DEVICE` +- monitored policy source count: `1` + +## Why It Is Credible + +- deterministic extraction and evaluation +- explicit unsupported-scope handling +- refusal-first `CANNOT_DETERMINE` behavior +- versioned rulebook and governance metadata +- golden output snapshots and regression tests + +## Why It Is Intentionally Narrow + +This repo is meant to be explainable by one person in an interview. It prioritizes rigor, auditability, and honest product boundaries over broad claims. diff --git a/PUSH_SUMMARY.md b/PUSH_SUMMARY.md new file mode 100644 index 0000000..c18896a --- /dev/null +++ b/PUSH_SUMMARY.md @@ -0,0 +1,73 @@ +# PUSH SUMMARY + +## Ready To Commit + +This change set is the full three-sprint upgrade plus the final freeze-polish pass. It is ready to commit as a single maturity upgrade if the final verification commands remain green. + +## File Change Summary By Area + +### Architecture / Core Engine + +- Shared typed service orchestration under `engine/service.py` +- Central config and structured logging under `engine/config.py` and `engine/logging_utils.py` +- Expanded schemas, rendering, demo-case helpers, acceptance normalization, and rulebook utilities +- Narrow deterministic extractor extension and additional supported procedures without changing the repo’s core readiness semantics + +### API / CLI / App Surfaces + +- New `api.py` FastAPI surface +- New `cli.py` demo and governance workflows +- Streamlit rebuild in `app.py` around the shared service layer with clearer operator-facing outputs + +### Governance / Rulebook / Drift + +- Structured provenance and supported-procedure metadata +- Governance-only drift reporting and markdown/JSON artifacts +- Reviewed vs active rulebook snapshots, manifest, diffing, and validation + +### Test Suite + +- Expanded unit, API, CLI, service, artifact, Streamlit sanity, rulebook, and acceptance coverage +- Golden snapshots under `test/golden/` + +### Artifacts / Scripts + +- Reproducible artifact generation under `scripts/generate_artifacts.py` +- Golden-output generation under `scripts/generate_golden_outputs.py` +- Stable demo and governance artifacts under `docs/artifacts/` + +### Docs / Interview Materials + +- Rewritten README and docs set for architecture, API, testing, demo walkthrough, and safety/scope +- Added product overview, why-this-exists narrative, limitations, next steps, and interview talking points +- Added implementation, second-pass, marathon-pass, and freeze audit records for review traceability + +## Intentionally Removed + +- `engine/score.py` + +It was empty and provided no value. + +## Files That Deserve Extra Human Review Before Commit + +- `rules/payer_rules.yaml` +- `rules/provenance.yaml` +- `rules/policy_sources.yaml` +- `engine/service.py` +- `engine/rulebook.py` +- `app.py` +- `README.md` +- `FINAL_FREEZE_AUDIT.md` +- `RELEASE_NOTES_LAST_3_SPRINTS.md` + +## Recommended Commit Message Options + +1. `Finalize deterministic prior auth copilot maturity pass and freeze audit` +2. `Ship enterprise-style v3 upgrade with governance, artifacts, and freeze polish` +3. `Complete prior auth readiness copilot overhaul and freeze-ready cleanup` + +## Safe To Push? + +Yes. + +The final freeze verification suite is green, and the remaining risks are documented scope limitations rather than unstable behavior. diff --git a/README.md b/README.md index c4afde2..df23311 100644 --- a/README.md +++ b/README.md @@ -2,70 +2,180 @@ Deterministic prior authorization readiness review for synthetic demo cases. -This repo checks whether a request is administratively ready against versioned payer rules. It does not make clinical judgments, predict approval, or act autonomously. +This repo checks whether a request is administratively ready against versioned payer rules. It does not make clinical judgments, predict approval, assess medical necessity, or act autonomously. -Current scope: the repo supports two Aetna demo procedures, `MRI_LUMBAR` and `CPAP_DEVICE`. Policy drift monitoring is configured only for `MRI_LUMBAR`. +## What This Repo Does -

- Prior Authorization Copilot demo showing a CANNOT_DETERMINE result with blockers, requirement-level reasoning, extracted facts, evidence mapping, and audit summary. -

+- extracts a narrow set of required facts from synthetic note text using deterministic rules +- evaluates those facts against versioned payer requirements +- returns requirement-level reasoning, blocker summaries, evidence mapping, and audit trace data +- exposes the same workflow through Streamlit, FastAPI, and a CLI +- monitors configured policy sources for drift without auto-changing rules or outcomes -## Current Repo +## What This Repo Does Not Do -- Deterministic extraction, evaluation, and letter drafting -- No LLM is used anywhere in the current implementation -- Synthetic inputs only -- Narrow rule coverage -- Streamlit demo UI plus pytest coverage +- no approval prediction +- no clinical decision support +- no claims adjudication +- no medical-necessity review +- no autonomous submission or outreach +- no real payer integrations +- no production or compliance claims -| Procedure | Supported in rules | Monitored for drift | -| --- | --- | --- | -| `MRI_LUMBAR` | Yes | Yes | -| `CPAP_DEVICE` | Yes | No | +## Why Deterministic First -Policy drift status applies only to configured monitored sources. In the current repo, runtime trust remains `demo` for both procedures because provenance is still curated offline in `rules/provenance.yaml`. +This problem is intentionally narrow. For a recruiter-facing and interview-defensible artifact, deterministic logic is the right backbone because it is: -## Core Semantics +- explainable requirement by requirement +- auditable with stable evidence references +- safe to refuse when documentation is missing +- testable with synthetic fixtures and regression cases -- `READY`: all required elements are documented and meet the current rule thresholds -- `NOT_READY`: all required elements are documented, but one or more do not meet threshold -- `CANNOT_DETERMINE`: one or more required elements are not documented +`CANNOT_DETERMINE` is a feature here, not a failure mode. -Any `NOT_DOCUMENTED` result must force `CANNOT_DETERMINE`. +## Current Supported Scope -## Limits +| Payer | Procedure | Supported in rules | Drift monitored | +| --- | --- | --- | --- | +| Aetna | `MRI_LUMBAR` | Yes | Yes | +| Aetna | `MRI_CERVICAL` | Yes | No | +| Aetna | `MRI_KNEE` | Yes | No | +| Aetna | `CPAP_DEVICE` | Yes | No | -- Uses synthetic notes and demo rules -- Supports a small number of procedures and phrasing patterns -- Does not integrate with an EHR, payer, or clearinghouse -- Policy drift monitoring is partial and only covers configured sources +Synthetic inputs only. Policy drift monitoring is governance-only and does not automatically update rules. Procedure registry output now also surfaces category, rule family, rule source label, last rule update, and last reviewed metadata. A lightweight rulebook registry now tracks reviewed and active snapshots separately from runtime drift monitoring. -## Canonical Local Setup +## Architecture At A Glance -Python version: `3.12.3` +- `engine/extract.py`: deterministic extraction +- `engine/evaluate.py`: requirement evaluation and frozen status semantics +- `engine/letter_draft.py`: write-only administrative letter drafting +- `engine/service.py`: shared orchestration for UI, API, CLI, and artifacts +- `engine/policy_monitor.py`: governance-only drift detection and snapshot handling +- `engine/rulebook.py`: versioned rulebook validation and diffing +- `engine/acceptance.py`: golden-output normalization for acceptance checks +- `app.py`: Streamlit operator demo +- `api.py`: FastAPI surface +- `cli.py`: local demo and export workflows + +More detail: [docs/architecture.md](docs/architecture.md) + +## Local Setup + +Python version used in this repo: `3.12.x` + +```bash +make install +make test +make lint +make acceptance +make smoke-ui +make verify +make run +``` + +If you prefer direct commands: ```bash -python3.12 -m venv .venv -source .venv/bin/activate -python -m pip install --upgrade pip -python -m pip install -r requirements.txt +python3 -m pip install -r requirements.txt pytest -q +pytest -q test/test_acceptance_snapshots.py +ruff check . +pytest -q test/test_streamlit_app.py streamlit run app.py ``` -CI runs `pytest -q`. That automated suite includes unit-style tests plus a regression check over the bundled synthetic eval cases. The UI separately surfaces the same bundled synthetic cases as a local demo gate. +## FastAPI + +Run locally: + +```bash +python3 -m uvicorn api:app --reload +``` + +Example calls: + +```bash +curl http://127.0.0.1:8000/health +curl http://127.0.0.1:8000/supported-procedures +curl http://127.0.0.1:8000/demo-cases +curl -X POST http://127.0.0.1:8000/evaluate \ + -H "Content-Type: application/json" \ + -d '{ + "payer": "Aetna", + "procedure_code": "MRI_LUMBAR", + "dx_codes": ["M54.16"], + "site_of_care": "outpatient", + "specialty": "Orthopedics", + "note_text": "Low back pain with right leg radiculopathy x 8 weeks. Completed PT for 8 weeks and NSAIDs with minimal improvement. Denies bowel/bladder incontinence. No saddle anesthesia. Prior imaging: lumbar xray inconclusive. Neuro exam: mild weakness dorsiflexion 4/5." + }' +``` + +Full API notes: [docs/api.md](docs/api.md) + +## CLI + +```bash +python3 cli.py status +python3 cli.py list-procedures +python3 cli.py list-demo-cases +python3 cli.py evaluate --demo-case MRI-01-complete +python3 cli.py evaluate --demo-case MRI-CERV-01-ready +python3 cli.py evaluate --demo-case MRI-KNEE-01-ready +python3 cli.py export-report --demo-case CPAP-02-borderline --output docs/artifacts/manual_export.json --with-letter +python3 cli.py drift-status +python3 cli.py rulebook-status +python3 cli.py rulebook-diff --from-release 2026-04-09-reviewed-v0.4 --to-release 2026-04-09-active-v0.5 +``` + +## Demo Artifacts -## What To Inspect +Stable sample outputs are generated under [docs/artifacts](docs/artifacts): -- [app.py](./app.py): Streamlit demo surface and monitored-source policy panel -- [engine/extract.py](./engine/extract.py): deterministic extraction -- [engine/evaluate.py](./engine/evaluate.py): requirement evaluation and overall status logic -- [rules/payer_rules.yaml](./rules/payer_rules.yaml): current supported procedures -- [EXTRACTION_CONTRACT.md](./EXTRACTION_CONTRACT.md): extraction behavior as implemented -- [FAILURE_MODES.md](./FAILURE_MODES.md): safety and failure boundaries +- [MRI-01-complete.json](docs/artifacts/MRI-01-complete.json) +- [MRI-08-edge-below-threshold.json](docs/artifacts/MRI-08-edge-below-threshold.json) +- [MRI-CERV-01-ready.json](docs/artifacts/MRI-CERV-01-ready.json) +- [MRI-KNEE-01-ready.json](docs/artifacts/MRI-KNEE-01-ready.json) +- [CPAP-02-borderline.json](docs/artifacts/CPAP-02-borderline.json) +- [drift_status.json](docs/artifacts/drift_status.json) +- [drift_report.md](docs/artifacts/drift_report.md) +- [featured_demo_cases.json](docs/artifacts/featured_demo_cases.json) +- [rulebook_status.json](docs/artifacts/rulebook_status.json) +- [rulebook_diff_reviewed_vs_active.json](docs/artifacts/rulebook_diff_reviewed_vs_active.json) +- [rulebook_diff_reviewed_vs_active.md](docs/artifacts/rulebook_diff_reviewed_vs_active.md) +- [status.json](docs/artifacts/status.json) -## Possible Extensions +Regenerate demo artifacts with: + +```bash +python3 -m scripts.generate_artifacts +``` + +Regenerate golden acceptance snapshots with: + +```bash +python3 -m scripts.generate_golden_outputs +``` -- Expand payer and procedure coverage with provenance updates and tests -- Add production integration layers outside this repo -- Evaluate whether optional LLM-assisted text formatting is worth adding behind strict contracts +## Key Docs + +- [docs/architecture.md](docs/architecture.md) +- [docs/api.md](docs/api.md) +- [docs/demo_walkthrough.md](docs/demo_walkthrough.md) +- [docs/testing.md](docs/testing.md) +- [docs/safety_and_scope.md](docs/safety_and_scope.md) +- [PRODUCT_OVERVIEW.md](PRODUCT_OVERVIEW.md) +- [WHY_THIS_EXISTS.md](WHY_THIS_EXISTS.md) +- [DESIGN_DECISIONS.md](DESIGN_DECISIONS.md) +- [LIMITATIONS.md](LIMITATIONS.md) +- [NEXT_STEPS.md](NEXT_STEPS.md) +- [INTERVIEW_TALKING_POINTS.md](INTERVIEW_TALKING_POINTS.md) + +## Repo Quality Gates + +- deterministic-only evaluation path +- synthetic fixtures only +- pytest regression coverage +- acceptance snapshots for representative product outputs +- structured outputs shared across UI, API, CLI, and exported artifacts +- explicit unsupported-scope handling +- honest scope and safety language diff --git a/RELEASE_NOTES_LAST_3_SPRINTS.md b/RELEASE_NOTES_LAST_3_SPRINTS.md new file mode 100644 index 0000000..db54d2d --- /dev/null +++ b/RELEASE_NOTES_LAST_3_SPRINTS.md @@ -0,0 +1,144 @@ +# RELEASE NOTES: LAST 3 SPRINTS + +Sprint boundaries below are inferred from `WORKLOG.md` and the plan files. They are trustworthy summaries of the actual upgrade sequence, even though some milestones were recorded inside broader overnight passes. + +## 1. Sprint 1 Summary + +### What Changed + +- Reframed the repo around a shared deterministic service layer instead of having orchestration live mainly inside `app.py`. +- Added typed schemas, central config, and structured logging. +- Added FastAPI and CLI surfaces over the same deterministic workflow. +- Rebuilt the Streamlit app to consume shared outputs instead of assembling evaluation state inline. +- Added reproducible artifact generation and substantially expanded tests. +- Rewrote the README and added core docs for architecture, API, testing, demo walkthrough, safety/scope, design decisions, limitations, next steps, and interview use. +- Added lightweight Ruff and CI support. + +### Why It Mattered + +Before this sprint, the repo was a strong deterministic demo with a good engine but limited product shape. After this sprint, it became a compact internal-product artifact with clearer boundaries, typed interfaces, reusable surfaces, reproducible outputs, and an interview-defensible story. + +### Key Files + +- `engine/service.py` +- `engine/schemas.py` +- `engine/config.py` +- `engine/logging_utils.py` +- `engine/rendering.py` +- `api.py` +- `cli.py` +- `app.py` +- `scripts/generate_artifacts.py` +- `README.md` +- `docs/architecture.md` +- `docs/api.md` +- `docs/testing.md` +- `.github/workflows/ci.yml` + +## 2. Sprint 2 Summary + +### What Changed + +- Added a new supported deterministic procedure, `MRI_CERVICAL`. +- Strengthened the supported-procedure registry with category, rule family, summary, supported sites, and rule-update metadata. +- Strengthened provenance output with rule source label, source URL where available, last reviewed, last updated, and monitored-source linkage. +- Improved demo-case taxonomy and featured-case metadata. +- Added Streamlit AppTest sanity coverage and stronger regression tests around metadata integrity, contradictory evidence, unsupported site handling, and artifact generation. +- Tightened Makefile ergonomics and normalized the demo walkthrough path. + +### Why It Mattered + +This sprint added depth without changing the architecture. The repo became more credible because it could show more than one deterministic imaging pathway, expose clearer rule provenance, and verify that the UI and metadata surfaces stayed intact. + +### Key Files + +- `rules/payer_rules.yaml` +- `rules/provenance.yaml` +- `rules/policy_sources.yaml` +- `inputs/synthetic_cases.json` +- `engine/service.py` +- `engine/schemas.py` +- `engine/provenance.py` +- `engine/policy_monitor.py` +- `app.py` +- `test/test_streamlit_app.py` +- `test/test_artifact_generation.py` +- `docs/demo_walkthrough.md` + +## 3. Sprint 3 Summary + +### What Changed + +- Added a narrow non-spine deterministic procedure, `MRI_KNEE`, using one carefully scoped new extractor field. +- Introduced a versioned rulebook with reviewed and active snapshots plus manifest-driven stage metadata. +- Added rulebook validation, status, and diff utilities through the service, API, and CLI. +- Deepened drift reporting with freshness metadata, stale-source signaling, and markdown/JSON governance artifacts. +- Added golden acceptance snapshots, normalization logic, and an acceptance command. +- Expanded recruiter-facing docs with clearer product and governance narrative. + +### Why It Mattered + +This sprint turned the repo from a good multi-surface demo into a more mature governance artifact. It showed not just deterministic evaluation, but also how rules are versioned, reviewed, promoted manually, and monitored for drift without pretending to auto-govern production policy logic. + +### Key Files + +- `engine/extract.py` +- `engine/rulebook.py` +- `engine/acceptance.py` +- `rulebook/manifest.yaml` +- `api.py` +- `cli.py` +- `scripts/generate_golden_outputs.py` +- `docs/artifacts/drift_report.md` +- `docs/artifacts/rulebook_diff_reviewed_vs_active.md` +- `PRODUCT_OVERVIEW.md` +- `WHY_THIS_EXISTS.md` +- `INTERVIEW_TALKING_POINTS.md` + +## 4. Final Freeze-Polish Summary + +### What Changed In This Audit Pass + +- Fixed portability problems in governance outputs by replacing machine-specific paths with repo-relative paths. +- Aligned stale-monitoring behavior so governance outputs and Streamlit gating both treat stale or missing baselines as review-required. +- Clarified demo-case expectation wording in the CLI and Streamlit UI. +- Clarified the difference between coarse synthetic fixture checks and exact acceptance snapshots. +- Removed one dead file and trimmed a few low-value duplicate paths in the app and artifact generation flow. +- Replaced remaining machine-specific markdown links with relative links. +- Added the final freeze audit, push summary, and this release-notes summary. + +### Why It Mattered + +These were small changes, but they removed the last rough edges that could undermine recruiter trust or handoff quality. The repo now reads more cleanly, travels better across machines, and tells a more consistent story. + +### Key Files + +- `engine/service.py` +- `engine/rulebook.py` +- `cli.py` +- `app.py` +- `scripts/generate_artifacts.py` +- `README.md` +- `EXTRACTION_CONTRACT.md` +- `FINAL_FREEZE_AUDIT.md` +- `PUSH_SUMMARY.md` + +## 5. Overall Repo Before Vs After + +### Before The 3 Sprints + +- Deterministic extraction and evaluation existed. +- The primary product surface was Streamlit. +- Docs, artifacts, and tests were narrower. +- Governance, provenance, rulebook, and portable product surfaces were limited or absent. + +### After The 3 Sprints + +- One deterministic workflow powers Streamlit, FastAPI, CLI, artifacts, and acceptance checks. +- The repo has four supported procedures, including a narrow non-spine case. +- Outputs expose richer provenance, rulebook stage metadata, drift artifacts, and audit traces. +- The repo includes portable docs, reproducible artifacts, golden outputs, interview materials, and a clear governance story. + +## 6. Final Narrative For Commit / Push Review + +Across the last three sprints, this repo was upgraded from a strong deterministic demo into a compact, interview-defensible internal-product artifact. The core identity stayed fixed: deterministic administrative readiness review, synthetic-only data, narrow explainable scope, refusal-first behavior, and governance-only drift monitoring. The work focused on product surfaces, provenance, rule governance, artifacts, tests, and docs rather than broad platform expansion, which is why the repo now feels substantially more mature without becoming harder to explain. diff --git a/SECOND_PASS_PLAN.md b/SECOND_PASS_PLAN.md new file mode 100644 index 0000000..d56196b --- /dev/null +++ b/SECOND_PASS_PLAN.md @@ -0,0 +1,98 @@ +# SECOND PASS PLAN + +## What Already Exists + +The first pass already established the repo as a compact enterprise-style artifact: + +- shared typed service layer for deterministic orchestration +- central config and structured logging +- FastAPI and CLI surfaces +- rebuilt Streamlit app using shared outputs +- reproducible JSON artifact generation +- expanded tests across service, API, CLI, extraction, rules, provenance, and drift logic +- improved README, architecture, testing, safety, and interview docs +- lightweight Ruff + CI + +The core architecture is not the problem anymore. The next gains should come from depth. + +## What Is Still Weak + +1. Supported deterministic scope is still narrow. + - Only two procedures are currently supported. + - The repo can support one more carefully chosen deterministic pathway without architecture churn. + +2. Rule registry and provenance detail are still thinner than they could be. + - Supported procedures expose only basic fields today. + - Provenance metadata exists but is not consistently normalized or surfaced as a strong registry story. + +3. Drift/governance surfaces are still fairly minimal. + - There is one monitored source. + - The repo could do a better job surfacing monitored-source metadata and rule lineage without widening scope. + +4. Demo quality can still improve. + - Artifacts exist, but demo-case taxonomy and walkthrough clarity can be stronger. + - There is still no direct Streamlit sanity coverage. + +5. Regression safety has room to grow. + - The most important remaining gaps are malformed/partial input handling, contradictory evidence behavior, registry metadata integrity, and UI startup sanity. + +## What This Pass Will Improve + +1. Add one carefully chosen new deterministic procedure if it can reuse the current extraction rigor cleanly. + - If a second new procedure starts to feel vague, stop at one. + +2. Strengthen procedure registry and provenance metadata. + - Add richer procedure-level metadata to the rule registry. + - Normalize and surface provenance fields that matter in interviews: + - rule version + - rule source label + - last reviewed / last updated + - monitored-for-drift status + - rule family / category where useful + +3. Improve governance and drift visibility without changing behavior. + - Keep drift monitoring governance-only. + - Make monitored source status easier to inspect through outputs and docs. + +4. Improve demo quality and UI sanity coverage. + - Refresh artifacts after the new procedure/metadata land. + - Add lightweight Streamlit sanity coverage rather than heavy browser infrastructure unless a minimal browser path proves low-cost. + +5. Expand regression tests around the repo’s product identity. + - new procedure coverage + - ambiguous or contradictory evidence handling + - registry/provenance integrity + - artifact generation + - Streamlit startup/smoke sanity + +6. Tighten local ergonomics where useful. + - Add only lightweight improvements, such as small Makefile target additions and clearer run/verify commands. + +## What Will Intentionally Remain Unchanged + +- no LLM layer +- no database +- no auth +- no claims workflow platform +- no payer scraping +- no approval prediction +- no medical-necessity or clinical recommendation logic +- no autonomous action +- no rewrite of the shared service architecture unless a concrete defect appears + +## Execution Order + +1. Re-audit and record second-pass scope +2. Add one carefully chosen deterministic procedure and corresponding synthetic fixtures +3. Enrich rule registry, provenance, and drift metadata surfaces +4. Add Streamlit sanity coverage and refresh demo artifacts +5. Expand regression tests and tighten local ergonomics +6. Update README, docs, interview notes, and next steps +7. Run Ruff, tests, artifact generation, and a final simplification pass + +## Decision Rules For This Pass + +- Prefer one well-finished new procedure over two shallow ones. +- If metadata and testing improvements are higher ROI than procedure count, bias toward metadata and tests. +- If browser-style screenshot automation becomes dependency-heavy, fall back to stronger artifact generation and Streamlit sanity coverage. +- Every change must remain explainable by one person in an interview. diff --git a/WHY_THIS_EXISTS.md b/WHY_THIS_EXISTS.md new file mode 100644 index 0000000..70f04b2 --- /dev/null +++ b/WHY_THIS_EXISTS.md @@ -0,0 +1,13 @@ +# Why This Exists + +Many prior authorization requests stall for administrative reasons before anyone reaches a deeper clinical review. + +This repo exists to show a disciplined alternative to vague AI demos: + +- start with a narrow problem +- keep the logic deterministic +- refuse when required evidence is missing +- separate governance from runtime behavior +- make every output inspectable + +It is intentionally not trying to solve all of prior auth. The point is to show good product framing, not fake completeness. diff --git a/WORKLOG.md b/WORKLOG.md new file mode 100644 index 0000000..bd6bc49 --- /dev/null +++ b/WORKLOG.md @@ -0,0 +1,319 @@ +# WORKLOG + +## Final Freeze Polish - 2026-04-09 - Audit, cleanup, and verification + +- What changed: + - Performed a final freeze-candidate audit across README, docs, Streamlit, CLI, governance outputs, generated artifacts, golden outputs, and tests. + - Fixed remaining portability issues by replacing machine-specific markdown links with relative links. + - Cleaned up final reviewer-facing wording: + - split `expected_status` from `fixture_label` in CLI demo-case output + - clarified featured-case wording in Streamlit + - clarified the difference between coarse synthetic fixture checks and golden acceptance snapshots + - Removed low-value dead or duplicate complexity: + - deleted empty `engine/score.py` + - reused shared featured-demo helpers in `app.py` + - removed repeated service calls in artifact generation + - Normalized governance file-path outputs to repo-relative values. + - Fixed the final governance-summary inconsistency so stale or missing monitored baselines now set `any_review_required=true`, which keeps JSON, CLI, and Streamlit gating aligned. + - Added final review artifacts: + - `FINAL_FREEZE_AUDIT.md` + - `PUSH_SUMMARY.md` + - `RELEASE_NOTES_LAST_3_SPRINTS.md` +- Why it changed: + - The repo no longer needed more product surface area. It needed a final trust pass that removed portability issues, ambiguous wording, and small inconsistencies before freeze. +- Tests run: + - `ruff check .` -> passed + - `pytest -q` -> `59 passed in 1.31s` + - `make acceptance` -> passed + - `make verify` -> passed + - `python3 cli.py list-demo-cases` -> passed + - `python3 cli.py list-procedures` -> passed + - `python3 cli.py drift-status --json` -> passed + - `python3 cli.py rulebook-status --json` -> passed + - `python3 cli.py evaluate --demo-case MRI-KNEE-01-ready` -> passed + - `python3 -m scripts.generate_artifacts` -> passed + - `python3 -m scripts.generate_golden_outputs` -> passed +- Blockers encountered: + - A final drift-summary mismatch surfaced during verification: the repo correctly flagged a stale monitored source, but the top-level `any_review_required` flag remained `false`. I fixed the summary semantics and regenerated artifacts and goldens. + - The Streamlit AppTest needed one small update to acknowledge the governance gate before loading a featured case, which now mirrors the actual UI flow. +- Follow-up items: + - Freeze the repo. + - Human reviewer can now commit and push with the generated summary files as the handoff packet. + +## Marathon Pass - 2026-04-09 - Re-audit and execution plan + +- What changed: + - Re-audited the repo after the first two upgrade passes. + - Confirmed the current baseline already includes: + - shared deterministic service surfaces + - typed procedure/provenance metadata + - three supported procedures + - governance-only drift reporting + - artifact generation + - regression and Streamlit sanity coverage + - Identified the next highest-ROI backlog for the marathon pass: + - one defensible non-spine procedure if possible + - rulebook versioning and promotion workflow + - stronger offline drift artifacts + - golden outputs and acceptance checks + - tighter docs and local ergonomics + - Created `MARATHON_PASS_PLAN.md` to keep the pass scoped and sequential. +- Why it changed: + - The repo is past the point where broad architecture work adds value. The remaining meaningful gains are depth, governance clarity, and regression safety. +- Tests run: + - Baseline assumed from the end of the prior pass: + - `ruff check .` -> passed + - `pytest -q` -> `45 passed in 0.73s` +- Blockers encountered: + - None. +- Follow-up items: + - Add a non-spine deterministic pathway only if it stays narrow and defensible. + - Build a lightweight rulebook and promotion workflow without changing runtime behavior. + - Add golden outputs, acceptance checks, and richer drift artifacts. + +## Marathon Pass - 2026-04-09 - Non-spine procedure depth and rulebook governance + +- What changed: + - Added a new supported deterministic non-spine procedure: `MRI_KNEE`. + - Introduced one narrow new extractor field, `mechanical_symptoms_documented`, to support a defensible orthopedic MRI contract without changing the engine shape. + - Added knee rule metadata, provenance, synthetic fixtures, featured demo coverage, artifacts, and regression tests. + - Built a lightweight rulebook registry under `rulebook/` with: + - reviewed snapshot `v0.4` + - active snapshot `v0.5` + - manifest-based stage assignment + - Added rulebook validation and diff utilities through the shared service, CLI, and API. + - Surfaced rulebook metadata in service status output and evaluation audit traces. +- Why it changed: + - The repo still had room for one more high-confidence deterministic pathway. After that, governance maturity was the next highest-ROI improvement because it strengthens the interview story without widening product claims. +- Tests run: + - `python3 -m json.tool inputs/synthetic_cases.json >/dev/null` -> passed + - `pytest -q test/test_extract_contracts.py test/test_service.py test/test_config_contracts.py test/test_api.py test/test_cli.py test/test_artifact_generation.py test/test_streamlit_app.py` -> `40 passed in 0.69s` + - `pytest -q test/test_rulebook.py test/test_service.py test/test_api.py test/test_cli.py` -> `28 passed in 0.51s` + - `python3 cli.py evaluate --demo-case MRI-KNEE-01-ready` -> passed + - `python3 cli.py rulebook-status` -> passed + - `python3 cli.py rulebook-diff --from-release 2026-04-09-reviewed-v0.4 --to-release 2026-04-09-active-v0.5` -> passed +- Blockers encountered: + - The first mechanical-symptom extractor pass over-matched denials as positives because of an overly broad fallback token match. I removed the fallback and kept only contextual positive patterns. +- Follow-up items: + - Deepen drift artifacts and stale-review visibility. + - Add golden outputs and an acceptance-style harness. + - Refresh docs and demo surfaces to explain the new procedure and rulebook workflow cleanly. + +## Marathon Pass - 2026-04-09 - Drift visibility, acceptance harness, docs, and final verification + +- What changed: + - Enriched drift status with freshness metadata, stale-source counts, review reasons, and diff-path visibility. + - Added markdown governance artifacts: + - `docs/artifacts/drift_report.md` + - `docs/artifacts/rulebook_diff_reviewed_vs_active.md` + - Added a golden-output acceptance harness: + - `engine/acceptance.py` + - `scripts/generate_golden_outputs.py` + - `test/golden/...` + - `test/test_acceptance_snapshots.py` + - Added lightweight Makefile ergonomics: + - `make acceptance` + - `make goldens` + - fallback to `python3` when `.venv` does not exist + - Updated README, API/architecture/testing/safety/demo docs, interview narrative, product overview, and next steps for the third pass. + - Added `PRODUCT_OVERVIEW.md` and `WHY_THIS_EXISTS.md` to make the recruiter-facing story faster to absorb. +- Why it changed: + - After scope and governance were in place, the highest-ROI work was locking behavior down and making the repo’s product narrative easier to understand quickly. +- Tests run: + - `pytest -q test/test_rulebook.py test/test_service.py test/test_api.py test/test_artifact_generation.py` -> `22 passed in 0.59s` + - `pytest -q test/test_acceptance_snapshots.py test/test_streamlit_app.py test/test_cli.py` -> `11 passed in 0.95s` + - `pytest -q` -> `59 passed in 1.48s` + - `ruff check .` -> passed + - `python3 -m py_compile app.py api.py cli.py engine/*.py scripts/*.py` -> passed + - `python3 -m scripts.generate_artifacts` -> passed + - `python3 -m scripts.generate_golden_outputs` -> passed + - `make acceptance` -> passed + - `make verify` -> passed +- Blockers encountered: + - `make acceptance` initially assumed `.venv/bin/python` existed. I changed the Makefile to fall back to `python3` automatically. + - Ruff flagged a few final line-length and import-order issues before the last verification sweep; those were cleaned up directly. +- Follow-up items: + - Optional: add screenshot automation later only if it stays lightweight and reproducible. + - Optional: add a second monitored source only with a clean offline baseline. + +## Second Pass - 2026-04-09 - Re-audit and plan + +- What changed: + - Re-audited the current second-pass starting point across rules, shared service, API, CLI, Streamlit app, artifacts, docs, and test coverage. + - Confirmed the repo is currently green before making changes. + - Identified the highest-ROI gaps for this pass: + - only two supported procedures + - limited rule/provenance metadata surfaced to users + - drift reporting is still minimal + - no direct Streamlit sanity coverage + - demo-case taxonomy and regression fixtures can be stronger + - Created `SECOND_PASS_PLAN.md` to scope the next set of improvements. +- Why it changed: + - The repo no longer needs broad architectural work. This pass should deepen credibility and regression safety around the existing deterministic backbone. +- Tests run: + - `pytest -q` -> `34 passed in 0.20s` + - `ruff check .` -> passed +- Blockers encountered: + - None. +- Follow-up items: + - Add carefully scoped new deterministic procedure coverage. + - Enrich supported-procedure and provenance metadata surfaces. + - Add lightweight Streamlit sanity coverage and refreshed artifacts. + - Expand regression fixtures and update docs accordingly. + +## Second Pass - 2026-04-09 - Deterministic coverage and metadata depth + +- What changed: + - Added a new supported deterministic procedure: `MRI_CERVICAL`. + - Extended the rule registry with procedure metadata for all supported procedures: + - category + - rule family + - summary + - supported sites + - last rule update + - rule notes + - Extended provenance metadata with structured rule source fields: + - rule source label + - source URL where available + - last reviewed + - rule last updated + - monitored source linkage + - Added a stronger monitored-source label to `rules/policy_sources.yaml`. + - Wired enriched metadata through the shared service, supported-procedure outputs, evaluation outputs, CLI summaries, Streamlit views, and drift status surfaces. + - Added cervical MRI synthetic demo cases and improved featured-case taxonomy with scenario labels and tags. +- Why it changed: + - The repo needed more depth, not more architecture. A second spine MRI pathway plus richer registry/provenance output materially improves realism and interview defensibility while preserving the deterministic core. +- Tests run: + - `python3 -m json.tool inputs/synthetic_cases.json >/dev/null` -> passed + - `pytest -q` -> `45 passed in 0.95s` + - `python3 cli.py list-procedures` -> passed +- Blockers encountered: + - None. +- Follow-up items: + - Refresh artifacts to include the new cervical pathway and featured-case registry. + - Add Streamlit sanity coverage and finish doc updates. + +## Second Pass - 2026-04-09 - Demo quality, regression hardening, and final verification + +- What changed: + - Added artifact-generation coverage and new generated outputs: + - `MRI-CERV-01-ready.json` + - `featured_demo_cases.json` + - `status.json` + - Added Streamlit AppTest sanity coverage to ensure the app loads and produces results from featured demo input. + - Expanded tests for: + - invalid rule metadata + - provenance metadata presence + - contradictory red-flag evidence precedence + - cervical procedure registry behavior + - unsupported site-of-care handling + - enriched drift surface integrity + - artifact generation output + - Added lightweight Makefile ergonomics: + - `make verify` + - `make smoke-ui` + - `make evaluate-case CASE=...` + - Normalized the walkthrough doc path to `docs/demo_walkthrough.md`. + - Updated README, architecture/API/testing/demo/interview/next-steps docs for the second pass. +- Why it changed: + - This step improves the repo’s live-demo clarity, regression safety, and local usability without widening scope into new infrastructure. +- Tests run: + - `ruff check .` -> passed + - `pytest -q` -> `45 passed in 0.73s` + - `python3 cli.py evaluate --demo-case MRI-CERV-01-ready` -> passed + - `python3 -m scripts.generate_artifacts` -> passed + - `python3 -m py_compile app.py api.py cli.py engine/*.py scripts/*.py` -> passed +- Blockers encountered: + - Two long-line lint failures in the Streamlit metadata panel were cleaned up before final verification. +- Follow-up items: + - Optional: add one more non-spine deterministic pathway only if it meets the same rigor. + - Optional: add screenshot automation later if it can remain lightweight and reproducible. + +## Milestone 0 - Audit and baseline + +- What changed: + - Audited the current repo structure, core engine modules, Streamlit app flow, rules/config files, tests, and CI. + - Established the first-pass upgrade strategy: preserve deterministic evaluation logic and add a thin shared service layer for API, CLI, UI, and artifact generation. + - Created this worklog to record overnight progress. +- Why it changed: + - The repo already has a strong deterministic core. The highest-leverage improvement is not a rewrite; it is turning the existing logic into a cleaner, typed, reusable product surface that is easier to defend in interviews. +- Tests run: + - `pytest -q` -> `21 passed in 0.10s` +- Blockers encountered: + - None. +- Follow-up items: + - Write `IMPLEMENTATION_PLAN.md`. + - Extract evaluation orchestration out of `app.py`. + - Add central config, structured logging, API, CLI, and stronger tests without widening scope. + +## Milestone 1 - Shared service foundation + +- What changed: + - Expanded `engine/schemas.py` with typed models for evidence spans, requirement definitions, supported procedures, demo cases, blockers, metrics, audit traces, drift reports, and service status. + - Added `engine/config.py` for central path/config validation and `.env.example` for lightweight local overrides. + - Added `engine/logging_utils.py` for structured JSON logging. + - Added `engine/demo_cases.py`, `engine/service.py`, and `engine/rendering.py` so Streamlit, API, CLI, and artifacts can share one deterministic orchestration path. + - Updated `engine/evaluate.py` to preserve normalized evidence span references in requirement results without changing readiness semantics. + - Added initial `api.py` and `cli.py` shells on top of the shared service. +- Why it changed: + - The repo’s main weakness was orchestration living inside `app.py`. This step creates a narrow enterprise-style application layer without rewriting the deterministic extraction or rule engine. +- Tests run: + - `pytest -q` -> `21 passed in 0.06s` +- Blockers encountered: + - None. +- Follow-up items: + - Route Streamlit through the shared service. + - Finish CLI/API behavior and add tests. + - Add reproducible artifact generation and updated docs. + +## Milestone 2 - Product surfaces and regression coverage + +- What changed: + - Rebuilt `app.py` as a thinner operator-style Streamlit UI that consumes the shared service instead of assembling evaluation state inline. + - Added a FastAPI surface in `api.py` with typed endpoints for health/status, supported procedures, demo cases, readiness evaluation, and drift status. + - Added `cli.py` for local demo workflows: status, procedure listing, demo case listing, evaluation, validation, export, and drift inspection. + - Added `scripts/generate_artifacts.py` for reproducible JSON artifacts under `docs/artifacts`. + - Expanded test coverage with new service, API, and CLI tests. + - Added minimal dependencies and Makefile targets required to run the new surfaces and lint checks. +- Why it changed: + - These changes convert the repo from a single-demo surface into a compact internal-product artifact with reusable programmatic interfaces, reproducible exports, and stronger interview defensibility. +- Tests run: + - `pytest -q` -> `34 passed in 0.19s` +- Blockers encountered: + - FastAPI was not installed locally, so I added the minimal dependencies required for the API, CLI tests, and linting. +- Follow-up items: + - Generate stable artifacts. + - Overhaul recruiter-facing docs and CI. + - Run full lint and final verification. + +## Milestone 3 - Docs, artifacts, CI, and final verification + +- What changed: + - Replaced the top-level README with a tighter product narrative, setup instructions, API/CLI examples, artifact links, and a docs index. + - Added the requested recruiter-facing docs: + - `docs/architecture.md` + - `docs/api.md` + - `docs/demo_walkthrough.md` + - `docs/testing.md` + - `docs/safety_and_scope.md` + - `DESIGN_DECISIONS.md` + - `LIMITATIONS.md` + - `NEXT_STEPS.md` + - `INTERVIEW_TALKING_POINTS.md` + - Added `pyproject.toml` for lightweight Ruff and pytest config. + - Updated GitHub Actions CI to run lint, tests, and artifact generation. + - Generated stable JSON artifacts under `docs/artifacts/`. + - Added `test/conftest.py` to make repo-root imports stable under pytest. + - Tuned the default log level to `WARNING` so CLI and artifact commands stay clean by default. +- Why it changed: + - This step turns the repo into a more defensible overnight artifact: easier to demo, easier to explain, easier to run locally, and more honest about scope and limitations. +- Tests run: + - `ruff check .` -> passed + - `pytest -q` -> `34 passed in 0.16s` + - `python3 cli.py evaluate --demo-case MRI-01-complete` -> passed + - `python3 -m scripts.generate_artifacts` -> passed +- Blockers encountered: + - `pytest` lost repo-root imports after the config pass, so I fixed that with `test/conftest.py` rather than relying on external environment behavior. +- Follow-up items: + - Optional: capture an updated Streamlit screenshot if the README needs a fresh visual. + - Optional: add one or two more supported procedures using the same deterministic discipline. diff --git a/api.py b/api.py new file mode 100644 index 0000000..5916997 --- /dev/null +++ b/api.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse + +from engine.schemas import ( + DemoCase, + DriftStatusReport, + ErrorResponse, + EvaluationResult, + PARequest, + RulebookDiffResponse, + RulebookStatusResponse, + StatusResponse, + SupportedProcedure, +) +from engine.service import InvalidRequestError, ReadinessService, ServiceError, UnsupportedScopeError + +app = FastAPI( + title="Prior Authorization Readiness Copilot API", + version="0.5.0", + description=( + "Deterministic administrative readiness review for synthetic prior authorization demo cases. " + "No clinical judgment, approval prediction, or autonomous action." + ), +) + +service = ReadinessService() + + +@app.exception_handler(UnsupportedScopeError) +@app.exception_handler(InvalidRequestError) +@app.exception_handler(ServiceError) +async def handle_service_error(_: Request, exc: ServiceError) -> JSONResponse: + error = ErrorResponse(error=exc.code, detail=str(exc)) + status_code = 400 if isinstance(exc, InvalidRequestError) else 422 if isinstance(exc, UnsupportedScopeError) else 500 + return JSONResponse(status_code=status_code, content=error.model_dump()) + + +@app.get("/", response_model=StatusResponse, tags=["status"]) +def root_status() -> StatusResponse: + return service.get_status() + + +@app.get("/health", response_model=StatusResponse, tags=["status"]) +@app.get("/status", response_model=StatusResponse, tags=["status"]) +def health_status() -> StatusResponse: + return service.get_status() + + +@app.get("/supported-procedures", response_model=list[SupportedProcedure], tags=["catalog"]) +def supported_procedures() -> list[SupportedProcedure]: + return service.list_supported_procedures() + + +@app.get("/demo-cases", response_model=list[DemoCase], tags=["catalog"]) +def demo_cases() -> list[DemoCase]: + return service.list_demo_case_summaries() + + +@app.post("/evaluate", response_model=EvaluationResult, tags=["evaluation"]) +def evaluate(request: PARequest) -> EvaluationResult: + return service.evaluate(request) + + +@app.get("/drift-status", response_model=DriftStatusReport, tags=["governance"]) +def drift_status() -> DriftStatusReport: + return service.get_drift_status() + + +@app.get("/rulebook", response_model=RulebookStatusResponse, tags=["governance"]) +def rulebook_status() -> RulebookStatusResponse: + return service.get_rulebook_status() + + +@app.get("/rulebook/diff", response_model=RulebookDiffResponse, tags=["governance"]) +def rulebook_diff(from_release_id: str, to_release_id: str) -> RulebookDiffResponse: + return service.get_rulebook_diff(from_release_id, to_release_id) diff --git a/app.py b/app.py index e065bbd..2955a7d 100644 --- a/app.py +++ b/app.py @@ -1,1066 +1,662 @@ -# -*- coding: utf-8 -*- +from __future__ import annotations - -import streamlit as st import json -import uuid -import hashlib -from datetime import datetime, timezone from pathlib import Path -from engine.rules_loader import load_rules -from engine.extract import extract_facts -from engine.evaluate import ( - evaluate_requirements, - compute_readiness_score, - compute_overall_status, -) -from engine.provenance import ( - get_provenance_entry, - load_provenance, - normalized_dx_codes, - policy_trust_from_provenance, -) - -BASE_DIR = Path(__file__).resolve().parent +import streamlit as st -# Schema + write-only letter drafting -from engine.schemas import PARequest, ReadinessReport, RequirementResult -from engine.letter_draft import draft_letter as draft_letter_writeonly +from engine.config import load_app_config +from engine.demo_cases import expected_overall_status_for_demo_case, featured_demo_cases +from engine.rendering import export_evaluation_payload +from engine.schemas import EvaluationResult, PARequest +from engine.service import ReadinessService, ServiceError +from engine.test_suites import run_cases -# Governance: policy drift monitor (offline UI reads artifacts; does not fetch internet) -from engine.policy_monitor import load_policy_sources, read_latest_snapshot +BASE_DIR = Path(__file__).resolve().parent -# ---------------------------- -# Page config + CSS -# ---------------------------- st.set_page_config(page_title="PA Readiness Copilot", layout="wide") st.markdown( """ """, unsafe_allow_html=True, ) -st.title("PA Readiness Copilot") -st.caption("Administrative decision support only. Not clinical decision support, approval prediction, or billing advice.") - - -# ---------------------------- -# Session state initialization -# ---------------------------- -if "last_eval" not in st.session_state: - st.session_state.last_eval = None -if "test_rows" not in st.session_state: - st.session_state.test_rows = None - -# Letter UI state (write-only) -if "letter_text" not in st.session_state: - st.session_state.letter_text = "" -if "letter_meta" not in st.session_state: - st.session_state.letter_meta = {} -if "letter_error" not in st.session_state: - st.session_state.letter_error = "" - -# Policy drift acknowledge gate state -if "ack_policy_drift" not in st.session_state: - st.session_state.ack_policy_drift = False +@st.cache_resource +def get_service() -> ReadinessService: + return ReadinessService(load_app_config(BASE_DIR)) -# ---------------------------- -# Load rules + provenance -# ---------------------------- -RULES_PATH = "rules/payer_rules.yaml" -rules = load_rules(RULES_PATH) -payers = sorted(rules["payers"].keys()) -PROV_PATH = "rules/provenance.yaml" -prov = load_provenance(PROV_PATH) - -SITE_OPTIONS = ["outpatient", "inpatient", "ASC", "office"] - - -# ---------------------------- -# Sidebar: Synthetic eval status (cached) -# ---------------------------- -st.sidebar.markdown("### 🧪 Synthetic Eval Status") - -# Manual cache bust button (prevents stale results after edits) -if st.sidebar.button("🔄 Refresh synthetic eval status", width="stretch"): - st.cache_data.clear() - st.rerun() +service = get_service() +config = service.config @st.cache_data(ttl=300) -def _get_test_health(): - from engine.test_suites import run_cases - - results = run_cases("rules/payer_rules.yaml", "inputs/synthetic_cases.json") - passed = sum(1 for r in results if r.get("pass") == "✅") - total = len(results) - return passed, total, results - - -try: - passed, total, test_results_cached = _get_test_health() - pass_rate = (passed / total * 100) if total > 0 else 0.0 - - tests_healthy = (total > 0 and passed == total) - st.session_state["tests_healthy"] = tests_healthy - - if pass_rate >= 90: - st.sidebar.success(f"✅ Synthetic eval: {passed}/{total} cases matched ({pass_rate:.0f}%)") - elif pass_rate >= 70: - st.sidebar.warning(f"⚠️ Synthetic eval: {passed}/{total} cases matched ({pass_rate:.0f}%)") - else: - st.sidebar.error(f"❌ Synthetic eval: {passed}/{total} cases matched ({pass_rate:.0f}%)") - - with st.sidebar.expander("View synthetic eval mismatches"): - failures = [r for r in test_results_cached if r.get("pass") == "❌"] - if failures: - for f in failures[:10]: - st.write(f"- {f.get('id')}: expected `{f.get('expected')}`, got `{f.get('predicted')}`") - else: - st.success("All bundled synthetic cases matched expected labels.") - -except Exception as e: - st.session_state["tests_healthy"] = False - st.sidebar.warning(f"Synthetic eval status unavailable: {e}") - - -# ---------------------------- -# Helpers -# ---------------------------- -def _hash_note(note: str) -> str: - h = hashlib.sha256((note or "").encode("utf-8")).hexdigest() - return h[:16] # short hash for readability - - -def _compute_metrics(score_info: dict) -> dict: - total = int(score_info.get("total", 0) or 0) - met = int(score_info.get("met_count", 0) or 0) - not_met = int(score_info.get("not_met_count", 0) or 0) - not_doc = int(score_info.get("not_documented_count", 0) or 0) +def get_synthetic_eval_status() -> tuple[int, int, list[dict]]: + rows = run_cases(str(config.rules_path), str(config.synthetic_cases_path)) + passed = sum(1 for row in rows if row.get("pass") == "✅") + return passed, len(rows), rows + + +def load_case_into_session(case: dict) -> None: + st.session_state["selected_demo_case_id"] = case["id"] + st.session_state["payer"] = case["payer"] + st.session_state["procedure_code"] = case["procedure_code"] + st.session_state["dx_codes"] = ", ".join(case.get("dx_codes", [])) + st.session_state["site_of_care"] = case.get("site_of_care", "outpatient") + st.session_state["specialty"] = case.get("specialty", "unknown") + st.session_state["note_text"] = case.get("note_text", "") + + +def current_request() -> PARequest: + dx_codes = [item.strip() for item in st.session_state.get("dx_codes", "").split(",") if item.strip()] + return PARequest( + payer=st.session_state["payer"], + procedure_code=st.session_state["procedure_code"], + dx_codes=dx_codes, + site_of_care=st.session_state["site_of_care"], + specialty=st.session_state["specialty"], + note_text=st.session_state["note_text"], + ) - extraction_success_rate = round(((met + not_met) / total * 100), 1) if total else 0.0 - compliance_rate = round((met / (met + not_met) * 100), 1) if (met + not_met) > 0 else None - return { - "extraction_success_rate": extraction_success_rate, - "extraction_failure_count": not_doc, - "compliance_rate": compliance_rate, - "compliant_count": met, - "non_compliant_count": not_met, +def status_panel(evaluation: EvaluationResult) -> None: + status = evaluation.overall_status + klass = { + "READY": "status-ready", + "NOT_READY": "status-not-ready", + "CANNOT_DETERMINE": "status-cannot-determine", + }.get(status, "status-unknown") + + summaries = { + "READY": ( + "Administratively ready under the current versioned demo rules.", + "All required elements were explicitly documented and met threshold.", + ), + "NOT_READY": ( + "Not ready to submit under the current versioned demo rules.", + "At least one required element was documented but failed threshold.", + ), + "CANNOT_DETERMINE": ( + "Readiness cannot be determined from the documentation provided.", + "At least one required element was missing or not explicit enough for deterministic extraction.", + ), } + headline, detail = summaries.get( + status, + ("Status unavailable.", "Unexpected status returned by the deterministic workflow."), + ) + st.markdown( + f""" +
+
Decision
+ {status}
+ {headline}
+ {detail} +
+ """, + unsafe_allow_html=True, + ) -def _render_key_value_block(title: str, payload: dict) -> None: - st.markdown(f"**{title}**") - lines = [] - for key, value in payload.items(): - rendered = "null" if value is None else str(value) - lines.append(f"{key}: {rendered}") - st.code("\n".join(lines) if lines else "(none)", language="text") - - -def _render_evidence_map_block(title: str, evidence_map: dict) -> None: - st.markdown(f"**{title}**") - if not evidence_map: - st.caption("No evidence spans captured.") - return - - blocks = [] - for key, spans in evidence_map.items(): - blocks.append(f"{key}:") - if not spans: - blocks.append(" (none)") - continue - for idx, span in enumerate(spans, start=1): - text = str(span.get("text", "")).strip() - start = span.get("start") - end = span.get("end") - blocks.append(f" [{idx}] {start}-{end}: {text}") - st.code("\n".join(blocks), language="text") - - -def _summary_text(value: object, fallback: str = "Not available") -> str: - if value is None: - return fallback - text = str(value).strip() - return text if text else fallback - - -def _render_audit_summary_card(audit: dict, provenance: dict, invariant_errors: list[str]) -> None: - blocking = audit.get("blocking_issues") or {} - not_documented = blocking.get("not_documented") or [] - not_met = blocking.get("not_met") or [] - total_blockers = len(not_documented) + len(not_met) - letter_artifacts = audit.get("letter_artifacts") or {} - - st.subheader("Audit Summary") - st.caption("Compact trust and traceability view. Full audit details remain available at the bottom.") - - top_row = st.columns(4) - with top_row[0]: - st.caption("Run ID") - st.code(_summary_text(audit.get("run_id")), language="text") - with top_row[1]: - st.caption("Note Hash") - st.code(_summary_text(audit.get("note_hash")), language="text") - with top_row[2]: - st.caption("Rules Version") - st.code(_summary_text(audit.get("rules_version")), language="text") - with top_row[3]: - st.caption("Trust Level") - st.code(_summary_text(str(audit.get("policy_trust_level", "")).upper()), language="text") - - bottom_row = st.columns(4) - with bottom_row[0]: - st.metric("Invariant Checks", "PASS" if not invariant_errors else "CHECK") - with bottom_row[1]: - st.metric("Total Blockers", total_blockers) - with bottom_row[2]: - st.metric("Missing Requirements", len(not_documented)) - with bottom_row[3]: - st.caption("Letter Hash") - st.code(_summary_text(letter_artifacts.get("letter_hash_sha256_16"), fallback="No letter draft yet"), language="text") - - source_name = provenance.get("source_name") or "Not documented" - source_type = provenance.get("source_type") or "Not documented" - last_reviewed = provenance.get("last_reviewed") or "Not documented" - st.caption(f"Policy source: {source_name} | Source type: {source_type} | Last reviewed: {last_reviewed}") - - -def _format_extracted_fact_value(key: str, value: object) -> str: - if value is None: - return "Missing from note" - - if key in {"conservative_therapy_weeks", "symptom_duration_weeks"}: - return f"{value} weeks" - - if key == "prior_imaging_result": - mapping = { - "none": "No prior imaging documented", - "inconclusive": "Prior imaging documented as inconclusive", - "abnormal": "Prior imaging documented as abnormal", - } - return mapping.get(str(value), str(value)) - if isinstance(value, bool): - return "Documented" if value else "Documented as absent" +def render_requirement_result(result) -> None: + default_open = result.status != "MET" + icon = {"MET": "✅", "NOT_MET": "⚠️", "NOT_DOCUMENTED": "❌"}.get(result.status, "❓") + with st.expander(f"{icon} {result.label}", expanded=default_open): + c1, c2 = st.columns([1.2, 2]) + with c1: + st.metric("Status", result.status) + with c2: + st.write(result.reason) - return str(value) + if result.evidence: + st.info(f"What the rule expects: {result.evidence}") + if result.evidence_snippets: + st.markdown("**Evidence found in the note**") + for snippet in result.evidence_snippets[:5]: + st.code(snippet, language="text") + else: + st.caption("No supporting snippet was captured for this requirement.") -@st.cache_data -def _load_synthetic_cases(cases_path: str = "inputs/synthetic_cases.json") -> list[dict]: - cases_file = (BASE_DIR / cases_path).resolve() - with cases_file.open("r", encoding="utf-8") as f: - return json.load(f) + if result.evidence_spans: + refs = [f"{span.start}-{span.end}" for span in result.evidence_spans[:5]] + st.caption(f"Normalized evidence references: {', '.join(refs)}") -def _featured_showcase_cases(cases: list[dict]) -> list[dict]: - featured = [case for case in cases if (case.get("showcase") or {}).get("featured")] - return sorted(featured, key=lambda case: (case.get("showcase") or {}).get("sort_order", 999)) +def render_fact_card(label: str, value: object, status: str) -> None: + if value is None: + display = "Missing from note" + elif isinstance(value, bool): + display = "Documented" if value else "Explicitly denied or absent" + elif label.endswith("(weeks)"): + display = f"{value} weeks" + else: + display = str(value) + + st.markdown(f"**{label}**") + st.write(display) + if status == "NOT_DOCUMENTED": + st.caption("Missing or not explicit enough for deterministic extraction.") + elif status == "NOT_MET": + st.caption("Documented, but below the current rule threshold.") + else: + st.caption("Captured and used in deterministic evaluation.") + + +def render_scope_panel() -> None: + st.markdown( + """ +
+
Scope
+ This product checks administrative readiness only. +

It does not make clinical judgments, predict approval, review medical necessity, or take autonomous action.

+

Synthetic demo inputs only. Human review still sits before any real submission workflow.

+
+ """, + unsafe_allow_html=True, + ) -def _showcase_status_chip(status: str) -> str: - chips = { - "READY": "READY", - "NOT_READY": "NOT_READY", - "CANNOT_DETERMINE": "CANNOT_DETERMINE", +supported_procedures = service.list_supported_procedures() +payers = sorted({item.payer for item in supported_procedures}) +procedures_by_payer = {payer: [item for item in supported_procedures if item.payer == payer] for payer in payers} +registry_rows = [ + { + "payer": procedure.payer, + "procedure_code": procedure.procedure_code, + "display_name": procedure.display_name, + "category": procedure.metadata.category, + "rule_family": procedure.metadata.rule_family, + "trust": procedure.policy_trust_level.upper(), + "drift_monitored": "Yes" if procedure.monitored_for_drift else "No", + "rule_source": procedure.provenance.rule_source_label or procedure.provenance.source_name or "n/a", + "last_rule_update": procedure.metadata.last_rule_update or "n/a", + "last_reviewed": procedure.provenance.last_reviewed or "n/a", } - rendered = chips.get(status, status or "UNKNOWN") - return f"`{rendered}`" - - -def _load_case_into_intake(case: dict) -> None: - payer = case.get("payer") or payers[0] - if payer not in rules["payers"]: - payer = payers[0] - - procedures = rules["payers"][payer]["procedures"] - proc_code = case.get("procedure_code") - if proc_code not in procedures: - proc_code = next(iter(procedures)) - - st.session_state["intake_payer"] = payer - st.session_state["intake_proc_code"] = proc_code - st.session_state["intake_dx_raw"] = ", ".join(case.get("dx_codes", [])) - st.session_state["intake_specialty"] = case.get("specialty", "") - st.session_state["intake_site"] = case.get("site_of_care", "outpatient") - st.session_state["intake_note_text"] = case.get("note_text", "") - st.session_state["selected_showcase_case_id"] = case.get("id") - - -synthetic_cases = _load_synthetic_cases() -featured_showcase_cases = _featured_showcase_cases(synthetic_cases) + for procedure in supported_procedures +] -if "selected_showcase_case_id" not in st.session_state: - st.session_state.selected_showcase_case_id = None - -if "intake_payer" not in st.session_state or st.session_state["intake_payer"] not in rules["payers"]: - st.session_state["intake_payer"] = payers[0] - -current_intake_payer = st.session_state["intake_payer"] -current_procedure_options = list(rules["payers"][current_intake_payer]["procedures"].keys()) -if "intake_proc_code" not in st.session_state or st.session_state["intake_proc_code"] not in current_procedure_options: - st.session_state["intake_proc_code"] = current_procedure_options[0] - -if "intake_dx_raw" not in st.session_state: - st.session_state["intake_dx_raw"] = "" -if "intake_specialty" not in st.session_state: - st.session_state["intake_specialty"] = "" -if "intake_site" not in st.session_state or st.session_state["intake_site"] not in SITE_OPTIONS: - st.session_state["intake_site"] = "outpatient" -if "intake_note_text" not in st.session_state: - st.session_state["intake_note_text"] = "" - - -# ---------------------------- -# Policy Drift Monitor (governance-only) -# ---------------------------- - -# Base directory for absolute path resolution (prevents Streamlit CWD issues) -BASE_DIR = Path(__file__).resolve().parent - -# Ensure drift ack state exists +if "last_eval_payload" not in st.session_state: + st.session_state["last_eval_payload"] = None +if "letter_text" not in st.session_state: + st.session_state["letter_text"] = "" +if "letter_meta" not in st.session_state: + st.session_state["letter_meta"] = {} if "ack_policy_drift" not in st.session_state: - st.session_state.ack_policy_drift = False + st.session_state["ack_policy_drift"] = False +if "selected_demo_case_id" not in st.session_state: + st.session_state["selected_demo_case_id"] = None +if "payer" not in st.session_state: + st.session_state["payer"] = payers[0] +if "procedure_code" not in st.session_state: + st.session_state["procedure_code"] = procedures_by_payer[st.session_state["payer"]][0].procedure_code +if "dx_codes" not in st.session_state: + st.session_state["dx_codes"] = "" +if "site_of_care" not in st.session_state: + st.session_state["site_of_care"] = config.allowed_sites[0] +if "specialty" not in st.session_state: + st.session_state["specialty"] = "" +if "note_text" not in st.session_state: + st.session_state["note_text"] = "" -def _read_drift_log(log_path: Path) -> list[dict]: - """ - Read append-only drift log. Ignores malformed lines. - """ - if not log_path.exists(): - return [] - - events: list[dict] = [] - with log_path.open("r", encoding="utf-8") as f: - for line in f: - line = line.strip() - if not line: - continue - try: - events.append(json.loads(line)) - except json.JSONDecodeError: - continue - - return events - - -def _policy_monitor_status( - snapshot_root: str = "policy_snapshots", - sources_path: str = "rules/policy_sources.yaml", -) -> tuple[list[dict], bool]: - """ - Offline UI status: - - Reads latest snapshots (baseline exists?) - - Reads drift_log.jsonl to determine REVIEW_REQUIRED - Does NOT fetch internet. +st.markdown( """ - snapshot_root_p = (BASE_DIR / snapshot_root).resolve() - log_path = snapshot_root_p / "drift_log.jsonl" - - # Load policy sources (absolute path) - try: - sources = load_policy_sources((BASE_DIR / sources_path).resolve()) - except Exception: - sources = [] - - # Load drift events - events = _read_drift_log(log_path) - - # Latest event per source (append-only; last occurrence wins) - latest_event_by_id: dict[str, dict] = {} - for e in events: - sid = e.get("id") - if sid: - latest_event_by_id[str(sid)] = e - - rows: list[dict] = [] - any_review_required = False - - for src in sources: - latest_snap = read_latest_snapshot(snapshot_root_p, src.id) - last_checked = latest_snap.get("fetched_at_utc") if latest_snap else None - - status = "NO_BASELINE" if latest_snap is None else "OK" - - last_evt = latest_event_by_id.get(src.id, {}) - if last_evt.get("event") == "POLICY_DRIFT_DETECTED": - status = "REVIEW_REQUIRED" - any_review_required = True - - rows.append( - { - "id": src.id, - "payer": src.payer, - "procedure_code": src.procedure_code, - "trust_level": src.trust_level, - "status": status, - "last_checked_utc": last_checked, - } - ) - - return rows, any_review_required +
+

Prior Authorization Readiness Copilot

+

Deterministic administrative readiness review for versioned payer rules and synthetic demo cases.

+

Narrow, explainable, auditable behavior. No clinical judgment. No approval prediction. No autonomous action.

+
+ """, + unsafe_allow_html=True, +) +hero_cols = st.columns(3) +with hero_cols[0]: + st.metric("Supported procedures", len(supported_procedures)) +with hero_cols[1]: + st.metric("Monitored policy sources", len(service.policy_sources)) +with hero_cols[2]: + st.metric("Synthetic demo cases", len(service.demo_cases)) -# ---------------------------- -# Global health banner (explicit gate) -# ---------------------------- -tests_healthy = bool(st.session_state.get("tests_healthy", False)) -if not tests_healthy: - st.error( - "🚫 **Synthetic Eval Mismatch** — bundled synthetic cases do not all match their expected labels. " - "Review the mismatches before trusting demo outputs." - ) +render_scope_panel() +with st.sidebar: + st.header("Quality Gates") + try: + passed, total, synthetic_rows = get_synthetic_eval_status() + if passed == total: + st.success(f"Synthetic eval suite: {passed}/{total}") + else: + st.error(f"Synthetic eval suite: {passed}/{total}") + st.caption("Coarse fixture-label regression. Exact output shapes are protected separately by acceptance snapshots.") + with st.expander("View synthetic evaluation details"): + failures = [row for row in synthetic_rows if row.get("pass") != "✅"] + if failures: + for failure in failures: + st.write( + f"- {failure['id']}: expected `{failure['expected']}`, got `{failure['predicted']}` ({failure['overall_status']})" + ) + else: + st.write("All bundled synthetic cases matched expected labels.") + except Exception as exc: # pragma: no cover - defensive UI path + passed, total = 0, 0 + synthetic_rows = [] + st.error(f"Synthetic eval suite unavailable: {exc}") -# ---------------------------- -# Policy Monitor panel + drift gate (shown before intake) -# ---------------------------- -try: - policy_rows, any_review_required = _policy_monitor_status() -except Exception as e: - policy_rows, any_review_required = [], False - st.warning(f"Policy monitor unavailable: {type(e).__name__}: {e}") + tests_healthy = bool(total and passed == total) -st.subheader("Policy Monitor (Configured Sources)") -st.caption("Shows drift status for configured monitored sources only. It does not auto-update rules or change outcomes.") + st.header("Supported Scope") + for procedure in supported_procedures: + monitored = "Yes" if procedure.monitored_for_drift else "No" + st.caption( + f"{procedure.payer} | {procedure.procedure_code} | {procedure.metadata.category} | " + f"trust={procedure.policy_trust_level} | drift monitored={monitored}" + ) -if policy_rows: - st.dataframe(policy_rows, width="stretch") - st.caption("Supported procedures without configured monitored sources do not appear in this table.") -else: - st.info("No policy sources configured (or policy_sources.yaml missing).") -if any_review_required: - st.warning("⚠️ Policy drift detected for one or more monitored sources — related rules may be stale. Verify policy and update rules/tests before trusting outputs.") - st.session_state.ack_policy_drift = st.checkbox( - "I acknowledge monitored-source drift; related demo outputs may be stale.", - value=st.session_state.ack_policy_drift, +drift_report = service.get_drift_status() +rulebook_status = service.get_rulebook_status() +st.subheader("Governance Monitor") +st.caption("Configured monitored sources only. Drift detection is governance-only and never changes rules automatically.") +st.dataframe([source.model_dump(mode="json") for source in drift_report.sources], width="stretch") +if drift_report.any_review_required: + st.warning( + "One or more monitored sources require governance review because a policy diff was detected " + "or the monitoring baseline is stale or missing." ) -else: - st.success("Policy drift status for monitored sources: OK (based on latest snapshots/log).") - st.session_state.ack_policy_drift = True - -policy_gate_block = any_review_required and (not st.session_state.get("ack_policy_drift", False)) - - -# ---------------------------- -# Featured showcase cases -# ---------------------------- -showcase_submitted = False -showcase_feedback = None - -st.subheader("Featured Showcase Cases") -st.caption( - "Fastest way to experience the demo. Choose a curated synthetic case to load it into the main intake below. " - "Loaded inputs stay editable for custom exploration." -) - -if featured_showcase_cases: - for start in range(0, len(featured_showcase_cases), 2): - cols = st.columns(2) - for col, case in zip(cols, featured_showcase_cases[start : start + 2]): - showcase = case.get("showcase") or {} - title = showcase.get("title", case.get("id", "Showcase case")) - description = showcase.get("description", "") - expected_status = showcase.get("expected_overall_status", "UNKNOWN") - why_interesting = showcase.get("why_interesting", "") - - with col: - st.markdown(f"#### {title}") - st.write(description) - st.markdown(f"**Expected outcome:** {_showcase_status_chip(expected_status)}") - if why_interesting: - st.caption(f"Why this case is useful: {why_interesting}") - if st.session_state.get("selected_showcase_case_id") == case.get("id"): - st.caption("Currently loaded in the intake below.") - - if st.button("Open This Demo Case", key=f"showcase_{case.get('id')}", width="stretch"): - _load_case_into_intake(case) - if tests_healthy and not policy_gate_block: - showcase_submitted = True - showcase_feedback = ( - "success", - f'Loaded "{title}" and ran the evaluation below. The intake remains fully editable.', - ) - elif policy_gate_block: - showcase_feedback = ( - "info", - f'Loaded "{title}" into the intake below. Acknowledge the policy drift gate to run the evaluation.', - ) - else: - showcase_feedback = ( - "info", - f'Loaded "{title}" into the intake below. Resolve the synthetic eval gate before running the evaluation.', - ) -else: - st.info("No featured showcase cases configured.") - -if showcase_feedback: - getattr(st, showcase_feedback[0])(showcase_feedback[1]) - - -# ---------------------------- -# Intake form -# ---------------------------- -st.markdown("### Intake") -current_showcase_case = next( - (case for case in featured_showcase_cases if case.get("id") == st.session_state.get("selected_showcase_case_id")), - None, -) -if current_showcase_case is not None: - st.caption( - f'Loaded showcase case: {(current_showcase_case.get("showcase") or {}).get("title", current_showcase_case.get("id"))}. ' - "You can edit any field below or replace the note for custom exploration." + st.session_state["ack_policy_drift"] = st.checkbox( + "I acknowledge governance issues may make related demo outputs stale.", + value=st.session_state["ack_policy_drift"], ) else: - st.caption("Paste your own synthetic note here, or start with one of the featured showcase cases above.") - -with st.form("pa_form", clear_on_submit=False): - c1, c2, c3 = st.columns(3) - - with c1: - payer = st.selectbox("Payer", payers, key="intake_payer") - - with c2: - procedures = rules["payers"][payer]["procedures"] - proc_code = st.selectbox("Procedure", list(procedures.keys()), key="intake_proc_code") - - with c3: - dx_raw = st.text_input("Dx codes (comma-separated)", placeholder="e.g., M54.5, M51.26", key="intake_dx_raw") - - specialty = st.text_input("Ordering specialty (optional)", placeholder="e.g., Orthopedics", key="intake_specialty") - site = st.selectbox("Site of care", SITE_OPTIONS, key="intake_site") - - note_text = st.text_area( - "Clinical note (mock/synthetic)", - height=220, - placeholder="Paste a synthetic clinical note here...", - key="intake_note_text", - ) + st.success("No monitored-source drift or stale/missing baselines currently require review.") + st.session_state["ack_policy_drift"] = True - tests_healthy = bool(st.session_state.get("tests_healthy", False)) - submitted = st.form_submit_button( - "Evaluate PA readiness", - disabled=(not tests_healthy) or policy_gate_block, +if drift_report.stale_source_count: + st.warning( + f"{drift_report.stale_source_count} monitored source(s) are stale relative to the configured check frequency. " + "This is a governance signal only; it does not auto-change rules." ) - -# ---------------------------- -# Evaluate action (persist results) -# ---------------------------- -if submitted or showcase_submitted: - # Clear prior letter UI output on new eval to avoid stale drafts - st.session_state.letter_text = "" - st.session_state.letter_meta = {} - st.session_state.letter_error = "" - - dx_codes = [x.strip() for x in (dx_raw or "").split(",") if x.strip()] - proc_obj = rules["payers"][payer]["procedures"][proc_code] - proc_name = proc_obj.get("display_name", proc_code) - requirements = proc_obj.get("required", []) - - facts, evidence_map = extract_facts(note_text) - results, reasons = evaluate_requirements(requirements, facts, evidence_map=evidence_map) - - overall = compute_overall_status(results) - score_info = compute_readiness_score(results) - - rows = [] - for rr in results: - rows.append( - { - "key": rr.key, - "label": rr.label, - "status": rr.status, - "reason": rr.reason, - "evidence_hint": rr.evidence or "", - "evidence_snippets": getattr(rr, "evidence_snippets", []) or [], - } - ) - - # Blocking issues - blocking_not_documented = [{"key": r["key"], "label": r["label"]} for r in rows if r["status"] == "NOT_DOCUMENTED"] - blocking_not_met = [{"key": r["key"], "label": r["label"]} for r in rows if r["status"] == "NOT_MET"] - - # Invariants - invariant_errors = [] - if blocking_not_documented and overall["overall_status"] != "CANNOT_DETERMINE": - invariant_errors.append("Invariant violation: NOT_DOCUMENTED blockers exist but overall_status is not CANNOT_DETERMINE.") - if (not blocking_not_documented) and blocking_not_met and overall["overall_status"] == "READY": - invariant_errors.append("Invariant violation: NOT_MET blockers exist but overall_status is READY.") - if (not blocking_not_documented) and (not blocking_not_met) and overall["overall_status"] != "READY": - invariant_errors.append("Invariant violation: no blockers exist but overall_status is not READY.") - - # Provenance snapshot + trust level - prov_info = get_provenance_entry(prov, payer, proc_code) - policy_trust_level = policy_trust_from_provenance(prov_info) - - # Metrics - metrics = _compute_metrics(score_info) - - run_id = str(uuid.uuid4()) - ts = datetime.now(timezone.utc).isoformat() - - audit = { - "run_id": run_id, - "timestamp_utc": ts, - "note_hash": _hash_note(note_text), - "note_length": len(note_text or ""), - "payer": payer, - "procedure_code": proc_code, - "procedure_name": proc_name, - "site_of_care": site, - "specialty": specialty, - "rules_version": rules.get("version"), - "policy_trust_level": policy_trust_level, - "provenance_snapshot": prov_info or {}, - "facts_extracted": facts, - "evidence_map": evidence_map, - "requirements_checked": [r["key"] for r in rows], - "overall_status": overall["overall_status"], - "submission_readiness": bool(overall["submission_readiness"]), - "blocking_issues": {"not_documented": blocking_not_documented, "not_met": blocking_not_met}, - "metrics": metrics, - "invariant_errors": invariant_errors, +policy_gate_block = drift_report.any_review_required and not st.session_state["ack_policy_drift"] +if not tests_healthy: + st.error("Evaluation is gated because the bundled synthetic regression suite is not fully green.") + +st.subheader("Rulebook Governance") +st.caption("Versioned reviewed and active snapshots make rule promotion inspectable. Monitoring never promotes rules automatically.") +rulebook_rows = [ + { + "release_id": release.release_id, + "stage": release.stage or "unassigned", + "rules_version": release.rules_version or "n/a", + "procedures": len(release.procedures), + "reviewed_at": release.reviewed_at or "n/a", + "runtime_match": "Yes" if release.runtime_matches else ("No" if release.runtime_matches is False else "n/a"), } + for release in rulebook_status.releases +] +st.dataframe(rulebook_rows, width="stretch") +if rulebook_status.validation_errors: + st.error("Rulebook validation errors detected.") + for item in rulebook_status.validation_errors: + st.write(f"- {item}") +else: + st.success(f"Active rulebook release: {rulebook_status.active_release_id or 'n/a'}") - # Build schema objects for write-only letter drafting - dx_codes_clean = normalized_dx_codes(dx_codes) +with st.expander("Promotion workflow", expanded=False): + st.write("- Draft: candidate snapshot awaiting human review.") + st.write("- Reviewed: validated snapshot kept for comparison and audit.") + st.write("- Active: runtime rulebook intentionally promoted by a human. Drift monitoring never auto-promotes.") - pa_model = PARequest( - payer=payer, - procedure_code=proc_code, - dx_codes=dx_codes_clean, - site_of_care=site, - specialty=(specialty or "unknown"), - note_text="", # intentionally NOT used by drafting - ) +st.subheader("Supported Procedure Registry") +st.caption("Compact view of the current deterministic scope, rule family, provenance label, and drift coverage.") +st.dataframe(registry_rows, width="stretch") +featured_cases = featured_demo_cases(config) - req_models = [ - RequirementResult( - key=r["key"], - label=r["label"], - status=r["status"], - reason=r["reason"], - evidence=(r.get("evidence_hint") or None), - evidence_snippets=(r.get("evidence_snippets") or []), - ) - for r in rows - ] - - report_model = ReadinessReport( - readiness_score=int(score_info.get("readiness_score", 0) or 0), - not_documented_count=int(score_info.get("not_documented_count", 0) or 0), - not_met_count=int(score_info.get("not_met_count", 0) or 0), - met_count=int(score_info.get("met_count", 0) or 0), - results=req_models, - rule_reasons=list(reasons or []), - audit_trail=dict(audit), - letter_draft="", # write-only; UI will populate separately - ) +st.subheader("Featured Demo Cases") +st.caption("Seeded examples for live demos. Each one remains fully editable after loading.") +showcase_submitted = False +showcase_message = None + +for start in range(0, len(featured_cases), 2): + cols = st.columns(2) + for col, case in zip(cols, featured_cases[start : start + 2]): + with col: + st.markdown(f"#### {case.showcase.get('title', case.id)}") + st.write(case.showcase.get("description", "Synthetic demo case.")) + expected_status = expected_overall_status_for_demo_case(case) + if expected_status: + st.caption(f"Expected overall status: {expected_status}") + elif case.expected_label: + st.caption(f"Fixture label: {case.expected_label}") + if case.showcase.get("scenario_type"): + st.caption(f"Scenario: {case.showcase.get('scenario_type')}") + if case.showcase.get("tags"): + st.caption(f"Tags: {', '.join(case.showcase.get('tags', []))}") + st.caption(case.showcase.get("why_interesting", "")) + if st.button("Load Demo Case", key=f"case_{case.id}", width="stretch"): + load_case_into_session(case.model_dump(mode="json")) + showcase_submitted = tests_healthy and not policy_gate_block + if showcase_submitted: + showcase_message = ( + "success", + f'Loaded "{case.showcase.get("title", case.id)}" and ran the evaluation.', + ) + elif policy_gate_block: + showcase_message = ( + "info", + f'Loaded "{case.showcase.get("title", case.id)}". Acknowledge the drift gate to run it.', + ) + else: + showcase_message = ( + "info", + f'Loaded "{case.showcase.get("title", case.id)}". Resolve the synthetic eval gate to run it.', + ) - st.session_state.last_eval = { - "payer": payer, - "proc_code": proc_code, - "proc_name": proc_name, - "dx_codes": dx_codes, - "facts": facts, - "evidence_map": evidence_map, - "rows": rows, - "reasons": reasons, - "overall": overall, - "score_info": score_info, - "metrics": metrics, - "audit": audit, - "invariant_errors": invariant_errors, - "policy_trust_level": policy_trust_level, - "provenance": prov_info or {}, - # NEW - "pa_model": pa_model, - "report_model": report_model, - } +if showcase_message: + getattr(st, showcase_message[0])(showcase_message[1]) -# ---------------------------- -# Outputs -# ---------------------------- -st.markdown("### Results") +st.subheader("Evaluate Request") +left, right = st.columns([1.5, 1]) -if st.session_state.last_eval is None: - st.info("Run an evaluation to see the result. The bundled synthetic evaluation suite remains available below.") -else: - ev = st.session_state.last_eval - overall = ev["overall"] - score_info = ev["score_info"] - metrics = ev["metrics"] - - status = overall["overall_status"] - submission_readiness = bool(overall["submission_readiness"]) - inv = ev.get("invariant_errors", []) - not_documented_items = [r for r in ev["rows"] if r["status"] == "NOT_DOCUMENTED"] - not_met_items = [r for r in ev["rows"] if r["status"] == "NOT_MET"] - total_blockers = len(not_documented_items) + len(not_met_items) - source_name = ev["provenance"].get("source_name") or "Not documented" - last_reviewed = ev["provenance"].get("last_reviewed") or "Not documented" - - # Decision first - if status == "CANNOT_DETERMINE": - st.warning( - "Administrative readiness **cannot be determined** — one or more required criteria are **not documented** in the note. " - "Add explicit documentation for the blocking items below." +with left: + with st.form("evaluation_form", clear_on_submit=False): + payer = st.selectbox("Payer", options=payers, key="payer") + procedure_options = procedures_by_payer[payer] + default_index = next( + (index for index, item in enumerate(procedure_options) if item.procedure_code == st.session_state.get("procedure_code")), + 0, ) - elif status == "NOT_READY": - st.error( - "Not ready to submit — one or more required criteria are **documented but not met**. " - "Review the failing items below." + selected_procedure = st.selectbox( + "Procedure", + options=procedure_options, + index=default_index, + format_func=lambda item: f"{item.procedure_code} | {item.display_name}", + key="procedure_selectbox", ) - elif status == "READY": - st.success( - "Administratively ready **per current rules** — all required criteria appear documented and met. " - "Human review still required." + st.session_state["procedure_code"] = selected_procedure.procedure_code + + st.text_input("Diagnosis codes (comma-separated)", key="dx_codes", placeholder="e.g., M54.16, G47.33") + st.selectbox("Site of care", options=config.allowed_sites, key="site_of_care") + st.text_input("Ordering specialty", key="specialty", placeholder="e.g., Orthopedics") + st.text_area( + "Synthetic note text", + key="note_text", + height=220, + placeholder="Paste or edit a synthetic note here.", ) - else: - st.info("Status unavailable (unexpected overall_status).") - if ev["policy_trust_level"] != "verified": - st.warning( - "Demo rules in use — requirements are manually curated for demonstration. " - "Verify against the official payer policy before any real submission." + submitted = st.form_submit_button( + "Run deterministic readiness review", + disabled=(not tests_healthy) or policy_gate_block, ) + +with right: + current_supported = service.get_supported_procedure( + st.session_state["payer"], + st.session_state["procedure_code"], + ) + st.markdown("#### Current Rule Summary") + st.caption(f"Category: {current_supported.metadata.category}") + st.caption(f"Rule family: {current_supported.metadata.rule_family}") + st.caption(f"Trust level: {current_supported.policy_trust_level.upper()}") + st.caption(f"Rule source: {current_supported.provenance.rule_source_label or current_supported.provenance.source_name or 'n/a'}") st.caption( - f"Policy source: {source_name} | Last reviewed: {last_reviewed} | Trust level: {str(ev['policy_trust_level']).upper()}" + f"Rule last updated: {current_supported.metadata.last_rule_update or 'n/a'} | " + f"Last reviewed: {current_supported.provenance.last_reviewed or 'n/a'}" ) + st.caption( + f"Drift monitoring: {'Configured' if current_supported.monitored_for_drift else 'Not configured for this procedure'}" + ) + if current_supported.monitored_for_drift: + st.caption( + f"Monitored source: {current_supported.provenance.monitored_source_name or current_supported.provenance.monitored_source_id}" + ) + for requirement in current_supported.requirements: + requirement_line = f"- {requirement.label} ({requirement.type})" + if requirement.min is not None: + requirement_line += f" | min={requirement.min:g}" + if requirement.allowed: + requirement_line += f" | allowed={', '.join(requirement.allowed)}" + st.write(requirement_line) + if current_supported.metadata.notes: + with st.expander("Rule notes", expanded=False): + for note in current_supported.metadata.notes: + st.write(f"- {note}") + with st.expander("Scope and limitations", expanded=False): + st.write("- Synthetic demo inputs only") + st.write("- Deterministic rule evaluation only") + st.write("- No approval prediction") + st.write("- No medical-necessity or clinical recommendation logic") + st.write("- Human review remains required before any real submission") + + +should_run = submitted or showcase_submitted +if should_run: + st.session_state["letter_text"] = "" + st.session_state["letter_meta"] = {} + try: + evaluation = service.evaluate(current_request()) + st.session_state["last_eval_payload"] = evaluation.model_dump(mode="json") + except ServiceError as exc: + st.error(str(exc)) + st.session_state["last_eval_payload"] = None - if inv: - st.error("Internal consistency checks require review before trusting this run.") - for msg in inv: - st.write(f"- {msg}") - - # Blocking items and reasons - st.subheader("What Needs Attention") - blocker_cols = st.columns(3) - with blocker_cols[0]: - st.metric("Total Blockers", total_blockers) - with blocker_cols[1]: - st.metric("Missing Requirements", len(not_documented_items)) - with blocker_cols[2]: - st.metric("Documented Failures", len(not_met_items)) - - if not not_documented_items and not not_met_items: - st.success("No blocking items detected under the current rules.") - else: - if not_documented_items: - st.markdown("**Missing documentation (drives `CANNOT_DETERMINE`):**") - for r in not_documented_items: - st.write(f"- {r['label']}: {r['reason']}") - if not_met_items: - st.markdown("**Documented but below threshold (drives `NOT_READY`):**") - for r in not_met_items: - st.write(f"- {r['label']}: {r['reason']}") - - # Requirement-by-requirement explanation - st.subheader("Why This Result") - st.caption("Each requirement shows its status, the reason it landed there, and supporting note text when available.") - status_emoji = {"MET": "✅", "NOT_MET": "⚠️", "NOT_DOCUMENTED": "❌"} - - for r in ev["rows"]: - emoji = status_emoji.get(r.get("status"), "❓") - expand_default = r.get("status") != "MET" - - with st.expander(f"{emoji} {r.get('label', '')}", expanded=expand_default): - st.write(f"**Status:** {r.get('status')}") - st.write(f"**Reason:** {r.get('reason')}") - - if r.get("evidence_hint"): - st.info(f"💡 **What to look for in the note:** {r['evidence_hint']}") - - snips = r.get("evidence_snippets") or [] - if snips: - st.markdown("**Evidence found in note:**") - for s in snips[:5]: - st.code(str(s), language="text") - else: - st.caption("No evidence snippet captured for this requirement.") - - st.subheader("Extracted Facts") - st.caption("Decision-relevant facts pulled from the note for this request.") - for start in range(0, len(ev["rows"]), 2): - cols = st.columns(2) - for col, r in zip(cols, ev["rows"][start : start + 2]): - key = r["key"] - fact_value = _format_extracted_fact_value(key, ev["facts"].get(key)) - span_count = len(ev["evidence_map"].get(key) or []) - - with col: - st.markdown(f"**{status_emoji.get(r.get('status'), '❓')} {r.get('label', '')}**") - st.write(fact_value) - if r["status"] == "NOT_DOCUMENTED": - st.caption("Missing from the note or not explicit enough for deterministic extraction.") - elif r["status"] == "NOT_MET": - st.caption("Captured from the note, but below the current rule threshold.") - elif span_count > 1: - st.caption("Captured from multiple note excerpts.") - else: - st.caption("Captured from the note.") - - st.subheader("Evidence Mapping") - st.caption("Compact note excerpts associated with the extracted facts above.") - for start in range(0, len(ev["rows"]), 2): - cols = st.columns(2) - for col, r in zip(cols, ev["rows"][start : start + 2]): - key = r["key"] - spans = ev["evidence_map"].get(key) or [] - - with col: - st.markdown(f"**{r.get('label', '')}**") - if spans: - st.caption(f"{len(spans)} supporting note excerpt(s) shown.") - for span in spans[:2]: - st.code(str(span.get("text", "")).strip(), language="text") - st.caption(f"Excerpt location: {span.get('start')}-{span.get('end')}") - if len(spans) > 2: - st.caption(f"+ {len(spans) - 2} more note excerpt(s) available in the raw details below.") - elif r["status"] == "NOT_DOCUMENTED": - st.caption("No supporting note excerpt was captured because the fact was missing or not explicit enough.") - else: - st.caption("No supporting note excerpt was captured for this fact.") - - _render_audit_summary_card(ev["audit"], ev["provenance"], inv) - - st.subheader("Secondary Diagnostics (Informational)") - st.caption("These convenience metrics support review but do not change the frozen status or blockers above.") - o1, o2, o3 = st.columns([1, 1, 1]) +st.subheader("Results") +if not st.session_state["last_eval_payload"]: + st.info("Run a demo case or submit synthetic input to inspect deterministic readiness results.") +else: + evaluation = EvaluationResult.model_validate(st.session_state["last_eval_payload"]) + status_panel(evaluation) - with o1: - st.metric("Score (informational only)", f"{score_info['readiness_score']}/100") - st.caption( - f"{score_info['met_count']} met | {score_info['not_documented_count']} missing | " - f"{score_info['not_met_count']} below threshold out of {score_info['total']} requirements." + if evaluation.policy_trust_level != "verified": + st.warning( + "This procedure currently uses DEMO trust. " + "The rule logic is still deterministic, but provenance remains curated for demonstration." ) - st.caption(f"Administratively ready under current demo rules: {'Yes' if submission_readiness else 'No'}") - with o2: - extraction_delta = ( - "0 missing" - if metrics["extraction_failure_count"] == 0 - else f"-{metrics['extraction_failure_count']} missing" + if evaluation.warnings: + with st.expander("Evaluation warnings", expanded=True): + for warning in evaluation.warnings: + st.write(f"- {warning}") + + metric_cols = st.columns(4) + with metric_cols[0]: + st.metric("Overall status", evaluation.overall_status) + with metric_cols[1]: + st.metric("Readiness score", f"{evaluation.readiness_score}/100") + with metric_cols[2]: + st.metric("Missing requirements", len(evaluation.blockers.not_documented)) + with metric_cols[3]: + st.metric("Documented failures", len(evaluation.blockers.not_met)) + + tabs = st.tabs(["Overview", "Requirement Reasoning", "Facts and Evidence", "Audit and Export"]) + + with tabs[0]: + st.markdown("#### Blockers") + if not evaluation.blockers.not_documented and not evaluation.blockers.not_met: + st.success("No blockers detected under the current rules.") + else: + if evaluation.blockers.not_documented: + st.markdown("**Missing documentation**") + for blocker in evaluation.blockers.not_documented: + st.write(f"- {blocker.label}: {blocker.reason}") + if evaluation.blockers.not_met: + st.markdown("**Documented but below threshold**") + for blocker in evaluation.blockers.not_met: + st.write(f"- {blocker.label}: {blocker.reason}") + + st.markdown("#### Procedure metadata") + rule_source_label = ( + evaluation.supported_procedure.provenance.rule_source_label + or evaluation.supported_procedure.provenance.source_name + or "n/a" ) - st.metric( - "Extraction Success", - f"{metrics['extraction_success_rate']}%", - delta=extraction_delta, + monitored_source_label = ( + evaluation.supported_procedure.provenance.monitored_source_name + or evaluation.supported_procedure.provenance.monitored_source_id + or "n/a" ) - st.caption("Higher missing counts usually reflect documentation gaps, not hidden inference.") - - with o3: - cr = metrics["compliance_rate"] - st.metric( - "Requirement Compliance", - f"{cr}%" if cr is not None else "N/A", - delta=f"{metrics['non_compliant_count']} below threshold", + st.write(f"- Payer: {evaluation.request.payer}") + st.write(f"- Procedure: {evaluation.request.procedure_code} ({evaluation.supported_procedure.display_name})") + st.write(f"- Category: {evaluation.supported_procedure.metadata.category}") + st.write(f"- Rule family: {evaluation.supported_procedure.metadata.rule_family}") + st.write(f"- Site of care: {evaluation.request.site_of_care}") + st.write(f"- Specialty: {evaluation.request.specialty}") + st.write(f"- Policy trust level: {evaluation.policy_trust_level.upper()}") + st.write(f"- Required field keys: {', '.join(evaluation.supported_procedure.required_field_keys)}") + st.write(f"- Rule source: {rule_source_label}") + st.write(f"- Last rule update: {evaluation.supported_procedure.metadata.last_rule_update or 'n/a'}") + st.write(f"- Last reviewed: {evaluation.supported_procedure.provenance.last_reviewed or 'n/a'}") + if evaluation.supported_procedure.monitored_for_drift: + st.write(f"- Monitored source: {monitored_source_label}") + + with tabs[1]: + st.caption("Requirement-level reasoning stays deterministic and traceable.") + for result in evaluation.results: + render_requirement_result(result) + + with tabs[2]: + st.markdown("#### Extracted facts") + for start in range(0, len(evaluation.results), 2): + cols = st.columns(2) + for col, result in zip(cols, evaluation.results[start : start + 2]): + with col: + render_fact_card(result.label, evaluation.facts.get(result.key), result.status) + + st.markdown("#### Evidence map") + for result in evaluation.results: + with st.expander(result.label, expanded=False): + spans = evaluation.evidence_map.get(result.key, []) + if spans: + for span in spans: + st.code(span.text, language="text") + st.caption(f"Character offsets: {span.start}-{span.end}") + else: + st.caption("No explicit evidence span was captured for this requirement.") + + with tabs[3]: + st.markdown("#### Audit summary") + audit_cols = st.columns(4) + with audit_cols[0]: + st.metric("Run ID", evaluation.audit_trail.run_id[:8]) + with audit_cols[1]: + st.metric("Note hash", evaluation.audit_trail.note_hash) + with audit_cols[2]: + st.metric("Rules version", evaluation.audit_trail.rules_version or "n/a") + with audit_cols[3]: + st.metric("Submission ready", "YES" if evaluation.submission_readiness else "NO") + + if evaluation.audit_trail.invariant_errors: + st.error("Invariant checks require review before trusting this run.") + for item in evaluation.audit_trail.invariant_errors: + st.write(f"- {item}") + + with st.expander("Structured provenance", expanded=False): + st.json(evaluation.provenance) + + letter_type = st.selectbox( + "Letter type", + options=["submission_cover_letter", "missing_info_request", "appeal_template"], + key="letter_type", ) - st.caption("Compliance is calculated only from documented requirements and remains secondary to the frozen status contract.") - - # ---------------------------- - # Letter Drafting UI (Write-only) - # ---------------------------- - st.subheader("Justification Letter (Write-only)") - - # Letter type selector (presentation only; does not change readiness logic) - letter_type = st.selectbox( - "Letter type", - ["submission_cover_letter", "missing_info_request", "appeal_template"], - index=0, - ) - - cA, cB = st.columns([1, 1]) - with cA: - generate_letter = st.button("Generate letter draft", type="primary", width="stretch") - with cB: - clear_letter = st.button("Clear draft", width="stretch") - - if clear_letter: - st.session_state.letter_text = "" - st.session_state.letter_meta = {} - st.session_state.letter_error = "" - - if generate_letter: - try: - pa_model: PARequest = ev["pa_model"] - report_model: ReadinessReport = ev["report_model"] - - letter_text, letter_meta = draft_letter_writeonly( - pa_model, - report_model, - letter_type=letter_type, - policy_trust_level=ev.get("policy_trust_level", "demo"), - ) - - st.session_state.letter_text = letter_text - st.session_state.letter_meta = letter_meta - st.session_state.letter_error = "" - - # Audit linkage without storing full letter content - ev["audit"]["letter_artifacts"] = { - "letter_type": letter_meta.get("letter_type"), - "letter_version": letter_meta.get("letter_version"), - "generated_timestamp_utc": letter_meta.get("generated_timestamp_utc"), - "letter_hash_sha256_16": letter_meta.get("letter_hash_sha256_16"), - "cited_snippets_count": letter_meta.get("cited_snippets_count"), - "overall_status": letter_meta.get("overall_status"), - "policy_trust_level": letter_meta.get("policy_trust_level"), - "draft_blocked": letter_meta.get("draft_blocked"), - } - - except Exception as e: - st.session_state.letter_error = f"{type(e).__name__}: {e}" - - if st.session_state.letter_error: - st.error(st.session_state.letter_error) - - if st.session_state.letter_text: - st.text_area("Letter draft (read-only)", value=st.session_state.letter_text, height=300) - - with st.expander("Letter metadata"): - st.code(json.dumps(st.session_state.letter_meta, indent=2), language="json") - - st.download_button( - "📥 Download Letter (.txt)", - data=st.session_state.letter_text, - file_name=f"pa_{letter_type}.txt", - mime="text/plain", - width="stretch", + letter_cols = st.columns([1, 1]) + with letter_cols[0]: + if st.button("Generate deterministic letter", width="stretch"): + letter_text, letter_meta = service.generate_letter(evaluation, letter_type=letter_type) + st.session_state["letter_text"] = letter_text + st.session_state["letter_meta"] = letter_meta + with letter_cols[1]: + if st.button("Clear letter", width="stretch"): + st.session_state["letter_text"] = "" + st.session_state["letter_meta"] = {} + + if st.session_state["letter_text"]: + st.markdown("**Letter draft**") + st.text_area("Deterministic administrative letter", value=st.session_state["letter_text"], height=300) + st.json(st.session_state["letter_meta"]) + + export_payload = export_evaluation_payload( + evaluation, + letter_text=st.session_state.get("letter_text") or None, + letter_meta=st.session_state.get("letter_meta") or None, ) st.download_button( - "📥 Download Letter Metadata (.json)", - data=json.dumps(st.session_state.letter_meta, indent=2), - file_name=f"pa_{letter_type}_metadata.json", + "Download JSON artifact", + data=json.dumps(export_payload, indent=2, sort_keys=True), + file_name=f"{evaluation.request.payer.lower()}_{evaluation.request.procedure_code.lower()}_{evaluation.audit_trail.run_id[:8]}.json", mime="application/json", - width="stretch", ) - else: - st.caption("No letter generated yet. Select a letter type and click **Generate letter draft**.") - - # Full audit/debug details - st.subheader("Full Audit & Debug Details") - with st.expander("Open raw audit JSON, extracted facts, and evidence spans"): - st.json(ev["audit"]) - if ev["reasons"]: - st.markdown("**Rule reasons**") - st.code("\n".join(ev["reasons"]), language="text") - _render_key_value_block("Extracted facts (raw)", ev["facts"]) - _render_evidence_map_block("Evidence map (raw spans)", ev["evidence_map"]) - - audit_json = json.dumps(ev["audit"], indent=2) - ts_local = datetime.now().strftime("%Y%m%d_%H%M%S") - st.download_button( - label="📥 Download Audit Trail (JSON)", - data=audit_json, - file_name=f"pa_audit_{ev['audit']['payer']}_{ev['audit']['procedure_code']}_{ts_local}.json", - mime="application/json", - width="stretch", - ) - - -# ---------------------------- -# Synthetic evaluation suite (manual run + export + inspect) -# ---------------------------- -st.markdown("### Advanced: Synthetic Evaluation Suite") - -run_tests = st.button("Run synthetic eval suite", width="stretch") - -if run_tests: - from engine.test_suites import run_cases - st.session_state.test_rows = run_cases("rules/payer_rules.yaml", "inputs/synthetic_cases.json") -if st.session_state.test_rows is None: - st.caption("Click **Run synthetic eval suite** to evaluate the rules engine on bundled synthetic cases.") -else: - st.dataframe(st.session_state.test_rows, width="stretch") - - test_json = json.dumps(st.session_state.test_rows, indent=2) - ts_local = datetime.now().strftime("%Y%m%d_%H%M%S") - st.download_button( - label="📥 Download Synthetic Eval Results (JSON)", - data=test_json, - file_name=f"pa_test_results_{ts_local}.json", - mime="application/json", - width="stretch", - ) - - st.markdown("---") - st.subheader("🔎 Inspect a Synthetic Case") - - # Load the raw cases so we can re-run extraction/eval for one selected case - _cases = synthetic_cases - - case_ids = [c.get("id") for c in _cases] - selected_id = st.selectbox("Select case", case_ids) - - case = next((c for c in _cases if c.get("id") == selected_id), None) - if case: - payer_i = case["payer"] - proc_i = case["procedure_code"] - note_i = case.get("note_text", "") - - proc_obj_i = rules["payers"][payer_i]["procedures"][proc_i] - reqs_i = proc_obj_i.get("required", []) - - facts_i, evidence_map_i = extract_facts(note_i) - results_i, _ = evaluate_requirements(reqs_i, facts_i, evidence_map=evidence_map_i) - - # Build rows including evidence_snippets - rows_i = [] - for rr in results_i: - rows_i.append( - { - "key": rr.key, - "label": rr.label, - "status": rr.status, - "reason": rr.reason, - "evidence_hint": rr.evidence or "", - "evidence_snippets": getattr(rr, "evidence_snippets", []) or [], - } - ) - - st.markdown("**Synthetic case note:**") - st.text_area("note_text", value=note_i, height=160) - - _render_key_value_block("Extracted facts", facts_i) - _render_evidence_map_block("Evidence map (raw spans)", evidence_map_i) - - st.markdown("**Explainable requirement results:**") - status_emoji = {"MET": "✅", "NOT_MET": "⚠️", "NOT_DOCUMENTED": "❌"} - for r in rows_i: - emoji = status_emoji.get(r.get("status"), "❓") - expand_default = r.get("status") != "MET" - with st.expander(f"{emoji} {r.get('label','')}", expanded=expand_default): - st.write(f"**Status:** {r.get('status')}") - st.write(f"**Reason:** {r.get('reason')}") - if r.get("evidence_hint"): - st.info(f"💡 **What to look for in the note:** {r['evidence_hint']}") - - snips = r.get("evidence_snippets") or [] - if snips: - st.markdown("**Evidence found in note:**") - for s in snips[:5]: - st.code(str(s), language="text") - else: - st.caption("No evidence snippet captured for this requirement.") + with st.expander("Raw evaluation payload", expanded=False): + st.json(export_payload) diff --git a/cli.py b/cli.py new file mode 100644 index 0000000..ac6b2f8 --- /dev/null +++ b/cli.py @@ -0,0 +1,187 @@ +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Sequence + +from engine.demo_cases import expected_overall_status_for_demo_case +from engine.rendering import ( + export_evaluation_payload, + render_cli_evaluation, + render_drift_status, + render_rulebook_diff, + render_rulebook_status, + write_json_artifact, +) +from engine.service import ReadinessService, ServiceError + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="pa-copilot", + description=( + "Deterministic prior authorization readiness review for synthetic demo cases. " + "Administrative readiness only; no clinical judgment or approval prediction." + ), + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + subparsers.add_parser("status", help="Show repo status and supported scope.") + + list_procedures = subparsers.add_parser("list-procedures", help="List supported payer/procedure combinations.") + list_procedures.add_argument("--json", action="store_true", help="Emit JSON instead of a text table.") + + list_demo_cases = subparsers.add_parser("list-demo-cases", help="List bundled synthetic demo cases.") + list_demo_cases.add_argument("--json", action="store_true", help="Emit JSON instead of a text table.") + + evaluate = subparsers.add_parser("evaluate", help="Evaluate one bundled synthetic demo case.") + evaluate.add_argument("--demo-case", required=True, help="Case ID from list-demo-cases.") + evaluate.add_argument("--json", action="store_true", help="Emit JSON instead of a text summary.") + + export = subparsers.add_parser("export-report", help="Export a stable JSON artifact for one demo case.") + export.add_argument("--demo-case", required=True, help="Case ID from list-demo-cases.") + export.add_argument("--output", required=True, help="Output path for the JSON artifact.") + export.add_argument("--with-letter", action="store_true", help="Include a letter draft in the exported artifact.") + export.add_argument( + "--letter-type", + default="submission_cover_letter", + choices=["submission_cover_letter", "missing_info_request", "appeal_template"], + help="Letter type used when --with-letter is set.", + ) + + drift = subparsers.add_parser("drift-status", help="Show current governance-only drift status.") + drift.add_argument("--json", action="store_true", help="Emit JSON instead of a text summary.") + + rulebook_status = subparsers.add_parser("rulebook-status", help="Show current rulebook registry and validation status.") + rulebook_status.add_argument("--json", action="store_true", help="Emit JSON instead of a text summary.") + + rulebook_diff = subparsers.add_parser("rulebook-diff", help="Diff two rulebook releases.") + rulebook_diff.add_argument("--from-release", required=True, help="Source release ID from rulebook/manifest.yaml.") + rulebook_diff.add_argument("--to-release", required=True, help="Target release ID from rulebook/manifest.yaml.") + rulebook_diff.add_argument("--json", action="store_true", help="Emit JSON instead of a text summary.") + + validate = subparsers.add_parser("validate-demo-case", help="Validate a bundled synthetic input before evaluation.") + validate.add_argument("--demo-case", required=True, help="Case ID from list-demo-cases.") + + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + service = ReadinessService() + + try: + if args.command == "status": + print(json.dumps(service.get_status().model_dump(mode="json"), indent=2, sort_keys=True)) + return 0 + + if args.command == "list-procedures": + procedures = service.list_supported_procedures() + if args.json: + print(json.dumps([item.model_dump(mode="json") for item in procedures], indent=2, sort_keys=True)) + else: + for item in procedures: + monitored = "yes" if item.monitored_for_drift else "no" + source_label = item.provenance.rule_source_label or item.provenance.source_name or "n/a" + print( + f"{item.payer}\t{item.procedure_code}\t{item.display_name}\t" + f"category={item.metadata.category}\tfamily={item.metadata.rule_family}\t" + f"trust={item.policy_trust_level}\tdrift_monitored={monitored}\t" + f"last_update={item.metadata.last_rule_update or 'n/a'}\tsource={source_label}" + ) + return 0 + + if args.command == "list-demo-cases": + demo_cases = service.list_demo_case_summaries() + if args.json: + print(json.dumps([item.model_dump(mode="json") for item in demo_cases], indent=2, sort_keys=True)) + else: + for case in demo_cases: + title = case.showcase.get("title") or case.id + scenario_type = case.showcase.get("scenario_type") or "standard" + expectation_parts = [] + expected_status = expected_overall_status_for_demo_case(case) + if expected_status: + expectation_parts.append(f"expected_status={expected_status}") + if case.expected_label: + expectation_parts.append(f"fixture_label={case.expected_label}") + expectation = "\t".join(expectation_parts) if expectation_parts else "expectation=n/a" + print(f"{case.id}\t{case.payer}\t{case.procedure_code}\t{expectation}\t{scenario_type}\t{title}") + return 0 + + if args.command == "evaluate": + request = service.get_demo_case_request(args.demo_case) + evaluation = service.evaluate(request) + if args.json: + print(json.dumps(export_evaluation_payload(evaluation), indent=2, sort_keys=True)) + else: + print(render_cli_evaluation(evaluation)) + return 0 + + if args.command == "export-report": + request = service.get_demo_case_request(args.demo_case) + evaluation = service.evaluate(request) + letter_text = None + letter_meta = None + if args.with_letter: + letter_text, letter_meta = service.generate_letter(evaluation, letter_type=args.letter_type) + artifact = export_evaluation_payload(evaluation, letter_text=letter_text, letter_meta=letter_meta) + output_path = write_json_artifact(artifact, Path(args.output)) + print(output_path) + return 0 + + if args.command == "drift-status": + report = service.get_drift_status() + if args.json: + print(json.dumps(report.model_dump(mode="json"), indent=2, sort_keys=True)) + else: + print(render_drift_status(report)) + return 0 + + if args.command == "rulebook-status": + report = service.get_rulebook_status() + if args.json: + print(json.dumps(report.model_dump(mode="json"), indent=2, sort_keys=True)) + else: + print(render_rulebook_status(report)) + return 0 + + if args.command == "rulebook-diff": + report = service.get_rulebook_diff(args.from_release, args.to_release) + if args.json: + print(json.dumps(report.model_dump(mode="json"), indent=2, sort_keys=True)) + else: + print(render_rulebook_diff(report)) + return 0 + + if args.command == "validate-demo-case": + request = service.get_demo_case_request(args.demo_case) + warnings = service.validate_request(request) + print(f"demo_case={args.demo_case}") + print(f"payer={request.payer}") + print(f"procedure_code={request.procedure_code}") + print(f"site_of_care={request.site_of_care}") + if warnings: + print("warnings:") + for warning in warnings: + print(f"- {warning}") + else: + print("warnings: none") + return 0 + + except KeyError as exc: + print(str(exc), file=sys.stderr) + return 2 + except ServiceError as exc: + print(str(exc), file=sys.stderr) + return 2 + + parser.error(f"Unsupported command: {args.command}") + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/DEMO_WALKTHROUGH.md b/docs/DEMO_WALKTHROUGH.md deleted file mode 100644 index a196242..0000000 --- a/docs/DEMO_WALKTHROUGH.md +++ /dev/null @@ -1,38 +0,0 @@ -# Demo Walkthrough - -This is the fastest way to explain the repo. - -## One-Sentence Pitch - -Prior Authorization Readiness Copilot is a deterministic demo that checks whether a prior auth request is administratively ready based on documented payer criteria, and returns `CANNOT_DETERMINE` when required documentation is missing. - -## What To Say First - -- The current repo is deterministic. -- The current repo does not use an LLM. -- It supports administrative readiness review only. -- Missing documentation leads to refusal, not inference. - -## Demo Flow - -1. Run `streamlit run app.py`. -2. Open a featured showcase case. -3. Show requirement-level outputs: `MET`, `NOT_MET`, `NOT_DOCUMENTED`. -4. Show the overall status mapping. -5. Show blockers, extracted facts, and evidence spans. -6. Generate a letter and note that drafting is deterministic and downstream of the evaluated results. -7. Show the policy monitor as governance support for configured monitored sources, not for every supported procedure. - -## Honest Limitations - -- Coverage is intentionally narrow. -- Inputs are synthetic. -- Extraction supports only the implemented phrasing patterns. -- Policy drift monitoring only applies to configured sources. In the current repo, that means `MRI_LUMBAR`, not `CPAP_DEVICE`. - -## Best Supporting Docs - -- [README.md](/Users/nicholasleko/projects/PriorAuthorizationCopilot/README.md) -- [EXTRACTION_CONTRACT.md](/Users/nicholasleko/projects/PriorAuthorizationCopilot/EXTRACTION_CONTRACT.md) -- [FAILURE_MODES.md](/Users/nicholasleko/projects/PriorAuthorizationCopilot/FAILURE_MODES.md) -- [MODEL_CARD.md](/Users/nicholasleko/projects/PriorAuthorizationCopilot/MODEL_CARD.md) diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 0000000..856675f --- /dev/null +++ b/docs/api.md @@ -0,0 +1,142 @@ +# API + +The FastAPI layer exposes the current deterministic capabilities of the repo without widening scope. + +Run locally: + +```bash +python3 -m uvicorn api:app --reload +``` + +Base URL in local examples: `http://127.0.0.1:8000` + +## Endpoints + +### `GET /health` + +Returns basic service status, including: + +- runtime `rules_version` +- active `rulebook_active_release_id` +- supported procedure count +- monitored source count + +```bash +curl http://127.0.0.1:8000/health +``` + +### `GET /supported-procedures` + +Lists payer/procedure combinations currently supported by the rules bundle, including: + +- procedure category +- rule family +- supported sites +- last rule update +- provenance summary +- monitored-for-drift status + +```bash +curl http://127.0.0.1:8000/supported-procedures +``` + +### `GET /demo-cases` + +Lists bundled synthetic demo cases. + +```bash +curl http://127.0.0.1:8000/demo-cases +``` + +### `POST /evaluate` + +Runs deterministic administrative readiness evaluation. + +```bash +curl -X POST http://127.0.0.1:8000/evaluate \ + -H "Content-Type: application/json" \ + -d '{ + "payer": "Aetna", + "procedure_code": "CPAP_DEVICE", + "dx_codes": ["G47.33"], + "site_of_care": "outpatient", + "specialty": "Sleep Medicine", + "note_text": "Dx: OSA. Sleep study completed 2024-05-18. AHI 22 documented. Requests CPAP E0601." + }' +``` + +Response highlights: + +- `overall_status` +- `submission_readiness` +- `results` +- `blockers` +- `facts` +- `evidence_map` +- `audit_trail` + +### `GET /drift-status` + +Returns governance-only drift status for configured monitored sources, including: + +- source name +- source type +- check frequency +- freshness status +- days since last snapshot check +- latest snapshot hash +- latest event +- latest diff path if present +- linked rule source label +- review reason when stale or drifted + +```bash +curl http://127.0.0.1:8000/drift-status +``` + +### `GET /rulebook` + +Returns the current rulebook manifest view, including: + +- active release ID +- stage assignments +- reviewed and active release metadata +- runtime-match validation for the active snapshot +- any manifest validation errors + +```bash +curl http://127.0.0.1:8000/rulebook +``` + +### `GET /rulebook/diff` + +Returns a structured diff between two rulebook releases. + +```bash +curl "http://127.0.0.1:8000/rulebook/diff?from_release_id=2026-04-09-reviewed-v0.4&to_release_id=2026-04-09-active-v0.5" +``` + +## Error Behavior + +Unsupported scope returns a structured error response like: + +```json +{ + "error": "unsupported_scope", + "detail": "Unsupported request scope ..." +} +``` + +The API is intentionally conservative: + +- unsupported procedures are rejected +- unsupported sites of care are rejected +- missing documentation does not raise an error; it drives `CANNOT_DETERMINE` +- governance endpoints never mutate runtime rules + +## Notes + +- The API uses synthetic-only demo logic. +- There is no persistence layer. +- There is no authentication layer. +- There is no autonomous action endpoint. diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..d064f5a --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,114 @@ +# Architecture + +## System Shape + +This repo is a compact deterministic application, not a platform. + +The architecture is intentionally split into a small number of explainable layers: + +1. data and rules +2. deterministic extraction +3. deterministic evaluation +4. shared orchestration +5. output surfaces +6. rulebook governance +7. governance-only drift monitoring + +## Module Boundaries + +### Domain and schemas + +- `engine/schemas.py` +- typed request, result, blocker, audit, drift, and supported-procedure models + +### Rule and provenance loading + +- `engine/rules_loader.py` +- `engine/provenance.py` +- load versioned payer rules, procedure registry metadata, and provenance metadata + +### Deterministic extraction + +- `engine/extract.py` +- converts note text into a narrow set of structured facts plus evidence spans +- conservative by design: missing or unclear information stays missing + +### Deterministic evaluation + +- `engine/evaluate.py` +- evaluates extracted facts against requirement definitions +- preserves frozen semantics: + - `READY`: all requirements met + - `NOT_READY`: all requirements documented, but at least one fails threshold + - `CANNOT_DETERMINE`: at least one required element is not documented + +### Shared application service + +- `engine/service.py` +- the main orchestration boundary +- validates scope +- normalizes request inputs +- calls extraction and evaluation +- computes blockers, metrics, audit trace, warnings, procedure registry metadata, and provenance summaries +- returns one standardized `EvaluationResult` + +### Rendering and artifacts + +- `engine/rendering.py` +- converts evaluation results into stable export payloads and CLI summaries + +### Acceptance harness + +- `engine/acceptance.py` +- normalizes stable product outputs into golden snapshots for regression protection + +### Demo case registry + +- `engine/demo_cases.py` +- loads reusable synthetic fixtures used by UI, CLI, tests, and artifact generation + +### Rulebook governance + +- `engine/rulebook.py` +- validates versioned rulebook snapshots +- diffs reviewed and active releases +- keeps promotion metadata separate from runtime drift monitoring + +### Governance-only drift monitoring + +- `engine/policy_monitor.py` +- snapshots monitored sources +- computes diffs and drift events +- never mutates rules automatically + +### App surfaces + +- `app.py`: Streamlit operator demo +- `api.py`: FastAPI endpoints +- `cli.py`: local demo and export commands + +## Runtime Flow + +1. A request enters through Streamlit, the API, CLI, or an artifact script. +2. `engine/service.py` validates scope and normalizes the request. +3. `engine/extract.py` deterministically extracts facts and evidence spans. +4. `engine/evaluate.py` applies rule requirements and returns requirement results. +5. `engine/service.py` assembles blockers, metrics, warnings, procedure metadata, provenance, rulebook metadata, and audit trace. +6. The surface renders or exports the same typed result. + +## Why This Shape Was Chosen + +- It keeps the deterministic core small and interview-explainable. +- It avoids pushing product logic into the Streamlit app. +- It gives the repo reusable API and CLI surfaces without introducing a database or service mesh. +- It supports stronger tests, stable exported artifacts, and a human-review governance story. + +## What Was Intentionally Left Simple + +- no database +- no auth +- no background workers +- no generic workflow engine +- no LLM orchestration layer + +Those would make the repo look larger, not stronger. diff --git a/docs/artifacts/CPAP-02-borderline.json b/docs/artifacts/CPAP-02-borderline.json new file mode 100644 index 0000000..4aca974 --- /dev/null +++ b/docs/artifacts/CPAP-02-borderline.json @@ -0,0 +1,448 @@ +{ + "audit_trail": { + "blocking_issues": { + "not_documented": [ + { + "key": "sleep_study_date", + "label": "Sleep study date documented", + "reason": "Not found in note. Add explicit statement.", + "status": "NOT_DOCUMENTED" + }, + { + "key": "ahi_documented", + "label": "AHI/RDI documented", + "reason": "Not found in note. Add explicit statement.", + "status": "NOT_DOCUMENTED" + } + ], + "not_met": [] + }, + "evaluation_warnings": [ + "Policy trust remains DEMO for this procedure. Verify against official policy before real-world use." + ], + "evidence_map": { + "ahi_documented": [ + { + "end": 88, + "start": 70, + "text": "AHI not documented" + } + ], + "osa_diagnosis": [ + { + "end": 3, + "start": 0, + "text": "OSA" + } + ] + }, + "facts_extracted": { + "ahi_documented": null, + "conservative_therapy_weeks": null, + "mechanical_symptoms_documented": null, + "neuro_red_flags_documented": null, + "osa_diagnosis": true, + "prior_imaging_result": null, + "sleep_study_date": null, + "symptom_duration_weeks": null + }, + "invariant_errors": [], + "metrics": { + "compliance_rate": 100.0, + "compliant_count": 1, + "extraction_failure_count": 2, + "extraction_success_rate": 33.3, + "non_compliant_count": 0 + }, + "note_hash": "5ef99d034f3c708e", + "note_length": 89, + "overall_status": "CANNOT_DETERMINE", + "payer": "Aetna", + "policy_trust_level": "demo", + "procedure_code": "CPAP_DEVICE", + "procedure_name": "CPAP Device (HCPCS E0601)", + "provenance_snapshot": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": null, + "monitored_source_id": null, + "monitored_source_name": null, + "monitored_source_owner": null, + "monitored_source_url": null, + "notes": "Rules are illustrative for demo.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of CPAP administrative documentation criteria", + "source_name": "Aetna DME policy (summary)", + "source_type": "manual_summary", + "source_url": null + }, + "requirements_checked": [ + "osa_diagnosis", + "sleep_study_date", + "ahi_documented" + ], + "rulebook_active_release_id": "2026-04-09-active-v0.5", + "rules_version": "0.5", + "run_id": "ac3bf875-685b-4d53-9310-18d736eb772d", + "site_of_care": "outpatient", + "specialty": "Primary Care", + "submission_readiness": false, + "timestamp_utc": "2026-04-09T15:24:39Z" + }, + "blockers": { + "not_documented": [ + { + "key": "sleep_study_date", + "label": "Sleep study date documented", + "reason": "Not found in note. Add explicit statement.", + "status": "NOT_DOCUMENTED" + }, + { + "key": "ahi_documented", + "label": "AHI/RDI documented", + "reason": "Not found in note. Add explicit statement.", + "status": "NOT_DOCUMENTED" + } + ], + "not_met": [] + }, + "evidence_map": { + "ahi_documented": [ + { + "end": 88, + "start": 70, + "text": "AHI not documented" + } + ], + "osa_diagnosis": [ + { + "end": 3, + "start": 0, + "text": "OSA" + } + ] + }, + "facts": { + "ahi_documented": null, + "conservative_therapy_weeks": null, + "mechanical_symptoms_documented": null, + "neuro_red_flags_documented": null, + "osa_diagnosis": true, + "prior_imaging_result": null, + "sleep_study_date": null, + "symptom_duration_weeks": null + }, + "letter": { + "metadata": { + "cited_snippets_count": 0, + "contains_missing_documentation": true, + "draft_blocked": true, + "draft_blocked_reasons": [ + "Prohibited language detected: 'diagnosis'" + ], + "generated_timestamp_utc": "2026-04-09T15:24:39Z", + "letter_hash_sha256_16": "24229731fe7162a2", + "letter_type": "submission_cover_letter", + "letter_version": "1.1", + "overall_status": "CANNOT_DETERMINE", + "policy_trust_level": "demo" + }, + "text": "DRAFT_BLOCKED\n\nThe letter was blocked due to prohibited language:\n- Prohibited language detected: 'diagnosis'\n" + }, + "metrics": { + "compliance_rate": 100.0, + "compliant_count": 1, + "extraction_failure_count": 2, + "extraction_success_rate": 33.3, + "non_compliant_count": 0 + }, + "overall_status": "CANNOT_DETERMINE", + "policy_trust_level": "demo", + "provenance": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": null, + "monitored_source_id": null, + "monitored_source_name": null, + "monitored_source_owner": null, + "monitored_source_url": null, + "notes": "Rules are illustrative for demo.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of CPAP administrative documentation criteria", + "source_name": "Aetna DME policy (summary)", + "source_type": "manual_summary", + "source_url": null + }, + "readiness_score": 33, + "report": { + "audit_trail": { + "blocking_issues": { + "not_documented": [ + { + "key": "sleep_study_date", + "label": "Sleep study date documented", + "reason": "Not found in note. Add explicit statement.", + "status": "NOT_DOCUMENTED" + }, + { + "key": "ahi_documented", + "label": "AHI/RDI documented", + "reason": "Not found in note. Add explicit statement.", + "status": "NOT_DOCUMENTED" + } + ], + "not_met": [] + }, + "evaluation_warnings": [ + "Policy trust remains DEMO for this procedure. Verify against official policy before real-world use." + ], + "evidence_map": { + "ahi_documented": [ + { + "end": 88, + "start": 70, + "text": "AHI not documented" + } + ], + "osa_diagnosis": [ + { + "end": 3, + "start": 0, + "text": "OSA" + } + ] + }, + "facts_extracted": { + "ahi_documented": null, + "conservative_therapy_weeks": null, + "mechanical_symptoms_documented": null, + "neuro_red_flags_documented": null, + "osa_diagnosis": true, + "prior_imaging_result": null, + "sleep_study_date": null, + "symptom_duration_weeks": null + }, + "invariant_errors": [], + "metrics": { + "compliance_rate": 100.0, + "compliant_count": 1, + "extraction_failure_count": 2, + "extraction_success_rate": 33.3, + "non_compliant_count": 0 + }, + "note_hash": "5ef99d034f3c708e", + "note_length": 89, + "overall_status": "CANNOT_DETERMINE", + "payer": "Aetna", + "policy_trust_level": "demo", + "procedure_code": "CPAP_DEVICE", + "procedure_name": "CPAP Device (HCPCS E0601)", + "provenance_snapshot": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": null, + "monitored_source_id": null, + "monitored_source_name": null, + "monitored_source_owner": null, + "monitored_source_url": null, + "notes": "Rules are illustrative for demo.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of CPAP administrative documentation criteria", + "source_name": "Aetna DME policy (summary)", + "source_type": "manual_summary", + "source_url": null + }, + "requirements_checked": [ + "osa_diagnosis", + "sleep_study_date", + "ahi_documented" + ], + "rulebook_active_release_id": "2026-04-09-active-v0.5", + "rules_version": "0.5", + "run_id": "ac3bf875-685b-4d53-9310-18d736eb772d", + "site_of_care": "outpatient", + "specialty": "Primary Care", + "submission_readiness": false, + "timestamp_utc": "2026-04-09T15:24:39Z" + }, + "letter_draft": "", + "met_count": 1, + "not_documented_count": 2, + "not_met_count": 0, + "readiness_score": 33, + "results": [ + { + "evidence": "Diagnosis present", + "evidence_snippets": [ + "OSA" + ], + "evidence_spans": [ + { + "end": 3, + "start": 0, + "text": "OSA" + } + ], + "key": "osa_diagnosis", + "label": "OSA diagnosis documented", + "reason": "Explicitly addressed in documentation (present/affirmed).", + "status": "MET" + }, + { + "evidence": "Date in chart", + "evidence_snippets": [], + "evidence_spans": [], + "key": "sleep_study_date", + "label": "Sleep study date documented", + "reason": "Not found in note. Add explicit statement.", + "status": "NOT_DOCUMENTED" + }, + { + "evidence": "AHI value included", + "evidence_snippets": [ + "AHI not documented" + ], + "evidence_spans": [ + { + "end": 88, + "start": 70, + "text": "AHI not documented" + } + ], + "key": "ahi_documented", + "label": "AHI/RDI documented", + "reason": "Not found in note. Add explicit statement.", + "status": "NOT_DOCUMENTED" + } + ], + "rule_reasons": [ + "Sleep study date documented: NOT_DOCUMENTED \u2014 Not found in note. Add explicit statement.", + "AHI/RDI documented: NOT_DOCUMENTED \u2014 Not found in note. Add explicit statement." + ] + }, + "request": { + "dx_codes": [ + "G47.33" + ], + "note_text": "OSA noted in problem list. Sleep study in 2023 mentioned but no date. AHI not documented.", + "payer": "Aetna", + "procedure_code": "CPAP_DEVICE", + "site_of_care": "outpatient", + "specialty": "Primary Care" + }, + "results": [ + { + "evidence": "Diagnosis present", + "evidence_snippets": [ + "OSA" + ], + "evidence_spans": [ + { + "end": 3, + "start": 0, + "text": "OSA" + } + ], + "key": "osa_diagnosis", + "label": "OSA diagnosis documented", + "reason": "Explicitly addressed in documentation (present/affirmed).", + "status": "MET" + }, + { + "evidence": "Date in chart", + "evidence_snippets": [], + "evidence_spans": [], + "key": "sleep_study_date", + "label": "Sleep study date documented", + "reason": "Not found in note. Add explicit statement.", + "status": "NOT_DOCUMENTED" + }, + { + "evidence": "AHI value included", + "evidence_snippets": [ + "AHI not documented" + ], + "evidence_spans": [ + { + "end": 88, + "start": 70, + "text": "AHI not documented" + } + ], + "key": "ahi_documented", + "label": "AHI/RDI documented", + "reason": "Not found in note. Add explicit statement.", + "status": "NOT_DOCUMENTED" + } + ], + "rule_reasons": [ + "Sleep study date documented: NOT_DOCUMENTED \u2014 Not found in note. Add explicit statement.", + "AHI/RDI documented: NOT_DOCUMENTED \u2014 Not found in note. Add explicit statement." + ], + "submission_readiness": false, + "supported_procedure": { + "display_name": "CPAP Device (HCPCS E0601)", + "metadata": { + "category": "durable_medical_equipment", + "last_rule_update": "2026-04-09", + "notes": [ + "The current demo checks for OSA diagnosis, dated sleep study evidence, and AHI/RDI documentation.", + "It does not determine clinical appropriateness or device approval likelihood." + ], + "rule_family": "sleep_study_documentation", + "summary": "Administrative readiness check for CPAP device requests using a narrow deterministic documentation contract.", + "supported_sites": [ + "outpatient" + ] + }, + "monitored_for_drift": false, + "payer": "Aetna", + "policy_trust_level": "demo", + "procedure_code": "CPAP_DEVICE", + "provenance": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": null, + "monitored_source_id": null, + "monitored_source_name": null, + "monitored_source_owner": null, + "monitored_source_url": null, + "notes": "Rules are illustrative for demo.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of CPAP administrative documentation criteria", + "source_name": "Aetna DME policy (summary)", + "source_type": "manual_summary", + "source_url": null + }, + "required_field_keys": [ + "osa_diagnosis", + "sleep_study_date", + "ahi_documented" + ], + "requirements": [ + { + "allowed": [], + "evidence": "Diagnosis present", + "key": "osa_diagnosis", + "label": "OSA diagnosis documented", + "min": null, + "type": "boolean" + }, + { + "allowed": [], + "evidence": "Date in chart", + "key": "sleep_study_date", + "label": "Sleep study date documented", + "min": null, + "type": "boolean" + }, + { + "allowed": [], + "evidence": "AHI value included", + "key": "ahi_documented", + "label": "AHI/RDI documented", + "min": null, + "type": "boolean" + } + ] + }, + "warnings": [ + "Policy trust remains DEMO for this procedure. Verify against official policy before real-world use." + ] +} diff --git a/docs/artifacts/MRI-01-complete.json b/docs/artifacts/MRI-01-complete.json new file mode 100644 index 0000000..2494459 --- /dev/null +++ b/docs/artifacts/MRI-01-complete.json @@ -0,0 +1,508 @@ +{ + "audit_trail": { + "blocking_issues": { + "not_documented": [], + "not_met": [] + }, + "evaluation_warnings": [ + "Policy trust remains DEMO for this procedure. Verify against official policy before real-world use." + ], + "evidence_map": { + "conservative_therapy_weeks": [ + { + "end": 78, + "start": 64, + "text": "PT for 8 weeks" + } + ], + "neuro_red_flags_documented": [ + { + "end": 239, + "start": 116, + "text": "Denies bowel/bladder incontinence. No saddle anesthesia. Prior imaging: lumbar xray inconclusive. Neuro exam: mild weakness" + } + ], + "prior_imaging_result": [ + { + "end": 199, + "start": 195, + "text": "xray" + } + ], + "symptom_duration_weeks": [ + { + "end": 52, + "start": 45, + "text": "8 weeks" + } + ] + }, + "facts_extracted": { + "ahi_documented": null, + "conservative_therapy_weeks": 8, + "mechanical_symptoms_documented": null, + "neuro_red_flags_documented": true, + "osa_diagnosis": null, + "prior_imaging_result": "inconclusive", + "sleep_study_date": null, + "symptom_duration_weeks": 8 + }, + "invariant_errors": [], + "metrics": { + "compliance_rate": 100.0, + "compliant_count": 4, + "extraction_failure_count": 0, + "extraction_success_rate": 100.0, + "non_compliant_count": 0 + }, + "note_hash": "82a37cb51ea58464", + "note_length": 257, + "overall_status": "READY", + "payer": "Aetna", + "policy_trust_level": "demo", + "procedure_code": "MRI_LUMBAR", + "procedure_name": "MRI Lumbar Spine (no contrast)", + "provenance_snapshot": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": "daily", + "monitored_source_id": "aetna_mri_lumbar", + "monitored_source_name": "Aetna CPB 0157", + "monitored_source_owner": "NickLeko", + "monitored_source_url": "https://www.aetna.com/cpb/medical/data/100_199/0157.html", + "notes": "Rules are illustrative for demo. Not a substitute for payer policy documents.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of spine MRI administrative criteria", + "source_name": "Aetna spine MRI policy summary", + "source_type": "manual_summary", + "source_url": "https://www.aetna.com/cpb/medical/data/100_199/0157.html" + }, + "requirements_checked": [ + "conservative_therapy_weeks", + "neuro_red_flags_documented", + "prior_imaging_result", + "symptom_duration_weeks" + ], + "rulebook_active_release_id": "2026-04-09-active-v0.5", + "rules_version": "0.5", + "run_id": "7270f995-611a-4770-bac4-4eabef6d64ea", + "site_of_care": "outpatient", + "specialty": "Orthopedics", + "submission_readiness": true, + "timestamp_utc": "2026-04-09T15:24:39Z" + }, + "blockers": { + "not_documented": [], + "not_met": [] + }, + "evidence_map": { + "conservative_therapy_weeks": [ + { + "end": 78, + "start": 64, + "text": "PT for 8 weeks" + } + ], + "neuro_red_flags_documented": [ + { + "end": 239, + "start": 116, + "text": "Denies bowel/bladder incontinence. No saddle anesthesia. Prior imaging: lumbar xray inconclusive. Neuro exam: mild weakness" + } + ], + "prior_imaging_result": [ + { + "end": 199, + "start": 195, + "text": "xray" + } + ], + "symptom_duration_weeks": [ + { + "end": 52, + "start": 45, + "text": "8 weeks" + } + ] + }, + "facts": { + "ahi_documented": null, + "conservative_therapy_weeks": 8, + "mechanical_symptoms_documented": null, + "neuro_red_flags_documented": true, + "osa_diagnosis": null, + "prior_imaging_result": "inconclusive", + "sleep_study_date": null, + "symptom_duration_weeks": 8 + }, + "letter": { + "metadata": { + "cited_snippets_count": 4, + "contains_missing_documentation": false, + "draft_blocked": false, + "draft_blocked_reasons": [], + "generated_timestamp_utc": "2026-04-09T15:24:39Z", + "letter_hash_sha256_16": "e8f947f931d9208d", + "letter_type": "submission_cover_letter", + "letter_version": "1.1", + "overall_status": "READY", + "policy_trust_level": "demo" + }, + "text": "PRIOR AUTHORIZATION ADMINISTRATIVE READINESS SUMMARY\n\nPayer: Aetna\nProcedure: MRI_LUMBAR\nSite of care: outpatient\nSpecialty: Orthopedics\nGenerated: 2026-04-09T15:24:39Z\nDx codes: M54.16\nPolicy trust level: DEMO \u2014 criteria are illustrative only. Verify against the official payer policy before submission.\n\nOverall Status: READY\n\nSummary:\nThis letter supports administrative submission readiness based on the documentation present in the record. This does not guarantee payer approval.\n\nRequirements:\n- Conservative therapy duration (weeks) (conservative_therapy_weeks): MET\n Reason: Documented value: 8.\n Evidence:\n - \"PT for 8 weeks\"\n- Neuro red flags explicitly addressed (present or denied) (neuro_red_flags_documented): MET\n Reason: Explicitly addressed in documentation (present/affirmed).\n Evidence:\n - \"Denies bowel/bladder incontinence. No saddle anesthesia. Prior imaging: lumbar xray inconclusive. Neuro exam: mild weakness\"\n- Prior imaging result (prior_imaging_result): MET\n Reason: Documented: inconclusive.\n Evidence:\n - \"xray\"\n- Symptom duration (weeks) (symptom_duration_weeks): MET\n Reason: Documented value: 8.\n Evidence:\n - \"8 weeks\"\nClosing:\nThis letter summarizes documentation-based administrative readiness for prior authorization submission. It does not provide clinical recommendations and does not predict approval outcomes.\n" + }, + "metrics": { + "compliance_rate": 100.0, + "compliant_count": 4, + "extraction_failure_count": 0, + "extraction_success_rate": 100.0, + "non_compliant_count": 0 + }, + "overall_status": "READY", + "policy_trust_level": "demo", + "provenance": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": "daily", + "monitored_source_id": "aetna_mri_lumbar", + "monitored_source_name": "Aetna CPB 0157", + "monitored_source_owner": "NickLeko", + "monitored_source_url": "https://www.aetna.com/cpb/medical/data/100_199/0157.html", + "notes": "Rules are illustrative for demo. Not a substitute for payer policy documents.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of spine MRI administrative criteria", + "source_name": "Aetna spine MRI policy summary", + "source_type": "manual_summary", + "source_url": "https://www.aetna.com/cpb/medical/data/100_199/0157.html" + }, + "readiness_score": 100, + "report": { + "audit_trail": { + "blocking_issues": { + "not_documented": [], + "not_met": [] + }, + "evaluation_warnings": [ + "Policy trust remains DEMO for this procedure. Verify against official policy before real-world use." + ], + "evidence_map": { + "conservative_therapy_weeks": [ + { + "end": 78, + "start": 64, + "text": "PT for 8 weeks" + } + ], + "neuro_red_flags_documented": [ + { + "end": 239, + "start": 116, + "text": "Denies bowel/bladder incontinence. No saddle anesthesia. Prior imaging: lumbar xray inconclusive. Neuro exam: mild weakness" + } + ], + "prior_imaging_result": [ + { + "end": 199, + "start": 195, + "text": "xray" + } + ], + "symptom_duration_weeks": [ + { + "end": 52, + "start": 45, + "text": "8 weeks" + } + ] + }, + "facts_extracted": { + "ahi_documented": null, + "conservative_therapy_weeks": 8, + "mechanical_symptoms_documented": null, + "neuro_red_flags_documented": true, + "osa_diagnosis": null, + "prior_imaging_result": "inconclusive", + "sleep_study_date": null, + "symptom_duration_weeks": 8 + }, + "invariant_errors": [], + "metrics": { + "compliance_rate": 100.0, + "compliant_count": 4, + "extraction_failure_count": 0, + "extraction_success_rate": 100.0, + "non_compliant_count": 0 + }, + "note_hash": "82a37cb51ea58464", + "note_length": 257, + "overall_status": "READY", + "payer": "Aetna", + "policy_trust_level": "demo", + "procedure_code": "MRI_LUMBAR", + "procedure_name": "MRI Lumbar Spine (no contrast)", + "provenance_snapshot": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": "daily", + "monitored_source_id": "aetna_mri_lumbar", + "monitored_source_name": "Aetna CPB 0157", + "monitored_source_owner": "NickLeko", + "monitored_source_url": "https://www.aetna.com/cpb/medical/data/100_199/0157.html", + "notes": "Rules are illustrative for demo. Not a substitute for payer policy documents.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of spine MRI administrative criteria", + "source_name": "Aetna spine MRI policy summary", + "source_type": "manual_summary", + "source_url": "https://www.aetna.com/cpb/medical/data/100_199/0157.html" + }, + "requirements_checked": [ + "conservative_therapy_weeks", + "neuro_red_flags_documented", + "prior_imaging_result", + "symptom_duration_weeks" + ], + "rulebook_active_release_id": "2026-04-09-active-v0.5", + "rules_version": "0.5", + "run_id": "7270f995-611a-4770-bac4-4eabef6d64ea", + "site_of_care": "outpatient", + "specialty": "Orthopedics", + "submission_readiness": true, + "timestamp_utc": "2026-04-09T15:24:39Z" + }, + "letter_draft": "", + "met_count": 4, + "not_documented_count": 0, + "not_met_count": 0, + "readiness_score": 100, + "results": [ + { + "evidence": "PT/NSAIDs/activity modification trial documented", + "evidence_snippets": [ + "PT for 8 weeks" + ], + "evidence_spans": [ + { + "end": 78, + "start": 64, + "text": "PT for 8 weeks" + } + ], + "key": "conservative_therapy_weeks", + "label": "Conservative therapy duration (weeks)", + "reason": "Documented value: 8.", + "status": "MET" + }, + { + "evidence": "Weakness, saddle anesthesia, bowel/bladder changes explicitly documented as present or denied", + "evidence_snippets": [ + "Denies bowel/bladder incontinence. No saddle anesthesia. Prior imaging: lumbar xray inconclusive. Neuro exam: mild weakness" + ], + "evidence_spans": [ + { + "end": 239, + "start": 116, + "text": "Denies bowel/bladder incontinence. No saddle anesthesia. Prior imaging: lumbar xray inconclusive. Neuro exam: mild weakness" + } + ], + "key": "neuro_red_flags_documented", + "label": "Neuro red flags explicitly addressed (present or denied)", + "reason": "Explicitly addressed in documentation (present/affirmed).", + "status": "MET" + }, + { + "evidence": "X-ray/CT results or note of none", + "evidence_snippets": [ + "xray" + ], + "evidence_spans": [ + { + "end": 199, + "start": 195, + "text": "xray" + } + ], + "key": "prior_imaging_result", + "label": "Prior imaging result", + "reason": "Documented: inconclusive.", + "status": "MET" + }, + { + "evidence": "Duration supports guideline-based escalation", + "evidence_snippets": [ + "8 weeks" + ], + "evidence_spans": [ + { + "end": 52, + "start": 45, + "text": "8 weeks" + } + ], + "key": "symptom_duration_weeks", + "label": "Symptom duration (weeks)", + "reason": "Documented value: 8.", + "status": "MET" + } + ], + "rule_reasons": [] + }, + "request": { + "dx_codes": [ + "M54.16" + ], + "note_text": "Low back pain with right leg radiculopathy x 8 weeks. Completed PT for 8 weeks and NSAIDs with minimal improvement. Denies bowel/bladder incontinence. No saddle anesthesia. Prior imaging: lumbar xray inconclusive. Neuro exam: mild weakness dorsiflexion 4/5.", + "payer": "Aetna", + "procedure_code": "MRI_LUMBAR", + "site_of_care": "outpatient", + "specialty": "Orthopedics" + }, + "results": [ + { + "evidence": "PT/NSAIDs/activity modification trial documented", + "evidence_snippets": [ + "PT for 8 weeks" + ], + "evidence_spans": [ + { + "end": 78, + "start": 64, + "text": "PT for 8 weeks" + } + ], + "key": "conservative_therapy_weeks", + "label": "Conservative therapy duration (weeks)", + "reason": "Documented value: 8.", + "status": "MET" + }, + { + "evidence": "Weakness, saddle anesthesia, bowel/bladder changes explicitly documented as present or denied", + "evidence_snippets": [ + "Denies bowel/bladder incontinence. No saddle anesthesia. Prior imaging: lumbar xray inconclusive. Neuro exam: mild weakness" + ], + "evidence_spans": [ + { + "end": 239, + "start": 116, + "text": "Denies bowel/bladder incontinence. No saddle anesthesia. Prior imaging: lumbar xray inconclusive. Neuro exam: mild weakness" + } + ], + "key": "neuro_red_flags_documented", + "label": "Neuro red flags explicitly addressed (present or denied)", + "reason": "Explicitly addressed in documentation (present/affirmed).", + "status": "MET" + }, + { + "evidence": "X-ray/CT results or note of none", + "evidence_snippets": [ + "xray" + ], + "evidence_spans": [ + { + "end": 199, + "start": 195, + "text": "xray" + } + ], + "key": "prior_imaging_result", + "label": "Prior imaging result", + "reason": "Documented: inconclusive.", + "status": "MET" + }, + { + "evidence": "Duration supports guideline-based escalation", + "evidence_snippets": [ + "8 weeks" + ], + "evidence_spans": [ + { + "end": 52, + "start": 45, + "text": "8 weeks" + } + ], + "key": "symptom_duration_weeks", + "label": "Symptom duration (weeks)", + "reason": "Documented value: 8.", + "status": "MET" + } + ], + "rule_reasons": [], + "submission_readiness": true, + "supported_procedure": { + "display_name": "MRI Lumbar Spine (no contrast)", + "metadata": { + "category": "advanced_imaging", + "last_rule_update": "2026-04-09", + "notes": [ + "Demo rule set only; not a substitute for payer policy review.", + "Requires conservative therapy, symptom duration, imaging context, and explicit red-flag review." + ], + "rule_family": "spine_mri_conservative_therapy", + "summary": "Administrative readiness check for lumbar spine MRI requests using a narrow deterministic evidence contract.", + "supported_sites": [ + "outpatient" + ] + }, + "monitored_for_drift": true, + "payer": "Aetna", + "policy_trust_level": "demo", + "procedure_code": "MRI_LUMBAR", + "provenance": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": "daily", + "monitored_source_id": "aetna_mri_lumbar", + "monitored_source_name": "Aetna CPB 0157", + "monitored_source_owner": "NickLeko", + "monitored_source_url": "https://www.aetna.com/cpb/medical/data/100_199/0157.html", + "notes": "Rules are illustrative for demo. Not a substitute for payer policy documents.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of spine MRI administrative criteria", + "source_name": "Aetna spine MRI policy summary", + "source_type": "manual_summary", + "source_url": "https://www.aetna.com/cpb/medical/data/100_199/0157.html" + }, + "required_field_keys": [ + "conservative_therapy_weeks", + "neuro_red_flags_documented", + "prior_imaging_result", + "symptom_duration_weeks" + ], + "requirements": [ + { + "allowed": [], + "evidence": "PT/NSAIDs/activity modification trial documented", + "key": "conservative_therapy_weeks", + "label": "Conservative therapy duration (weeks)", + "min": 6.0, + "type": "number" + }, + { + "allowed": [], + "evidence": "Weakness, saddle anesthesia, bowel/bladder changes explicitly documented as present or denied", + "key": "neuro_red_flags_documented", + "label": "Neuro red flags explicitly addressed (present or denied)", + "min": null, + "type": "boolean" + }, + { + "allowed": [ + "none", + "inconclusive", + "abnormal" + ], + "evidence": "X-ray/CT results or note of none", + "key": "prior_imaging_result", + "label": "Prior imaging result", + "min": null, + "type": "enum" + }, + { + "allowed": [], + "evidence": "Duration supports guideline-based escalation", + "key": "symptom_duration_weeks", + "label": "Symptom duration (weeks)", + "min": 6.0, + "type": "number" + } + ] + }, + "warnings": [ + "Policy trust remains DEMO for this procedure. Verify against official policy before real-world use." + ] +} diff --git a/docs/artifacts/MRI-08-edge-below-threshold.json b/docs/artifacts/MRI-08-edge-below-threshold.json new file mode 100644 index 0000000..56cc53e --- /dev/null +++ b/docs/artifacts/MRI-08-edge-below-threshold.json @@ -0,0 +1,553 @@ +{ + "audit_trail": { + "blocking_issues": { + "not_documented": [], + "not_met": [ + { + "key": "conservative_therapy_weeks", + "label": "Conservative therapy duration (weeks)", + "reason": "Documented value (5) below requirement (>= 6.0). Clarify or justify.", + "status": "NOT_MET" + }, + { + "key": "symptom_duration_weeks", + "label": "Symptom duration (weeks)", + "reason": "Documented value (5) below requirement (>= 6.0). Clarify or justify.", + "status": "NOT_MET" + } + ] + }, + "evaluation_warnings": [ + "Policy trust remains DEMO for this procedure. Verify against official policy before real-world use." + ], + "evidence_map": { + "conservative_therapy_weeks": [ + { + "end": 37, + "start": 25, + "text": "PT x 5 weeks" + } + ], + "neuro_red_flags_documented": [ + { + "end": 87, + "start": 50, + "text": "Denies weakness. Denies bowel/bladder" + } + ], + "prior_imaging_result": [ + { + "end": 113, + "start": 97, + "text": "No prior imaging" + } + ], + "symptom_duration_weeks": [ + { + "end": 23, + "start": 16, + "text": "5 weeks" + } + ] + }, + "facts_extracted": { + "ahi_documented": null, + "conservative_therapy_weeks": 5, + "mechanical_symptoms_documented": null, + "neuro_red_flags_documented": true, + "osa_diagnosis": null, + "prior_imaging_result": "none", + "sleep_study_date": null, + "symptom_duration_weeks": 5 + }, + "invariant_errors": [], + "metrics": { + "compliance_rate": 50.0, + "compliant_count": 2, + "extraction_failure_count": 0, + "extraction_success_rate": 100.0, + "non_compliant_count": 2 + }, + "note_hash": "94ee8ba7171f7dea", + "note_length": 114, + "overall_status": "NOT_READY", + "payer": "Aetna", + "policy_trust_level": "demo", + "procedure_code": "MRI_LUMBAR", + "procedure_name": "MRI Lumbar Spine (no contrast)", + "provenance_snapshot": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": "daily", + "monitored_source_id": "aetna_mri_lumbar", + "monitored_source_name": "Aetna CPB 0157", + "monitored_source_owner": "NickLeko", + "monitored_source_url": "https://www.aetna.com/cpb/medical/data/100_199/0157.html", + "notes": "Rules are illustrative for demo. Not a substitute for payer policy documents.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of spine MRI administrative criteria", + "source_name": "Aetna spine MRI policy summary", + "source_type": "manual_summary", + "source_url": "https://www.aetna.com/cpb/medical/data/100_199/0157.html" + }, + "requirements_checked": [ + "conservative_therapy_weeks", + "neuro_red_flags_documented", + "prior_imaging_result", + "symptom_duration_weeks" + ], + "rulebook_active_release_id": "2026-04-09-active-v0.5", + "rules_version": "0.5", + "run_id": "ceb2430a-f92a-4c57-9eca-27e0ad7bdcf7", + "site_of_care": "outpatient", + "specialty": "Primary Care", + "submission_readiness": false, + "timestamp_utc": "2026-04-09T15:24:39Z" + }, + "blockers": { + "not_documented": [], + "not_met": [ + { + "key": "conservative_therapy_weeks", + "label": "Conservative therapy duration (weeks)", + "reason": "Documented value (5) below requirement (>= 6.0). Clarify or justify.", + "status": "NOT_MET" + }, + { + "key": "symptom_duration_weeks", + "label": "Symptom duration (weeks)", + "reason": "Documented value (5) below requirement (>= 6.0). Clarify or justify.", + "status": "NOT_MET" + } + ] + }, + "evidence_map": { + "conservative_therapy_weeks": [ + { + "end": 37, + "start": 25, + "text": "PT x 5 weeks" + } + ], + "neuro_red_flags_documented": [ + { + "end": 87, + "start": 50, + "text": "Denies weakness. Denies bowel/bladder" + } + ], + "prior_imaging_result": [ + { + "end": 113, + "start": 97, + "text": "No prior imaging" + } + ], + "symptom_duration_weeks": [ + { + "end": 23, + "start": 16, + "text": "5 weeks" + } + ] + }, + "facts": { + "ahi_documented": null, + "conservative_therapy_weeks": 5, + "mechanical_symptoms_documented": null, + "neuro_red_flags_documented": true, + "osa_diagnosis": null, + "prior_imaging_result": "none", + "sleep_study_date": null, + "symptom_duration_weeks": 5 + }, + "letter": { + "metadata": { + "cited_snippets_count": 4, + "contains_missing_documentation": false, + "draft_blocked": false, + "draft_blocked_reasons": [], + "generated_timestamp_utc": "2026-04-09T15:24:39Z", + "letter_hash_sha256_16": "69829b27e9b0253a", + "letter_type": "submission_cover_letter", + "letter_version": "1.1", + "overall_status": "NOT_READY", + "policy_trust_level": "demo" + }, + "text": "PRIOR AUTHORIZATION ADMINISTRATIVE READINESS SUMMARY\n\nPayer: Aetna\nProcedure: MRI_LUMBAR\nSite of care: outpatient\nSpecialty: Primary Care\nGenerated: 2026-04-09T15:24:39Z\nDx codes: M54.5\nPolicy trust level: DEMO \u2014 criteria are illustrative only. Verify against the official payer policy before submission.\n\nOverall Status: NOT_READY\n\nSummary:\nThe request is not administratively ready for submission because one or more documented requirements do not meet thresholds. This does not represent a clinical judgment and does not guarantee payer approval.\n\nRequirements:\n- Conservative therapy duration (weeks) (conservative_therapy_weeks): NOT_MET\n Reason: Documented value (5) below requirement (>= 6.0). Clarify or justify.\n Evidence:\n - \"PT x 5 weeks\"\n- Neuro red flags explicitly addressed (present or denied) (neuro_red_flags_documented): MET\n Reason: Explicitly addressed in documentation (present/affirmed).\n Evidence:\n - \"Denies weakness. Denies bowel/bladder\"\n- Prior imaging result (prior_imaging_result): MET\n Reason: Documented: none.\n Evidence:\n - \"No prior imaging\"\n- Symptom duration (weeks) (symptom_duration_weeks): NOT_MET\n Reason: Documented value (5) below requirement (>= 6.0). Clarify or justify.\n Evidence:\n - \"5 weeks\"\nClosing:\nThis letter summarizes documentation-based administrative readiness for prior authorization submission. It does not provide clinical recommendations and does not predict approval outcomes.\n" + }, + "metrics": { + "compliance_rate": 50.0, + "compliant_count": 2, + "extraction_failure_count": 0, + "extraction_success_rate": 100.0, + "non_compliant_count": 2 + }, + "overall_status": "NOT_READY", + "policy_trust_level": "demo", + "provenance": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": "daily", + "monitored_source_id": "aetna_mri_lumbar", + "monitored_source_name": "Aetna CPB 0157", + "monitored_source_owner": "NickLeko", + "monitored_source_url": "https://www.aetna.com/cpb/medical/data/100_199/0157.html", + "notes": "Rules are illustrative for demo. Not a substitute for payer policy documents.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of spine MRI administrative criteria", + "source_name": "Aetna spine MRI policy summary", + "source_type": "manual_summary", + "source_url": "https://www.aetna.com/cpb/medical/data/100_199/0157.html" + }, + "readiness_score": 75, + "report": { + "audit_trail": { + "blocking_issues": { + "not_documented": [], + "not_met": [ + { + "key": "conservative_therapy_weeks", + "label": "Conservative therapy duration (weeks)", + "reason": "Documented value (5) below requirement (>= 6.0). Clarify or justify.", + "status": "NOT_MET" + }, + { + "key": "symptom_duration_weeks", + "label": "Symptom duration (weeks)", + "reason": "Documented value (5) below requirement (>= 6.0). Clarify or justify.", + "status": "NOT_MET" + } + ] + }, + "evaluation_warnings": [ + "Policy trust remains DEMO for this procedure. Verify against official policy before real-world use." + ], + "evidence_map": { + "conservative_therapy_weeks": [ + { + "end": 37, + "start": 25, + "text": "PT x 5 weeks" + } + ], + "neuro_red_flags_documented": [ + { + "end": 87, + "start": 50, + "text": "Denies weakness. Denies bowel/bladder" + } + ], + "prior_imaging_result": [ + { + "end": 113, + "start": 97, + "text": "No prior imaging" + } + ], + "symptom_duration_weeks": [ + { + "end": 23, + "start": 16, + "text": "5 weeks" + } + ] + }, + "facts_extracted": { + "ahi_documented": null, + "conservative_therapy_weeks": 5, + "mechanical_symptoms_documented": null, + "neuro_red_flags_documented": true, + "osa_diagnosis": null, + "prior_imaging_result": "none", + "sleep_study_date": null, + "symptom_duration_weeks": 5 + }, + "invariant_errors": [], + "metrics": { + "compliance_rate": 50.0, + "compliant_count": 2, + "extraction_failure_count": 0, + "extraction_success_rate": 100.0, + "non_compliant_count": 2 + }, + "note_hash": "94ee8ba7171f7dea", + "note_length": 114, + "overall_status": "NOT_READY", + "payer": "Aetna", + "policy_trust_level": "demo", + "procedure_code": "MRI_LUMBAR", + "procedure_name": "MRI Lumbar Spine (no contrast)", + "provenance_snapshot": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": "daily", + "monitored_source_id": "aetna_mri_lumbar", + "monitored_source_name": "Aetna CPB 0157", + "monitored_source_owner": "NickLeko", + "monitored_source_url": "https://www.aetna.com/cpb/medical/data/100_199/0157.html", + "notes": "Rules are illustrative for demo. Not a substitute for payer policy documents.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of spine MRI administrative criteria", + "source_name": "Aetna spine MRI policy summary", + "source_type": "manual_summary", + "source_url": "https://www.aetna.com/cpb/medical/data/100_199/0157.html" + }, + "requirements_checked": [ + "conservative_therapy_weeks", + "neuro_red_flags_documented", + "prior_imaging_result", + "symptom_duration_weeks" + ], + "rulebook_active_release_id": "2026-04-09-active-v0.5", + "rules_version": "0.5", + "run_id": "ceb2430a-f92a-4c57-9eca-27e0ad7bdcf7", + "site_of_care": "outpatient", + "specialty": "Primary Care", + "submission_readiness": false, + "timestamp_utc": "2026-04-09T15:24:39Z" + }, + "letter_draft": "", + "met_count": 2, + "not_documented_count": 0, + "not_met_count": 2, + "readiness_score": 75, + "results": [ + { + "evidence": "PT/NSAIDs/activity modification trial documented", + "evidence_snippets": [ + "PT x 5 weeks" + ], + "evidence_spans": [ + { + "end": 37, + "start": 25, + "text": "PT x 5 weeks" + } + ], + "key": "conservative_therapy_weeks", + "label": "Conservative therapy duration (weeks)", + "reason": "Documented value (5) below requirement (>= 6.0). Clarify or justify.", + "status": "NOT_MET" + }, + { + "evidence": "Weakness, saddle anesthesia, bowel/bladder changes explicitly documented as present or denied", + "evidence_snippets": [ + "Denies weakness. Denies bowel/bladder" + ], + "evidence_spans": [ + { + "end": 87, + "start": 50, + "text": "Denies weakness. Denies bowel/bladder" + } + ], + "key": "neuro_red_flags_documented", + "label": "Neuro red flags explicitly addressed (present or denied)", + "reason": "Explicitly addressed in documentation (present/affirmed).", + "status": "MET" + }, + { + "evidence": "X-ray/CT results or note of none", + "evidence_snippets": [ + "No prior imaging" + ], + "evidence_spans": [ + { + "end": 113, + "start": 97, + "text": "No prior imaging" + } + ], + "key": "prior_imaging_result", + "label": "Prior imaging result", + "reason": "Documented: none.", + "status": "MET" + }, + { + "evidence": "Duration supports guideline-based escalation", + "evidence_snippets": [ + "5 weeks" + ], + "evidence_spans": [ + { + "end": 23, + "start": 16, + "text": "5 weeks" + } + ], + "key": "symptom_duration_weeks", + "label": "Symptom duration (weeks)", + "reason": "Documented value (5) below requirement (>= 6.0). Clarify or justify.", + "status": "NOT_MET" + } + ], + "rule_reasons": [ + "Conservative therapy duration (weeks): NOT_MET \u2014 Documented value (5) below requirement (>= 6.0). Clarify or justify.", + "Symptom duration (weeks): NOT_MET \u2014 Documented value (5) below requirement (>= 6.0). Clarify or justify." + ] + }, + "request": { + "dx_codes": [ + "M54.5" + ], + "note_text": "Low back pain x 5 weeks. PT x 5 weeks documented. Denies weakness. Denies bowel/bladder changes. No prior imaging.", + "payer": "Aetna", + "procedure_code": "MRI_LUMBAR", + "site_of_care": "outpatient", + "specialty": "Primary Care" + }, + "results": [ + { + "evidence": "PT/NSAIDs/activity modification trial documented", + "evidence_snippets": [ + "PT x 5 weeks" + ], + "evidence_spans": [ + { + "end": 37, + "start": 25, + "text": "PT x 5 weeks" + } + ], + "key": "conservative_therapy_weeks", + "label": "Conservative therapy duration (weeks)", + "reason": "Documented value (5) below requirement (>= 6.0). Clarify or justify.", + "status": "NOT_MET" + }, + { + "evidence": "Weakness, saddle anesthesia, bowel/bladder changes explicitly documented as present or denied", + "evidence_snippets": [ + "Denies weakness. Denies bowel/bladder" + ], + "evidence_spans": [ + { + "end": 87, + "start": 50, + "text": "Denies weakness. Denies bowel/bladder" + } + ], + "key": "neuro_red_flags_documented", + "label": "Neuro red flags explicitly addressed (present or denied)", + "reason": "Explicitly addressed in documentation (present/affirmed).", + "status": "MET" + }, + { + "evidence": "X-ray/CT results or note of none", + "evidence_snippets": [ + "No prior imaging" + ], + "evidence_spans": [ + { + "end": 113, + "start": 97, + "text": "No prior imaging" + } + ], + "key": "prior_imaging_result", + "label": "Prior imaging result", + "reason": "Documented: none.", + "status": "MET" + }, + { + "evidence": "Duration supports guideline-based escalation", + "evidence_snippets": [ + "5 weeks" + ], + "evidence_spans": [ + { + "end": 23, + "start": 16, + "text": "5 weeks" + } + ], + "key": "symptom_duration_weeks", + "label": "Symptom duration (weeks)", + "reason": "Documented value (5) below requirement (>= 6.0). Clarify or justify.", + "status": "NOT_MET" + } + ], + "rule_reasons": [ + "Conservative therapy duration (weeks): NOT_MET \u2014 Documented value (5) below requirement (>= 6.0). Clarify or justify.", + "Symptom duration (weeks): NOT_MET \u2014 Documented value (5) below requirement (>= 6.0). Clarify or justify." + ], + "submission_readiness": false, + "supported_procedure": { + "display_name": "MRI Lumbar Spine (no contrast)", + "metadata": { + "category": "advanced_imaging", + "last_rule_update": "2026-04-09", + "notes": [ + "Demo rule set only; not a substitute for payer policy review.", + "Requires conservative therapy, symptom duration, imaging context, and explicit red-flag review." + ], + "rule_family": "spine_mri_conservative_therapy", + "summary": "Administrative readiness check for lumbar spine MRI requests using a narrow deterministic evidence contract.", + "supported_sites": [ + "outpatient" + ] + }, + "monitored_for_drift": true, + "payer": "Aetna", + "policy_trust_level": "demo", + "procedure_code": "MRI_LUMBAR", + "provenance": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": "daily", + "monitored_source_id": "aetna_mri_lumbar", + "monitored_source_name": "Aetna CPB 0157", + "monitored_source_owner": "NickLeko", + "monitored_source_url": "https://www.aetna.com/cpb/medical/data/100_199/0157.html", + "notes": "Rules are illustrative for demo. Not a substitute for payer policy documents.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of spine MRI administrative criteria", + "source_name": "Aetna spine MRI policy summary", + "source_type": "manual_summary", + "source_url": "https://www.aetna.com/cpb/medical/data/100_199/0157.html" + }, + "required_field_keys": [ + "conservative_therapy_weeks", + "neuro_red_flags_documented", + "prior_imaging_result", + "symptom_duration_weeks" + ], + "requirements": [ + { + "allowed": [], + "evidence": "PT/NSAIDs/activity modification trial documented", + "key": "conservative_therapy_weeks", + "label": "Conservative therapy duration (weeks)", + "min": 6.0, + "type": "number" + }, + { + "allowed": [], + "evidence": "Weakness, saddle anesthesia, bowel/bladder changes explicitly documented as present or denied", + "key": "neuro_red_flags_documented", + "label": "Neuro red flags explicitly addressed (present or denied)", + "min": null, + "type": "boolean" + }, + { + "allowed": [ + "none", + "inconclusive", + "abnormal" + ], + "evidence": "X-ray/CT results or note of none", + "key": "prior_imaging_result", + "label": "Prior imaging result", + "min": null, + "type": "enum" + }, + { + "allowed": [], + "evidence": "Duration supports guideline-based escalation", + "key": "symptom_duration_weeks", + "label": "Symptom duration (weeks)", + "min": 6.0, + "type": "number" + } + ] + }, + "warnings": [ + "Policy trust remains DEMO for this procedure. Verify against official policy before real-world use." + ] +} diff --git a/docs/artifacts/MRI-CERV-01-ready.json b/docs/artifacts/MRI-CERV-01-ready.json new file mode 100644 index 0000000..1e78962 --- /dev/null +++ b/docs/artifacts/MRI-CERV-01-ready.json @@ -0,0 +1,508 @@ +{ + "audit_trail": { + "blocking_issues": { + "not_documented": [], + "not_met": [] + }, + "evaluation_warnings": [ + "Policy trust remains DEMO for this procedure. Verify against official policy before real-world use." + ], + "evidence_map": { + "conservative_therapy_weeks": [ + { + "end": 64, + "start": 50, + "text": "PT for 8 weeks" + } + ], + "neuro_red_flags_documented": [ + { + "end": 150, + "start": 113, + "text": "Denies weakness. Denies bowel/bladder" + } + ], + "prior_imaging_result": [ + { + "end": 179, + "start": 175, + "text": "xray" + } + ], + "symptom_duration_weeks": [ + { + "end": 48, + "start": 41, + "text": "8 weeks" + } + ] + }, + "facts_extracted": { + "ahi_documented": null, + "conservative_therapy_weeks": 8, + "mechanical_symptoms_documented": null, + "neuro_red_flags_documented": true, + "osa_diagnosis": null, + "prior_imaging_result": "inconclusive", + "sleep_study_date": null, + "symptom_duration_weeks": 8 + }, + "invariant_errors": [], + "metrics": { + "compliance_rate": 100.0, + "compliant_count": 4, + "extraction_failure_count": 0, + "extraction_success_rate": 100.0, + "non_compliant_count": 0 + }, + "note_hash": "449f7c6ef1838637", + "note_length": 193, + "overall_status": "READY", + "payer": "Aetna", + "policy_trust_level": "demo", + "procedure_code": "MRI_CERVICAL", + "procedure_name": "MRI Cervical Spine (no contrast)", + "provenance_snapshot": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": null, + "monitored_source_id": null, + "monitored_source_name": null, + "monitored_source_owner": null, + "monitored_source_url": null, + "notes": "Rules are illustrative for demo and intentionally mirror the narrow spine MRI contract.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of cervical spine MRI administrative criteria", + "source_name": "Aetna cervical spine MRI policy summary", + "source_type": "manual_summary", + "source_url": null + }, + "requirements_checked": [ + "conservative_therapy_weeks", + "neuro_red_flags_documented", + "prior_imaging_result", + "symptom_duration_weeks" + ], + "rulebook_active_release_id": "2026-04-09-active-v0.5", + "rules_version": "0.5", + "run_id": "e3e18d4f-137e-4b42-94f8-2f35c1a29f17", + "site_of_care": "outpatient", + "specialty": "Orthopedics", + "submission_readiness": true, + "timestamp_utc": "2026-04-09T15:24:39Z" + }, + "blockers": { + "not_documented": [], + "not_met": [] + }, + "evidence_map": { + "conservative_therapy_weeks": [ + { + "end": 64, + "start": 50, + "text": "PT for 8 weeks" + } + ], + "neuro_red_flags_documented": [ + { + "end": 150, + "start": 113, + "text": "Denies weakness. Denies bowel/bladder" + } + ], + "prior_imaging_result": [ + { + "end": 179, + "start": 175, + "text": "xray" + } + ], + "symptom_duration_weeks": [ + { + "end": 48, + "start": 41, + "text": "8 weeks" + } + ] + }, + "facts": { + "ahi_documented": null, + "conservative_therapy_weeks": 8, + "mechanical_symptoms_documented": null, + "neuro_red_flags_documented": true, + "osa_diagnosis": null, + "prior_imaging_result": "inconclusive", + "sleep_study_date": null, + "symptom_duration_weeks": 8 + }, + "letter": { + "metadata": { + "cited_snippets_count": 4, + "contains_missing_documentation": false, + "draft_blocked": false, + "draft_blocked_reasons": [], + "generated_timestamp_utc": "2026-04-09T15:24:39Z", + "letter_hash_sha256_16": "3af85eb36469835a", + "letter_type": "submission_cover_letter", + "letter_version": "1.1", + "overall_status": "READY", + "policy_trust_level": "demo" + }, + "text": "PRIOR AUTHORIZATION ADMINISTRATIVE READINESS SUMMARY\n\nPayer: Aetna\nProcedure: MRI_CERVICAL\nSite of care: outpatient\nSpecialty: Orthopedics\nGenerated: 2026-04-09T15:24:39Z\nDx codes: M54.12\nPolicy trust level: DEMO \u2014 criteria are illustrative only. Verify against the official payer policy before submission.\n\nOverall Status: READY\n\nSummary:\nThis letter supports administrative submission readiness based on the documentation present in the record. This does not guarantee payer approval.\n\nRequirements:\n- Conservative therapy duration (weeks) (conservative_therapy_weeks): MET\n Reason: Documented value: 8.\n Evidence:\n - \"PT for 8 weeks\"\n- Neuro red flags explicitly addressed (present or denied) (neuro_red_flags_documented): MET\n Reason: Explicitly addressed in documentation (present/affirmed).\n Evidence:\n - \"Denies weakness. Denies bowel/bladder\"\n- Prior imaging result (prior_imaging_result): MET\n Reason: Documented: inconclusive.\n Evidence:\n - \"xray\"\n- Symptom duration (weeks) (symptom_duration_weeks): MET\n Reason: Documented value: 8.\n Evidence:\n - \"8 weeks\"\nClosing:\nThis letter summarizes documentation-based administrative readiness for prior authorization submission. It does not provide clinical recommendations and does not predict approval outcomes.\n" + }, + "metrics": { + "compliance_rate": 100.0, + "compliant_count": 4, + "extraction_failure_count": 0, + "extraction_success_rate": 100.0, + "non_compliant_count": 0 + }, + "overall_status": "READY", + "policy_trust_level": "demo", + "provenance": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": null, + "monitored_source_id": null, + "monitored_source_name": null, + "monitored_source_owner": null, + "monitored_source_url": null, + "notes": "Rules are illustrative for demo and intentionally mirror the narrow spine MRI contract.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of cervical spine MRI administrative criteria", + "source_name": "Aetna cervical spine MRI policy summary", + "source_type": "manual_summary", + "source_url": null + }, + "readiness_score": 100, + "report": { + "audit_trail": { + "blocking_issues": { + "not_documented": [], + "not_met": [] + }, + "evaluation_warnings": [ + "Policy trust remains DEMO for this procedure. Verify against official policy before real-world use." + ], + "evidence_map": { + "conservative_therapy_weeks": [ + { + "end": 64, + "start": 50, + "text": "PT for 8 weeks" + } + ], + "neuro_red_flags_documented": [ + { + "end": 150, + "start": 113, + "text": "Denies weakness. Denies bowel/bladder" + } + ], + "prior_imaging_result": [ + { + "end": 179, + "start": 175, + "text": "xray" + } + ], + "symptom_duration_weeks": [ + { + "end": 48, + "start": 41, + "text": "8 weeks" + } + ] + }, + "facts_extracted": { + "ahi_documented": null, + "conservative_therapy_weeks": 8, + "mechanical_symptoms_documented": null, + "neuro_red_flags_documented": true, + "osa_diagnosis": null, + "prior_imaging_result": "inconclusive", + "sleep_study_date": null, + "symptom_duration_weeks": 8 + }, + "invariant_errors": [], + "metrics": { + "compliance_rate": 100.0, + "compliant_count": 4, + "extraction_failure_count": 0, + "extraction_success_rate": 100.0, + "non_compliant_count": 0 + }, + "note_hash": "449f7c6ef1838637", + "note_length": 193, + "overall_status": "READY", + "payer": "Aetna", + "policy_trust_level": "demo", + "procedure_code": "MRI_CERVICAL", + "procedure_name": "MRI Cervical Spine (no contrast)", + "provenance_snapshot": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": null, + "monitored_source_id": null, + "monitored_source_name": null, + "monitored_source_owner": null, + "monitored_source_url": null, + "notes": "Rules are illustrative for demo and intentionally mirror the narrow spine MRI contract.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of cervical spine MRI administrative criteria", + "source_name": "Aetna cervical spine MRI policy summary", + "source_type": "manual_summary", + "source_url": null + }, + "requirements_checked": [ + "conservative_therapy_weeks", + "neuro_red_flags_documented", + "prior_imaging_result", + "symptom_duration_weeks" + ], + "rulebook_active_release_id": "2026-04-09-active-v0.5", + "rules_version": "0.5", + "run_id": "e3e18d4f-137e-4b42-94f8-2f35c1a29f17", + "site_of_care": "outpatient", + "specialty": "Orthopedics", + "submission_readiness": true, + "timestamp_utc": "2026-04-09T15:24:39Z" + }, + "letter_draft": "", + "met_count": 4, + "not_documented_count": 0, + "not_met_count": 0, + "readiness_score": 100, + "results": [ + { + "evidence": "PT/NSAIDs/activity modification trial documented", + "evidence_snippets": [ + "PT for 8 weeks" + ], + "evidence_spans": [ + { + "end": 64, + "start": 50, + "text": "PT for 8 weeks" + } + ], + "key": "conservative_therapy_weeks", + "label": "Conservative therapy duration (weeks)", + "reason": "Documented value: 8.", + "status": "MET" + }, + { + "evidence": "Weakness, numbness, bowel/bladder changes, or related escalation findings explicitly documented", + "evidence_snippets": [ + "Denies weakness. Denies bowel/bladder" + ], + "evidence_spans": [ + { + "end": 150, + "start": 113, + "text": "Denies weakness. Denies bowel/bladder" + } + ], + "key": "neuro_red_flags_documented", + "label": "Neuro red flags explicitly addressed (present or denied)", + "reason": "Explicitly addressed in documentation (present/affirmed).", + "status": "MET" + }, + { + "evidence": "X-ray/CT results or note of none", + "evidence_snippets": [ + "xray" + ], + "evidence_spans": [ + { + "end": 179, + "start": 175, + "text": "xray" + } + ], + "key": "prior_imaging_result", + "label": "Prior imaging result", + "reason": "Documented: inconclusive.", + "status": "MET" + }, + { + "evidence": "Duration supports guideline-based escalation", + "evidence_snippets": [ + "8 weeks" + ], + "evidence_spans": [ + { + "end": 48, + "start": 41, + "text": "8 weeks" + } + ], + "key": "symptom_duration_weeks", + "label": "Symptom duration (weeks)", + "reason": "Documented value: 8.", + "status": "MET" + } + ], + "rule_reasons": [] + }, + "request": { + "dx_codes": [ + "M54.12" + ], + "note_text": "Neck pain with right arm radiculopathy x 8 weeks. PT for 8 weeks and NSAIDs documented with minimal improvement. Denies weakness. Denies bowel/bladder changes. Prior cervical xray inconclusive.", + "payer": "Aetna", + "procedure_code": "MRI_CERVICAL", + "site_of_care": "outpatient", + "specialty": "Orthopedics" + }, + "results": [ + { + "evidence": "PT/NSAIDs/activity modification trial documented", + "evidence_snippets": [ + "PT for 8 weeks" + ], + "evidence_spans": [ + { + "end": 64, + "start": 50, + "text": "PT for 8 weeks" + } + ], + "key": "conservative_therapy_weeks", + "label": "Conservative therapy duration (weeks)", + "reason": "Documented value: 8.", + "status": "MET" + }, + { + "evidence": "Weakness, numbness, bowel/bladder changes, or related escalation findings explicitly documented", + "evidence_snippets": [ + "Denies weakness. Denies bowel/bladder" + ], + "evidence_spans": [ + { + "end": 150, + "start": 113, + "text": "Denies weakness. Denies bowel/bladder" + } + ], + "key": "neuro_red_flags_documented", + "label": "Neuro red flags explicitly addressed (present or denied)", + "reason": "Explicitly addressed in documentation (present/affirmed).", + "status": "MET" + }, + { + "evidence": "X-ray/CT results or note of none", + "evidence_snippets": [ + "xray" + ], + "evidence_spans": [ + { + "end": 179, + "start": 175, + "text": "xray" + } + ], + "key": "prior_imaging_result", + "label": "Prior imaging result", + "reason": "Documented: inconclusive.", + "status": "MET" + }, + { + "evidence": "Duration supports guideline-based escalation", + "evidence_snippets": [ + "8 weeks" + ], + "evidence_spans": [ + { + "end": 48, + "start": 41, + "text": "8 weeks" + } + ], + "key": "symptom_duration_weeks", + "label": "Symptom duration (weeks)", + "reason": "Documented value: 8.", + "status": "MET" + } + ], + "rule_reasons": [], + "submission_readiness": true, + "supported_procedure": { + "display_name": "MRI Cervical Spine (no contrast)", + "metadata": { + "category": "advanced_imaging", + "last_rule_update": "2026-04-09", + "notes": [ + "Intentionally mirrors the lumbar MRI demo pathway to show reusable deterministic architecture.", + "Human review remains required before any real submission." + ], + "rule_family": "spine_mri_conservative_therapy", + "summary": "Administrative readiness check for cervical spine MRI requests using the same narrow deterministic evidence contract as lumbar MRI.", + "supported_sites": [ + "outpatient" + ] + }, + "monitored_for_drift": false, + "payer": "Aetna", + "policy_trust_level": "demo", + "procedure_code": "MRI_CERVICAL", + "provenance": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": null, + "monitored_source_id": null, + "monitored_source_name": null, + "monitored_source_owner": null, + "monitored_source_url": null, + "notes": "Rules are illustrative for demo and intentionally mirror the narrow spine MRI contract.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of cervical spine MRI administrative criteria", + "source_name": "Aetna cervical spine MRI policy summary", + "source_type": "manual_summary", + "source_url": null + }, + "required_field_keys": [ + "conservative_therapy_weeks", + "neuro_red_flags_documented", + "prior_imaging_result", + "symptom_duration_weeks" + ], + "requirements": [ + { + "allowed": [], + "evidence": "PT/NSAIDs/activity modification trial documented", + "key": "conservative_therapy_weeks", + "label": "Conservative therapy duration (weeks)", + "min": 6.0, + "type": "number" + }, + { + "allowed": [], + "evidence": "Weakness, numbness, bowel/bladder changes, or related escalation findings explicitly documented", + "key": "neuro_red_flags_documented", + "label": "Neuro red flags explicitly addressed (present or denied)", + "min": null, + "type": "boolean" + }, + { + "allowed": [ + "none", + "inconclusive", + "abnormal" + ], + "evidence": "X-ray/CT results or note of none", + "key": "prior_imaging_result", + "label": "Prior imaging result", + "min": null, + "type": "enum" + }, + { + "allowed": [], + "evidence": "Duration supports guideline-based escalation", + "key": "symptom_duration_weeks", + "label": "Symptom duration (weeks)", + "min": 6.0, + "type": "number" + } + ] + }, + "warnings": [ + "Policy trust remains DEMO for this procedure. Verify against official policy before real-world use." + ] +} diff --git a/docs/artifacts/MRI-KNEE-01-ready.json b/docs/artifacts/MRI-KNEE-01-ready.json new file mode 100644 index 0000000..5f9cbeb --- /dev/null +++ b/docs/artifacts/MRI-KNEE-01-ready.json @@ -0,0 +1,507 @@ +{ + "audit_trail": { + "blocking_issues": { + "not_documented": [], + "not_met": [] + }, + "evaluation_warnings": [ + "Policy trust remains DEMO for this procedure. Verify against official policy before real-world use." + ], + "evidence_map": { + "conservative_therapy_weeks": [ + { + "end": 77, + "start": 63, + "text": "PT for 8 weeks" + } + ], + "mechanical_symptoms_documented": [ + { + "end": 41, + "start": 16, + "text": "with locking and catching" + } + ], + "prior_imaging_result": [ + { + "end": 137, + "start": 126, + "text": "xray normal" + } + ], + "symptom_duration_weeks": [ + { + "end": 51, + "start": 44, + "text": "8 weeks" + } + ] + }, + "facts_extracted": { + "ahi_documented": null, + "conservative_therapy_weeks": 8, + "mechanical_symptoms_documented": true, + "neuro_red_flags_documented": null, + "osa_diagnosis": null, + "prior_imaging_result": "inconclusive", + "sleep_study_date": null, + "symptom_duration_weeks": 8 + }, + "invariant_errors": [], + "metrics": { + "compliance_rate": 100.0, + "compliant_count": 4, + "extraction_failure_count": 0, + "extraction_success_rate": 100.0, + "non_compliant_count": 0 + }, + "note_hash": "59421e8561fe0462", + "note_length": 151, + "overall_status": "READY", + "payer": "Aetna", + "policy_trust_level": "demo", + "procedure_code": "MRI_KNEE", + "procedure_name": "MRI Knee (no contrast)", + "provenance_snapshot": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": null, + "monitored_source_id": null, + "monitored_source_name": null, + "monitored_source_owner": null, + "monitored_source_url": null, + "notes": "Rules are illustrative for demo and intentionally limited to documentation completeness.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of knee MRI administrative documentation criteria", + "source_name": "Aetna knee MRI policy summary", + "source_type": "manual_summary", + "source_url": null + }, + "requirements_checked": [ + "conservative_therapy_weeks", + "symptom_duration_weeks", + "prior_imaging_result", + "mechanical_symptoms_documented" + ], + "rulebook_active_release_id": "2026-04-09-active-v0.5", + "rules_version": "0.5", + "run_id": "b51fa27b-65d0-4826-bd84-9ae81c89d3d6", + "site_of_care": "outpatient", + "specialty": "Orthopedics", + "submission_readiness": true, + "timestamp_utc": "2026-04-09T15:24:39Z" + }, + "blockers": { + "not_documented": [], + "not_met": [] + }, + "evidence_map": { + "conservative_therapy_weeks": [ + { + "end": 77, + "start": 63, + "text": "PT for 8 weeks" + } + ], + "mechanical_symptoms_documented": [ + { + "end": 41, + "start": 16, + "text": "with locking and catching" + } + ], + "prior_imaging_result": [ + { + "end": 137, + "start": 126, + "text": "xray normal" + } + ], + "symptom_duration_weeks": [ + { + "end": 51, + "start": 44, + "text": "8 weeks" + } + ] + }, + "facts": { + "ahi_documented": null, + "conservative_therapy_weeks": 8, + "mechanical_symptoms_documented": true, + "neuro_red_flags_documented": null, + "osa_diagnosis": null, + "prior_imaging_result": "inconclusive", + "sleep_study_date": null, + "symptom_duration_weeks": 8 + }, + "letter": { + "metadata": { + "cited_snippets_count": 4, + "contains_missing_documentation": false, + "draft_blocked": false, + "draft_blocked_reasons": [], + "generated_timestamp_utc": "2026-04-09T15:24:39Z", + "letter_hash_sha256_16": "f82cadf34fa88699", + "letter_type": "submission_cover_letter", + "letter_version": "1.1", + "overall_status": "READY", + "policy_trust_level": "demo" + }, + "text": "PRIOR AUTHORIZATION ADMINISTRATIVE READINESS SUMMARY\n\nPayer: Aetna\nProcedure: MRI_KNEE\nSite of care: outpatient\nSpecialty: Orthopedics\nGenerated: 2026-04-09T15:24:39Z\nDx codes: M25.561\nPolicy trust level: DEMO \u2014 criteria are illustrative only. Verify against the official payer policy before submission.\n\nOverall Status: READY\n\nSummary:\nThis letter supports administrative submission readiness based on the documentation present in the record. This does not guarantee payer approval.\n\nRequirements:\n- Conservative therapy duration (weeks) (conservative_therapy_weeks): MET\n Reason: Documented value: 8.\n Evidence:\n - \"PT for 8 weeks\"\n- Symptom duration (weeks) (symptom_duration_weeks): MET\n Reason: Documented value: 8.\n Evidence:\n - \"8 weeks\"\n- Prior imaging result (prior_imaging_result): MET\n Reason: Documented: inconclusive.\n Evidence:\n - \"xray normal\"\n- Mechanical symptoms explicitly addressed (present or denied) (mechanical_symptoms_documented): MET\n Reason: Explicitly addressed in documentation (present/affirmed).\n Evidence:\n - \"with locking and catching\"\nClosing:\nThis letter summarizes documentation-based administrative readiness for prior authorization submission. It does not provide clinical recommendations and does not predict approval outcomes.\n" + }, + "metrics": { + "compliance_rate": 100.0, + "compliant_count": 4, + "extraction_failure_count": 0, + "extraction_success_rate": 100.0, + "non_compliant_count": 0 + }, + "overall_status": "READY", + "policy_trust_level": "demo", + "provenance": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": null, + "monitored_source_id": null, + "monitored_source_name": null, + "monitored_source_owner": null, + "monitored_source_url": null, + "notes": "Rules are illustrative for demo and intentionally limited to documentation completeness.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of knee MRI administrative documentation criteria", + "source_name": "Aetna knee MRI policy summary", + "source_type": "manual_summary", + "source_url": null + }, + "readiness_score": 100, + "report": { + "audit_trail": { + "blocking_issues": { + "not_documented": [], + "not_met": [] + }, + "evaluation_warnings": [ + "Policy trust remains DEMO for this procedure. Verify against official policy before real-world use." + ], + "evidence_map": { + "conservative_therapy_weeks": [ + { + "end": 77, + "start": 63, + "text": "PT for 8 weeks" + } + ], + "mechanical_symptoms_documented": [ + { + "end": 41, + "start": 16, + "text": "with locking and catching" + } + ], + "prior_imaging_result": [ + { + "end": 137, + "start": 126, + "text": "xray normal" + } + ], + "symptom_duration_weeks": [ + { + "end": 51, + "start": 44, + "text": "8 weeks" + } + ] + }, + "facts_extracted": { + "ahi_documented": null, + "conservative_therapy_weeks": 8, + "mechanical_symptoms_documented": true, + "neuro_red_flags_documented": null, + "osa_diagnosis": null, + "prior_imaging_result": "inconclusive", + "sleep_study_date": null, + "symptom_duration_weeks": 8 + }, + "invariant_errors": [], + "metrics": { + "compliance_rate": 100.0, + "compliant_count": 4, + "extraction_failure_count": 0, + "extraction_success_rate": 100.0, + "non_compliant_count": 0 + }, + "note_hash": "59421e8561fe0462", + "note_length": 151, + "overall_status": "READY", + "payer": "Aetna", + "policy_trust_level": "demo", + "procedure_code": "MRI_KNEE", + "procedure_name": "MRI Knee (no contrast)", + "provenance_snapshot": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": null, + "monitored_source_id": null, + "monitored_source_name": null, + "monitored_source_owner": null, + "monitored_source_url": null, + "notes": "Rules are illustrative for demo and intentionally limited to documentation completeness.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of knee MRI administrative documentation criteria", + "source_name": "Aetna knee MRI policy summary", + "source_type": "manual_summary", + "source_url": null + }, + "requirements_checked": [ + "conservative_therapy_weeks", + "symptom_duration_weeks", + "prior_imaging_result", + "mechanical_symptoms_documented" + ], + "rulebook_active_release_id": "2026-04-09-active-v0.5", + "rules_version": "0.5", + "run_id": "b51fa27b-65d0-4826-bd84-9ae81c89d3d6", + "site_of_care": "outpatient", + "specialty": "Orthopedics", + "submission_readiness": true, + "timestamp_utc": "2026-04-09T15:24:39Z" + }, + "letter_draft": "", + "met_count": 4, + "not_documented_count": 0, + "not_met_count": 0, + "readiness_score": 100, + "results": [ + { + "evidence": "PT/activity modification/NSAID trial documented", + "evidence_snippets": [ + "PT for 8 weeks" + ], + "evidence_spans": [ + { + "end": 77, + "start": 63, + "text": "PT for 8 weeks" + } + ], + "key": "conservative_therapy_weeks", + "label": "Conservative therapy duration (weeks)", + "reason": "Documented value: 8.", + "status": "MET" + }, + { + "evidence": "Persistent symptoms documented long enough to justify escalation", + "evidence_snippets": [ + "8 weeks" + ], + "evidence_spans": [ + { + "end": 51, + "start": 44, + "text": "8 weeks" + } + ], + "key": "symptom_duration_weeks", + "label": "Symptom duration (weeks)", + "reason": "Documented value: 8.", + "status": "MET" + }, + { + "evidence": "Prior knee imaging such as x-ray is documented and not simply absent", + "evidence_snippets": [ + "xray normal" + ], + "evidence_spans": [ + { + "end": 137, + "start": 126, + "text": "xray normal" + } + ], + "key": "prior_imaging_result", + "label": "Prior imaging result", + "reason": "Documented: inconclusive.", + "status": "MET" + }, + { + "evidence": "Locking, catching, buckling, giving way, or instability explicitly documented", + "evidence_snippets": [ + "with locking and catching" + ], + "evidence_spans": [ + { + "end": 41, + "start": 16, + "text": "with locking and catching" + } + ], + "key": "mechanical_symptoms_documented", + "label": "Mechanical symptoms explicitly addressed (present or denied)", + "reason": "Explicitly addressed in documentation (present/affirmed).", + "status": "MET" + } + ], + "rule_reasons": [] + }, + "request": { + "dx_codes": [ + "M25.561" + ], + "note_text": "Right knee pain with locking and catching x 8 weeks. Completed PT for 8 weeks and NSAIDs with minimal improvement. Prior knee xray normal/unremarkable.", + "payer": "Aetna", + "procedure_code": "MRI_KNEE", + "site_of_care": "outpatient", + "specialty": "Orthopedics" + }, + "results": [ + { + "evidence": "PT/activity modification/NSAID trial documented", + "evidence_snippets": [ + "PT for 8 weeks" + ], + "evidence_spans": [ + { + "end": 77, + "start": 63, + "text": "PT for 8 weeks" + } + ], + "key": "conservative_therapy_weeks", + "label": "Conservative therapy duration (weeks)", + "reason": "Documented value: 8.", + "status": "MET" + }, + { + "evidence": "Persistent symptoms documented long enough to justify escalation", + "evidence_snippets": [ + "8 weeks" + ], + "evidence_spans": [ + { + "end": 51, + "start": 44, + "text": "8 weeks" + } + ], + "key": "symptom_duration_weeks", + "label": "Symptom duration (weeks)", + "reason": "Documented value: 8.", + "status": "MET" + }, + { + "evidence": "Prior knee imaging such as x-ray is documented and not simply absent", + "evidence_snippets": [ + "xray normal" + ], + "evidence_spans": [ + { + "end": 137, + "start": 126, + "text": "xray normal" + } + ], + "key": "prior_imaging_result", + "label": "Prior imaging result", + "reason": "Documented: inconclusive.", + "status": "MET" + }, + { + "evidence": "Locking, catching, buckling, giving way, or instability explicitly documented", + "evidence_snippets": [ + "with locking and catching" + ], + "evidence_spans": [ + { + "end": 41, + "start": 16, + "text": "with locking and catching" + } + ], + "key": "mechanical_symptoms_documented", + "label": "Mechanical symptoms explicitly addressed (present or denied)", + "reason": "Explicitly addressed in documentation (present/affirmed).", + "status": "MET" + } + ], + "rule_reasons": [], + "submission_readiness": true, + "supported_procedure": { + "display_name": "MRI Knee (no contrast)", + "metadata": { + "category": "advanced_imaging", + "last_rule_update": "2026-04-09", + "notes": [ + "This demo pathway checks only administrative completeness, not orthopedic appropriateness.", + "Requires symptom duration, conservative therapy, prior imaging context, and explicit review of mechanical symptoms." + ], + "rule_family": "extremity_mri_conservative_therapy", + "summary": "Administrative readiness check for knee MRI requests using a narrow deterministic documentation contract.", + "supported_sites": [ + "outpatient" + ] + }, + "monitored_for_drift": false, + "payer": "Aetna", + "policy_trust_level": "demo", + "procedure_code": "MRI_KNEE", + "provenance": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": null, + "monitored_source_id": null, + "monitored_source_name": null, + "monitored_source_owner": null, + "monitored_source_url": null, + "notes": "Rules are illustrative for demo and intentionally limited to documentation completeness.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of knee MRI administrative documentation criteria", + "source_name": "Aetna knee MRI policy summary", + "source_type": "manual_summary", + "source_url": null + }, + "required_field_keys": [ + "conservative_therapy_weeks", + "symptom_duration_weeks", + "prior_imaging_result", + "mechanical_symptoms_documented" + ], + "requirements": [ + { + "allowed": [], + "evidence": "PT/activity modification/NSAID trial documented", + "key": "conservative_therapy_weeks", + "label": "Conservative therapy duration (weeks)", + "min": 6.0, + "type": "number" + }, + { + "allowed": [], + "evidence": "Persistent symptoms documented long enough to justify escalation", + "key": "symptom_duration_weeks", + "label": "Symptom duration (weeks)", + "min": 6.0, + "type": "number" + }, + { + "allowed": [ + "inconclusive", + "abnormal" + ], + "evidence": "Prior knee imaging such as x-ray is documented and not simply absent", + "key": "prior_imaging_result", + "label": "Prior imaging result", + "min": null, + "type": "enum" + }, + { + "allowed": [], + "evidence": "Locking, catching, buckling, giving way, or instability explicitly documented", + "key": "mechanical_symptoms_documented", + "label": "Mechanical symptoms explicitly addressed (present or denied)", + "min": null, + "type": "boolean" + } + ] + }, + "warnings": [ + "Policy trust remains DEMO for this procedure. Verify against official policy before real-world use." + ] +} diff --git a/docs/artifacts/demo_cases.json b/docs/artifacts/demo_cases.json new file mode 100644 index 0000000..2498159 --- /dev/null +++ b/docs/artifacts/demo_cases.json @@ -0,0 +1,518 @@ +[ + { + "dx_codes": [ + "M54.16" + ], + "expected_label": "complete", + "id": "MRI-01-complete", + "note_text": "Low back pain with right leg radiculopathy x 8 weeks. Completed PT for 8 weeks and NSAIDs with minimal improvement. Denies bowel/bladder incontinence. No saddle anesthesia. Prior imaging: lumbar xray inconclusive. Neuro exam: mild weakness dorsiflexion 4/5.", + "payer": "Aetna", + "procedure_code": "MRI_LUMBAR", + "showcase": { + "description": "A clean lumbar MRI request with therapy duration, symptom duration, imaging context, and red-flag documentation all present.", + "expected_overall_status": "READY", + "featured": true, + "scenario_type": "Ready path", + "sort_order": 1, + "tags": [ + "spine MRI", + "happy path", + "evidence map" + ], + "title": "Lumbar MRI ready for administrative review", + "why_interesting": "Shows the straight-line READY path with evidence snippets across each requirement." + }, + "site_of_care": "outpatient", + "specialty": "Orthopedics" + }, + { + "dx_codes": [ + "M51.26" + ], + "expected_label": "complete", + "id": "MRI-02-complete", + "note_text": "Back pain x 2 months. Trial of activity modification + NSAIDs and PT x 6 weeks documented, persistent symptoms. Reports progressive weakness. Prior CT described as abnormal with disc bulge/stenosis.", + "payer": "Aetna", + "procedure_code": "MRI_LUMBAR", + "showcase": {}, + "site_of_care": "outpatient", + "specialty": "Primary Care" + }, + { + "dx_codes": [ + "M54.5" + ], + "expected_label": "incomplete", + "id": "MRI-03-borderline", + "note_text": "Low back pain x 6 weeks. Tried PT for 6 weeks. No imaging yet. No red flags mentioned.", + "payer": "Aetna", + "procedure_code": "MRI_LUMBAR", + "showcase": {}, + "site_of_care": "outpatient", + "specialty": "Primary Care" + }, + { + "dx_codes": [ + "M54.16" + ], + "expected_label": "complete", + "id": "MRI-04-borderline", + "note_text": "Back pain x 3 months. PT x 10 weeks with no improvement. Imaging noted but result unclear. Denies weakness. Denies bowel/bladder changes.", + "payer": "Aetna", + "procedure_code": "MRI_LUMBAR", + "showcase": {}, + "site_of_care": "outpatient", + "specialty": "Orthopedics" + }, + { + "dx_codes": [ + "M54.5" + ], + "expected_label": "incomplete", + "id": "MRI-05-incomplete", + "note_text": "Back pain. Wants MRI. No duration provided. No mention of prior treatment or imaging.", + "payer": "Aetna", + "procedure_code": "MRI_LUMBAR", + "showcase": {}, + "site_of_care": "outpatient", + "specialty": "Primary Care" + }, + { + "dx_codes": [ + "M54.16" + ], + "expected_label": "incomplete", + "id": "MRI-06-incomplete", + "note_text": "Radicular pain x 4 weeks. PT for 4 weeks. No imaging documented. No red flags documented.", + "payer": "Aetna", + "procedure_code": "MRI_LUMBAR", + "showcase": {}, + "site_of_care": "outpatient", + "specialty": "Orthopedics" + }, + { + "dx_codes": [ + "G47.33" + ], + "expected_label": "complete", + "id": "CPAP-01-complete", + "note_text": "Dx: OSA. Sleep study completed 2024-05-18. AHI 22 documented. Requests CPAP E0601.", + "payer": "Aetna", + "procedure_code": "CPAP_DEVICE", + "showcase": {}, + "site_of_care": "outpatient", + "specialty": "Sleep Medicine" + }, + { + "dx_codes": [ + "G47.33" + ], + "expected_label": "incomplete", + "id": "CPAP-02-borderline", + "note_text": "OSA noted in problem list. Sleep study in 2023 mentioned but no date. AHI not documented.", + "payer": "Aetna", + "procedure_code": "CPAP_DEVICE", + "showcase": { + "description": "The note mentions OSA and a past sleep study, but it leaves out the date and AHI details required to determine readiness.", + "expected_overall_status": "CANNOT_DETERMINE", + "featured": true, + "scenario_type": "Refusal-first", + "sort_order": 3, + "tags": [ + "DME", + "missing documentation", + "cannot determine" + ], + "title": "CPAP request refused for missing documentation", + "why_interesting": "Shows refusal-first behavior when required evidence is missing instead of inferred." + }, + "site_of_care": "outpatient", + "specialty": "Primary Care" + }, + { + "dx_codes": [], + "expected_label": "incomplete", + "id": "CPAP-03-incomplete", + "note_text": "Snoring and fatigue. Request CPAP.", + "payer": "Aetna", + "procedure_code": "CPAP_DEVICE", + "showcase": {}, + "site_of_care": "outpatient", + "specialty": "Primary Care" + }, + { + "dx_codes": [ + "G47.33" + ], + "expected_label": "incomplete", + "id": "CPAP-04-incomplete", + "note_text": "OSA. AHI documented. Sleep study date not documented.", + "payer": "Aetna", + "procedure_code": "CPAP_DEVICE", + "showcase": {}, + "site_of_care": "outpatient", + "specialty": "Sleep Medicine" + }, + { + "dx_codes": [ + "M54.16" + ], + "expected_label": "complete", + "id": "MRI-07-edge-exact-threshold", + "note_text": "Low back pain x 6 weeks exactly. PT x 6 weeks and NSAIDs documented without improvement. Denies weakness. Denies bowel/bladder changes. No saddle anesthesia. No prior imaging.", + "payer": "Aetna", + "procedure_code": "MRI_LUMBAR", + "showcase": {}, + "site_of_care": "outpatient", + "specialty": "Primary Care" + }, + { + "dx_codes": [ + "M54.5" + ], + "expected_label": "incomplete", + "id": "MRI-08-edge-below-threshold", + "note_text": "Low back pain x 5 weeks. PT x 5 weeks documented. Denies weakness. Denies bowel/bladder changes. No prior imaging.", + "payer": "Aetna", + "procedure_code": "MRI_LUMBAR", + "showcase": { + "description": "This note documents the right fields, but both symptom duration and therapy duration are still below threshold.", + "expected_overall_status": "NOT_READY", + "featured": true, + "scenario_type": "Documented failure", + "sort_order": 2, + "tags": [ + "threshold miss", + "not ready", + "fully documented" + ], + "title": "Documented, but not yet ready to submit", + "why_interesting": "Shows the difference between documented failure and missingness." + }, + "site_of_care": "outpatient", + "specialty": "Primary Care" + }, + { + "dx_codes": [ + "M54.16" + ], + "expected_label": "complete", + "id": "MRI-09-negation-variant", + "note_text": "Back pain x 8 weeks. PT for 8 weeks with minimal relief. Weakness absent. Bowel/bladder function intact. No saddle anesthesia. No prior imaging yet.", + "payer": "Aetna", + "procedure_code": "MRI_LUMBAR", + "showcase": {}, + "site_of_care": "outpatient", + "specialty": "Orthopedics" + }, + { + "dx_codes": [ + "M54.5" + ], + "expected_label": "incomplete", + "id": "MRI-10-doc-gap-redflags", + "note_text": "Low back pain x 8 weeks. PT x 8 weeks documented. No prior imaging. No red flags mentioned.", + "payer": "Aetna", + "procedure_code": "MRI_LUMBAR", + "showcase": {}, + "site_of_care": "outpatient", + "specialty": "Primary Care" + }, + { + "dx_codes": [ + "M54.16" + ], + "expected_label": "incomplete", + "id": "MRI-11-conflict-deny-then-report", + "note_text": "Back pain x 8 weeks. PT x 8 weeks. Denies weakness earlier in visit. Later note: reports progressive weakness and urinary retention.", + "payer": "Aetna", + "procedure_code": "MRI_LUMBAR", + "showcase": {}, + "site_of_care": "outpatient", + "specialty": "Primary Care" + }, + { + "dx_codes": [ + "M54.16" + ], + "expected_label": "complete", + "id": "MRI-12-imaging-normal", + "note_text": "Back pain x 10 weeks. PT x 10 weeks. Lumbar xray normal/unremarkable. Denies bowel/bladder changes. Denies weakness.", + "payer": "Aetna", + "procedure_code": "MRI_LUMBAR", + "showcase": {}, + "site_of_care": "outpatient", + "specialty": "Orthopedics" + }, + { + "dx_codes": [ + "M54.5" + ], + "expected_label": "complete", + "id": "MRI-13-imaging-unclear", + "note_text": "Back pain x 10 weeks. PT x 10 weeks. Prior imaging performed but findings unclear. Denies weakness. Denies bowel/bladder changes.", + "payer": "Aetna", + "procedure_code": "MRI_LUMBAR", + "showcase": {}, + "site_of_care": "outpatient", + "specialty": "Primary Care" + }, + { + "dx_codes": [ + "M54.5" + ], + "expected_label": "incomplete", + "id": "MRI-14-therapy-context-missing", + "note_text": "Back pain x 8 weeks. No prior imaging. Denies weakness. Denies bowel/bladder changes.", + "payer": "Aetna", + "procedure_code": "MRI_LUMBAR", + "showcase": {}, + "site_of_care": "outpatient", + "specialty": "Primary Care" + }, + { + "dx_codes": [ + "G47.33" + ], + "expected_label": "incomplete", + "id": "CPAP-05-edge-with-date-no-ahi", + "note_text": "OSA diagnosed. Sleep study completed 2023-11-10. CPAP requested. (AHI not stated.)", + "payer": "Aetna", + "procedure_code": "CPAP_DEVICE", + "showcase": {}, + "site_of_care": "outpatient", + "specialty": "Sleep Medicine" + }, + { + "dx_codes": [ + "G47.33" + ], + "expected_label": "incomplete", + "id": "CPAP-06-edge-with-ahi-no-date", + "note_text": "OSA. AHI 18 documented. Sleep study mentioned but no date provided.", + "payer": "Aetna", + "procedure_code": "CPAP_DEVICE", + "showcase": {}, + "site_of_care": "outpatient", + "specialty": "Sleep Medicine" + }, + { + "dx_codes": [ + "G47.33" + ], + "expected_label": "complete", + "id": "CPAP-07-complete-alt-format-date", + "note_text": "Dx obstructive sleep apnea. Sleep study 2024/06/01. RDI 28. Request CPAP E0601.", + "payer": "Aetna", + "procedure_code": "CPAP_DEVICE", + "showcase": { + "description": "A CPAP request that includes OSA documentation, a slash-formatted sleep study date, and an RDI value.", + "expected_overall_status": "READY", + "featured": true, + "scenario_type": "Ready path", + "sort_order": 4, + "tags": [ + "date parsing", + "DME", + "cross-domain" + ], + "title": "CPAP ready case with alternative date formatting", + "why_interesting": "Shows cross-domain breadth and a small extraction robustness edge case." + }, + "site_of_care": "outpatient", + "specialty": "Sleep Medicine" + }, + { + "dx_codes": [ + "M54.12" + ], + "expected_label": "complete", + "id": "MRI-CERV-01-ready", + "note_text": "Neck pain with right arm radiculopathy x 8 weeks. PT for 8 weeks and NSAIDs documented with minimal improvement. Denies weakness. Denies bowel/bladder changes. Prior cervical xray inconclusive.", + "payer": "Aetna", + "procedure_code": "MRI_CERVICAL", + "showcase": { + "description": "A second spine MRI procedure that uses the same deterministic evidence contract as lumbar MRI.", + "expected_overall_status": "READY", + "featured": true, + "scenario_type": "New procedure coverage", + "sort_order": 5, + "tags": [ + "cervical MRI", + "registry depth", + "shared contract" + ], + "title": "Cervical MRI ready case using the new supported pathway", + "why_interesting": "Shows that the shared service and rule registry can support new deterministic procedures without changing the engine shape." + }, + "site_of_care": "outpatient", + "specialty": "Orthopedics" + }, + { + "dx_codes": [ + "M54.2" + ], + "expected_label": "incomplete", + "id": "MRI-CERV-02-not-ready", + "note_text": "Neck pain x 4 weeks. PT x 4 weeks documented. Denies weakness. Denies bowel/bladder changes. Prior cervical xray unremarkable.", + "payer": "Aetna", + "procedure_code": "MRI_CERVICAL", + "showcase": {}, + "site_of_care": "outpatient", + "specialty": "Primary Care" + }, + { + "dx_codes": [ + "M54.12" + ], + "expected_label": "incomplete", + "id": "MRI-CERV-03-cannot-determine", + "note_text": "Neck pain x 8 weeks. PT x 8 weeks documented. No prior imaging. No red flags mentioned.", + "payer": "Aetna", + "procedure_code": "MRI_CERVICAL", + "showcase": {}, + "site_of_care": "outpatient", + "specialty": "Primary Care" + }, + { + "dx_codes": [ + "M54.12" + ], + "expected_label": "complete", + "id": "MRI-CERV-04-positive-redflags", + "note_text": "Cervical radicular pain x 2 months. Completed physical therapy for 6 weeks. Reports progressive weakness in the right hand. Prior CT described as abnormal with degenerative stenosis.", + "payer": "Aetna", + "procedure_code": "MRI_CERVICAL", + "showcase": {}, + "site_of_care": "outpatient", + "specialty": "Neurology" + }, + { + "dx_codes": [ + "M25.561" + ], + "expected_label": "complete", + "id": "MRI-KNEE-01-ready", + "note_text": "Right knee pain with locking and catching x 8 weeks. Completed PT for 8 weeks and NSAIDs with minimal improvement. Prior knee xray normal/unremarkable.", + "payer": "Aetna", + "procedure_code": "MRI_KNEE", + "showcase": { + "description": "A non-spine advanced imaging request that stays narrow: symptom duration, therapy duration, prior xray context, and explicit mechanical symptom review.", + "expected_overall_status": "READY", + "featured": true, + "scenario_type": "Non-spine coverage", + "sort_order": 6, + "tags": [ + "knee MRI", + "orthopedics", + "mechanical symptoms" + ], + "title": "Knee MRI ready case with explicit mechanical symptoms", + "why_interesting": "Shows deeper deterministic product coverage without changing the engine shape or widening into clinical scoring." + }, + "site_of_care": "outpatient", + "specialty": "Orthopedics" + }, + { + "dx_codes": [ + "M25.561" + ], + "expected_label": "incomplete", + "id": "MRI-KNEE-02-not-ready-no-imaging", + "note_text": "Right knee pain with locking x 8 weeks. PT x 8 weeks and activity modification documented. No prior imaging yet.", + "payer": "Aetna", + "procedure_code": "MRI_KNEE", + "showcase": {}, + "site_of_care": "outpatient", + "specialty": "Orthopedics" + }, + { + "dx_codes": [ + "M25.561" + ], + "expected_label": "incomplete", + "id": "MRI-KNEE-03-cannot-determine", + "note_text": "Right knee pain x 8 weeks. PT x 8 weeks documented. Prior knee xray normal/unremarkable. MRI requested.", + "payer": "Aetna", + "procedure_code": "MRI_KNEE", + "showcase": {}, + "site_of_care": "outpatient", + "specialty": "Primary Care" + }, + { + "dx_codes": [ + "M25.561" + ], + "expected_label": "complete", + "id": "MRI-KNEE-04-ready-denied-mechanical", + "note_text": "Right knee pain x 10 weeks. Completed PT for 6 weeks and home exercise. Denies locking or instability. Prior knee xray findings unclear.", + "payer": "Aetna", + "procedure_code": "MRI_KNEE", + "showcase": {}, + "site_of_care": "outpatient", + "specialty": "Sports Medicine" + }, + { + "dx_codes": [ + "M25.561" + ], + "expected_label": "complete", + "id": "MRI-KNEE-05-conflict-deny-then-positive", + "note_text": "Right knee pain x 8 weeks. PT x 8 weeks documented. Denies locking earlier in visit. Later note: reports buckling with stairs. Prior knee xray normal.", + "payer": "Aetna", + "procedure_code": "MRI_KNEE", + "showcase": {}, + "site_of_care": "outpatient", + "specialty": "Orthopedics" + }, + { + "dx_codes": [], + "expected_label": "incomplete", + "id": "CPAP-08-doc-gap-osa-not-stated", + "note_text": "Sleep study completed 2024-01-05. AHI 22 documented. Request CPAP.", + "payer": "Aetna", + "procedure_code": "CPAP_DEVICE", + "showcase": {}, + "site_of_care": "outpatient", + "specialty": "Primary Care" + }, + { + "dx_codes": [ + "G47.33" + ], + "expected_label": "incomplete", + "id": "CPAP-09-noise-contains-date-unrelated", + "note_text": "OSA listed. Follow-up scheduled 2024-05-18. AHI not documented. Sleep study mentioned in 2023 but no sleep study date.", + "payer": "Aetna", + "procedure_code": "CPAP_DEVICE", + "showcase": {}, + "site_of_care": "outpatient", + "specialty": "Primary Care" + }, + { + "dx_codes": [ + "M54.5" + ], + "expected_label": "complete", + "id": "MRI-15-noise-unrelated-weeks", + "note_text": "Back pain x 8 weeks. Patient is 6 weeks pregnant. PT x 8 weeks documented. Denies bowel/bladder changes. Denies weakness. No imaging yet.", + "payer": "Aetna", + "procedure_code": "MRI_LUMBAR", + "showcase": {}, + "site_of_care": "outpatient", + "specialty": "Primary Care" + }, + { + "dx_codes": [ + "M54.16" + ], + "expected_label": "incomplete", + "id": "MRI-16-pt-mentioned-no-duration", + "note_text": "Back pain x 8 weeks. Completed PT and NSAIDs, duration not specified. Denies weakness. Denies bowel/bladder changes. No prior imaging.", + "payer": "Aetna", + "procedure_code": "MRI_LUMBAR", + "showcase": {}, + "site_of_care": "outpatient", + "specialty": "Primary Care" + } +] diff --git a/docs/artifacts/drift_report.md b/docs/artifacts/drift_report.md new file mode 100644 index 0000000..1970111 --- /dev/null +++ b/docs/artifacts/drift_report.md @@ -0,0 +1,17 @@ +# Drift Report + +- Review required: YES +- Stale monitored sources: 1 + +## Sources + +### Aetna MRI_LUMBAR +- Source: Aetna CPB 0157 +- Status: OK +- Freshness: STALE +- Last checked: 2026-02-06T06:12:45Z +- Rule source label: Human-curated summary of spine MRI administrative criteria +- Last rule reviewed: 2026-04-09 +- Review reason: Snapshot exceeds the configured daily monitoring window. +- Snapshot path: policy_snapshots/aetna_mri_lumbar/latest.json +- Diff path: n/a diff --git a/docs/artifacts/drift_status.json b/docs/artifacts/drift_status.json new file mode 100644 index 0000000..5ad30a4 --- /dev/null +++ b/docs/artifacts/drift_status.json @@ -0,0 +1,29 @@ +{ + "any_review_required": true, + "sources": [ + { + "check_frequency": "daily", + "days_since_last_checked": 62, + "freshness_status": "STALE", + "id": "aetna_mri_lumbar", + "last_checked_utc": "2026-02-06T06:12:45Z", + "last_rule_reviewed": "2026-04-09", + "latest_diff_path": null, + "latest_event": "BOOTSTRAP_SNAPSHOT_CREATED", + "latest_hash": "480f08eba64b3ad166504d92675a8751c598f4c4f364c38aaf702c346e525209", + "latest_snapshot_path": "policy_snapshots/aetna_mri_lumbar/latest.json", + "notes": "Monitored for drift; rules curated offline.", + "owner": "NickLeko", + "payer": "Aetna", + "procedure_code": "MRI_LUMBAR", + "review_reason": "Snapshot exceeds the configured daily monitoring window.", + "rule_source_label": "Human-curated summary of spine MRI administrative criteria", + "source_name": "Aetna CPB 0157", + "source_type": "official_policy_web", + "status": "OK", + "trust_level": "verified", + "url": "https://www.aetna.com/cpb/medical/data/100_199/0157.html" + } + ], + "stale_source_count": 1 +} diff --git a/docs/artifacts/featured_demo_cases.json b/docs/artifacts/featured_demo_cases.json new file mode 100644 index 0000000..83a4015 --- /dev/null +++ b/docs/artifacts/featured_demo_cases.json @@ -0,0 +1,158 @@ +[ + { + "dx_codes": [ + "M54.16" + ], + "expected_label": "complete", + "id": "MRI-01-complete", + "note_text": "Low back pain with right leg radiculopathy x 8 weeks. Completed PT for 8 weeks and NSAIDs with minimal improvement. Denies bowel/bladder incontinence. No saddle anesthesia. Prior imaging: lumbar xray inconclusive. Neuro exam: mild weakness dorsiflexion 4/5.", + "payer": "Aetna", + "procedure_code": "MRI_LUMBAR", + "showcase": { + "description": "A clean lumbar MRI request with therapy duration, symptom duration, imaging context, and red-flag documentation all present.", + "expected_overall_status": "READY", + "featured": true, + "scenario_type": "Ready path", + "sort_order": 1, + "tags": [ + "spine MRI", + "happy path", + "evidence map" + ], + "title": "Lumbar MRI ready for administrative review", + "why_interesting": "Shows the straight-line READY path with evidence snippets across each requirement." + }, + "site_of_care": "outpatient", + "specialty": "Orthopedics" + }, + { + "dx_codes": [ + "G47.33" + ], + "expected_label": "incomplete", + "id": "CPAP-02-borderline", + "note_text": "OSA noted in problem list. Sleep study in 2023 mentioned but no date. AHI not documented.", + "payer": "Aetna", + "procedure_code": "CPAP_DEVICE", + "showcase": { + "description": "The note mentions OSA and a past sleep study, but it leaves out the date and AHI details required to determine readiness.", + "expected_overall_status": "CANNOT_DETERMINE", + "featured": true, + "scenario_type": "Refusal-first", + "sort_order": 3, + "tags": [ + "DME", + "missing documentation", + "cannot determine" + ], + "title": "CPAP request refused for missing documentation", + "why_interesting": "Shows refusal-first behavior when required evidence is missing instead of inferred." + }, + "site_of_care": "outpatient", + "specialty": "Primary Care" + }, + { + "dx_codes": [ + "M54.5" + ], + "expected_label": "incomplete", + "id": "MRI-08-edge-below-threshold", + "note_text": "Low back pain x 5 weeks. PT x 5 weeks documented. Denies weakness. Denies bowel/bladder changes. No prior imaging.", + "payer": "Aetna", + "procedure_code": "MRI_LUMBAR", + "showcase": { + "description": "This note documents the right fields, but both symptom duration and therapy duration are still below threshold.", + "expected_overall_status": "NOT_READY", + "featured": true, + "scenario_type": "Documented failure", + "sort_order": 2, + "tags": [ + "threshold miss", + "not ready", + "fully documented" + ], + "title": "Documented, but not yet ready to submit", + "why_interesting": "Shows the difference between documented failure and missingness." + }, + "site_of_care": "outpatient", + "specialty": "Primary Care" + }, + { + "dx_codes": [ + "G47.33" + ], + "expected_label": "complete", + "id": "CPAP-07-complete-alt-format-date", + "note_text": "Dx obstructive sleep apnea. Sleep study 2024/06/01. RDI 28. Request CPAP E0601.", + "payer": "Aetna", + "procedure_code": "CPAP_DEVICE", + "showcase": { + "description": "A CPAP request that includes OSA documentation, a slash-formatted sleep study date, and an RDI value.", + "expected_overall_status": "READY", + "featured": true, + "scenario_type": "Ready path", + "sort_order": 4, + "tags": [ + "date parsing", + "DME", + "cross-domain" + ], + "title": "CPAP ready case with alternative date formatting", + "why_interesting": "Shows cross-domain breadth and a small extraction robustness edge case." + }, + "site_of_care": "outpatient", + "specialty": "Sleep Medicine" + }, + { + "dx_codes": [ + "M54.12" + ], + "expected_label": "complete", + "id": "MRI-CERV-01-ready", + "note_text": "Neck pain with right arm radiculopathy x 8 weeks. PT for 8 weeks and NSAIDs documented with minimal improvement. Denies weakness. Denies bowel/bladder changes. Prior cervical xray inconclusive.", + "payer": "Aetna", + "procedure_code": "MRI_CERVICAL", + "showcase": { + "description": "A second spine MRI procedure that uses the same deterministic evidence contract as lumbar MRI.", + "expected_overall_status": "READY", + "featured": true, + "scenario_type": "New procedure coverage", + "sort_order": 5, + "tags": [ + "cervical MRI", + "registry depth", + "shared contract" + ], + "title": "Cervical MRI ready case using the new supported pathway", + "why_interesting": "Shows that the shared service and rule registry can support new deterministic procedures without changing the engine shape." + }, + "site_of_care": "outpatient", + "specialty": "Orthopedics" + }, + { + "dx_codes": [ + "M25.561" + ], + "expected_label": "complete", + "id": "MRI-KNEE-01-ready", + "note_text": "Right knee pain with locking and catching x 8 weeks. Completed PT for 8 weeks and NSAIDs with minimal improvement. Prior knee xray normal/unremarkable.", + "payer": "Aetna", + "procedure_code": "MRI_KNEE", + "showcase": { + "description": "A non-spine advanced imaging request that stays narrow: symptom duration, therapy duration, prior xray context, and explicit mechanical symptom review.", + "expected_overall_status": "READY", + "featured": true, + "scenario_type": "Non-spine coverage", + "sort_order": 6, + "tags": [ + "knee MRI", + "orthopedics", + "mechanical symptoms" + ], + "title": "Knee MRI ready case with explicit mechanical symptoms", + "why_interesting": "Shows deeper deterministic product coverage without changing the engine shape or widening into clinical scoring." + }, + "site_of_care": "outpatient", + "specialty": "Orthopedics" + } +] diff --git a/docs/artifacts/rulebook_diff_reviewed_vs_active.json b/docs/artifacts/rulebook_diff_reviewed_vs_active.json new file mode 100644 index 0000000..914e158 --- /dev/null +++ b/docs/artifacts/rulebook_diff_reviewed_vs_active.json @@ -0,0 +1,25 @@ +{ + "added_procedures": [ + "MRI_KNEE" + ], + "changed_policy_sources": [], + "changed_procedures": [], + "changed_provenance": [ + "MRI_KNEE" + ], + "from_release_id": "2026-04-09-reviewed-v0.4", + "from_stage": "reviewed", + "removed_procedures": [], + "rules_version_from": "0.4", + "rules_version_to": "0.5", + "summary_lines": [ + "Rules version: 0.4 -> 0.5", + "Added procedures: MRI_KNEE", + "Removed procedures: none", + "Changed procedures: none", + "Changed provenance entries: MRI_KNEE", + "Changed policy source entries: none" + ], + "to_release_id": "2026-04-09-active-v0.5", + "to_stage": "active" +} diff --git a/docs/artifacts/rulebook_diff_reviewed_vs_active.md b/docs/artifacts/rulebook_diff_reviewed_vs_active.md new file mode 100644 index 0000000..275ca78 --- /dev/null +++ b/docs/artifacts/rulebook_diff_reviewed_vs_active.md @@ -0,0 +1,12 @@ +# Rulebook Diff + +- From: `2026-04-09-reviewed-v0.4` (reviewed) +- To: `2026-04-09-active-v0.5` (active) + +## Summary +- Rules version: 0.4 -> 0.5 +- Added procedures: MRI_KNEE +- Removed procedures: none +- Changed procedures: none +- Changed provenance entries: MRI_KNEE +- Changed policy source entries: none diff --git a/docs/artifacts/rulebook_status.json b/docs/artifacts/rulebook_status.json new file mode 100644 index 0000000..6bc24c7 --- /dev/null +++ b/docs/artifacts/rulebook_status.json @@ -0,0 +1,62 @@ +{ + "active_release_id": "2026-04-09-active-v0.5", + "manifest_version": "1", + "releases": [ + { + "based_on_release_id": "2026-04-09-reviewed-v0.4", + "created_at": "2026-04-09T12:15:00Z", + "files": { + "policy_sources_path": "rulebook/releases/2026-04-09-active-v0.5/policy_sources.yaml", + "provenance_path": "rulebook/releases/2026-04-09-active-v0.5/provenance.yaml", + "rules_path": "rulebook/releases/2026-04-09-active-v0.5/payer_rules.yaml" + }, + "notes": [ + "This active snapshot should match the runtime files under rules/." + ], + "procedures": [ + "CPAP_DEVICE", + "MRI_CERVICAL", + "MRI_KNEE", + "MRI_LUMBAR" + ], + "release_id": "2026-04-09-active-v0.5", + "reviewed_at": "2026-04-09", + "reviewer": "demo-maintainer", + "rules_version": "0.5", + "runtime_matches": true, + "stage": "active", + "summary": "Active rulebook after the narrow non-spine knee MRI expansion." + }, + { + "based_on_release_id": null, + "created_at": "2026-04-09T10:45:00Z", + "files": { + "policy_sources_path": "rulebook/releases/2026-04-09-reviewed-v0.4/policy_sources.yaml", + "provenance_path": "rulebook/releases/2026-04-09-reviewed-v0.4/provenance.yaml", + "rules_path": "rulebook/releases/2026-04-09-reviewed-v0.4/payer_rules.yaml" + }, + "notes": [ + "Historical reviewed snapshot retained for governance diffs." + ], + "procedures": [ + "CPAP_DEVICE", + "MRI_CERVICAL", + "MRI_LUMBAR" + ], + "release_id": "2026-04-09-reviewed-v0.4", + "reviewed_at": "2026-04-09", + "reviewer": "demo-maintainer", + "rules_version": "0.4", + "runtime_matches": null, + "stage": "reviewed", + "summary": "Reviewed rulebook before the narrow non-spine knee MRI expansion." + } + ], + "runtime_rules_version": "0.5", + "stage_assignments": { + "active": "2026-04-09-active-v0.5", + "draft": null, + "reviewed": "2026-04-09-reviewed-v0.4" + }, + "validation_errors": [] +} diff --git a/docs/artifacts/status.json b/docs/artifacts/status.json new file mode 100644 index 0000000..96f6ac1 --- /dev/null +++ b/docs/artifacts/status.json @@ -0,0 +1,10 @@ +{ + "demo_cases": 34, + "monitored_policy_sources": 1, + "rulebook_active_release_id": "2026-04-09-active-v0.5", + "rulebook_active_rules_version": "0.5", + "rules_version": "0.5", + "service": "Prior Authorization Readiness Copilot", + "supported_procedures": 4, + "synthetic_only": true +} diff --git a/docs/artifacts/supported_procedures.json b/docs/artifacts/supported_procedures.json new file mode 100644 index 0000000..6cae9e5 --- /dev/null +++ b/docs/artifacts/supported_procedures.json @@ -0,0 +1,300 @@ +[ + { + "display_name": "CPAP Device (HCPCS E0601)", + "metadata": { + "category": "durable_medical_equipment", + "last_rule_update": "2026-04-09", + "notes": [ + "The current demo checks for OSA diagnosis, dated sleep study evidence, and AHI/RDI documentation.", + "It does not determine clinical appropriateness or device approval likelihood." + ], + "rule_family": "sleep_study_documentation", + "summary": "Administrative readiness check for CPAP device requests using a narrow deterministic documentation contract.", + "supported_sites": [ + "outpatient" + ] + }, + "monitored_for_drift": false, + "payer": "Aetna", + "policy_trust_level": "demo", + "procedure_code": "CPAP_DEVICE", + "provenance": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": null, + "monitored_source_id": null, + "monitored_source_name": null, + "monitored_source_owner": null, + "monitored_source_url": null, + "notes": "Rules are illustrative for demo.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of CPAP administrative documentation criteria", + "source_name": "Aetna DME policy (summary)", + "source_type": "manual_summary", + "source_url": null + }, + "required_field_keys": [ + "osa_diagnosis", + "sleep_study_date", + "ahi_documented" + ], + "requirements": [ + { + "allowed": [], + "evidence": "Diagnosis present", + "key": "osa_diagnosis", + "label": "OSA diagnosis documented", + "min": null, + "type": "boolean" + }, + { + "allowed": [], + "evidence": "Date in chart", + "key": "sleep_study_date", + "label": "Sleep study date documented", + "min": null, + "type": "boolean" + }, + { + "allowed": [], + "evidence": "AHI value included", + "key": "ahi_documented", + "label": "AHI/RDI documented", + "min": null, + "type": "boolean" + } + ] + }, + { + "display_name": "MRI Cervical Spine (no contrast)", + "metadata": { + "category": "advanced_imaging", + "last_rule_update": "2026-04-09", + "notes": [ + "Intentionally mirrors the lumbar MRI demo pathway to show reusable deterministic architecture.", + "Human review remains required before any real submission." + ], + "rule_family": "spine_mri_conservative_therapy", + "summary": "Administrative readiness check for cervical spine MRI requests using the same narrow deterministic evidence contract as lumbar MRI.", + "supported_sites": [ + "outpatient" + ] + }, + "monitored_for_drift": false, + "payer": "Aetna", + "policy_trust_level": "demo", + "procedure_code": "MRI_CERVICAL", + "provenance": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": null, + "monitored_source_id": null, + "monitored_source_name": null, + "monitored_source_owner": null, + "monitored_source_url": null, + "notes": "Rules are illustrative for demo and intentionally mirror the narrow spine MRI contract.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of cervical spine MRI administrative criteria", + "source_name": "Aetna cervical spine MRI policy summary", + "source_type": "manual_summary", + "source_url": null + }, + "required_field_keys": [ + "conservative_therapy_weeks", + "neuro_red_flags_documented", + "prior_imaging_result", + "symptom_duration_weeks" + ], + "requirements": [ + { + "allowed": [], + "evidence": "PT/NSAIDs/activity modification trial documented", + "key": "conservative_therapy_weeks", + "label": "Conservative therapy duration (weeks)", + "min": 6.0, + "type": "number" + }, + { + "allowed": [], + "evidence": "Weakness, numbness, bowel/bladder changes, or related escalation findings explicitly documented", + "key": "neuro_red_flags_documented", + "label": "Neuro red flags explicitly addressed (present or denied)", + "min": null, + "type": "boolean" + }, + { + "allowed": [ + "none", + "inconclusive", + "abnormal" + ], + "evidence": "X-ray/CT results or note of none", + "key": "prior_imaging_result", + "label": "Prior imaging result", + "min": null, + "type": "enum" + }, + { + "allowed": [], + "evidence": "Duration supports guideline-based escalation", + "key": "symptom_duration_weeks", + "label": "Symptom duration (weeks)", + "min": 6.0, + "type": "number" + } + ] + }, + { + "display_name": "MRI Knee (no contrast)", + "metadata": { + "category": "advanced_imaging", + "last_rule_update": "2026-04-09", + "notes": [ + "This demo pathway checks only administrative completeness, not orthopedic appropriateness.", + "Requires symptom duration, conservative therapy, prior imaging context, and explicit review of mechanical symptoms." + ], + "rule_family": "extremity_mri_conservative_therapy", + "summary": "Administrative readiness check for knee MRI requests using a narrow deterministic documentation contract.", + "supported_sites": [ + "outpatient" + ] + }, + "monitored_for_drift": false, + "payer": "Aetna", + "policy_trust_level": "demo", + "procedure_code": "MRI_KNEE", + "provenance": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": null, + "monitored_source_id": null, + "monitored_source_name": null, + "monitored_source_owner": null, + "monitored_source_url": null, + "notes": "Rules are illustrative for demo and intentionally limited to documentation completeness.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of knee MRI administrative documentation criteria", + "source_name": "Aetna knee MRI policy summary", + "source_type": "manual_summary", + "source_url": null + }, + "required_field_keys": [ + "conservative_therapy_weeks", + "symptom_duration_weeks", + "prior_imaging_result", + "mechanical_symptoms_documented" + ], + "requirements": [ + { + "allowed": [], + "evidence": "PT/activity modification/NSAID trial documented", + "key": "conservative_therapy_weeks", + "label": "Conservative therapy duration (weeks)", + "min": 6.0, + "type": "number" + }, + { + "allowed": [], + "evidence": "Persistent symptoms documented long enough to justify escalation", + "key": "symptom_duration_weeks", + "label": "Symptom duration (weeks)", + "min": 6.0, + "type": "number" + }, + { + "allowed": [ + "inconclusive", + "abnormal" + ], + "evidence": "Prior knee imaging such as x-ray is documented and not simply absent", + "key": "prior_imaging_result", + "label": "Prior imaging result", + "min": null, + "type": "enum" + }, + { + "allowed": [], + "evidence": "Locking, catching, buckling, giving way, or instability explicitly documented", + "key": "mechanical_symptoms_documented", + "label": "Mechanical symptoms explicitly addressed (present or denied)", + "min": null, + "type": "boolean" + } + ] + }, + { + "display_name": "MRI Lumbar Spine (no contrast)", + "metadata": { + "category": "advanced_imaging", + "last_rule_update": "2026-04-09", + "notes": [ + "Demo rule set only; not a substitute for payer policy review.", + "Requires conservative therapy, symptom duration, imaging context, and explicit red-flag review." + ], + "rule_family": "spine_mri_conservative_therapy", + "summary": "Administrative readiness check for lumbar spine MRI requests using a narrow deterministic evidence contract.", + "supported_sites": [ + "outpatient" + ] + }, + "monitored_for_drift": true, + "payer": "Aetna", + "policy_trust_level": "demo", + "procedure_code": "MRI_LUMBAR", + "provenance": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": "daily", + "monitored_source_id": "aetna_mri_lumbar", + "monitored_source_name": "Aetna CPB 0157", + "monitored_source_owner": "NickLeko", + "monitored_source_url": "https://www.aetna.com/cpb/medical/data/100_199/0157.html", + "notes": "Rules are illustrative for demo. Not a substitute for payer policy documents.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of spine MRI administrative criteria", + "source_name": "Aetna spine MRI policy summary", + "source_type": "manual_summary", + "source_url": "https://www.aetna.com/cpb/medical/data/100_199/0157.html" + }, + "required_field_keys": [ + "conservative_therapy_weeks", + "neuro_red_flags_documented", + "prior_imaging_result", + "symptom_duration_weeks" + ], + "requirements": [ + { + "allowed": [], + "evidence": "PT/NSAIDs/activity modification trial documented", + "key": "conservative_therapy_weeks", + "label": "Conservative therapy duration (weeks)", + "min": 6.0, + "type": "number" + }, + { + "allowed": [], + "evidence": "Weakness, saddle anesthesia, bowel/bladder changes explicitly documented as present or denied", + "key": "neuro_red_flags_documented", + "label": "Neuro red flags explicitly addressed (present or denied)", + "min": null, + "type": "boolean" + }, + { + "allowed": [ + "none", + "inconclusive", + "abnormal" + ], + "evidence": "X-ray/CT results or note of none", + "key": "prior_imaging_result", + "label": "Prior imaging result", + "min": null, + "type": "enum" + }, + { + "allowed": [], + "evidence": "Duration supports guideline-based escalation", + "key": "symptom_duration_weeks", + "label": "Symptom duration (weeks)", + "min": 6.0, + "type": "number" + } + ] + } +] diff --git a/docs/demo_walkthrough.md b/docs/demo_walkthrough.md new file mode 100644 index 0000000..0e3229e --- /dev/null +++ b/docs/demo_walkthrough.md @@ -0,0 +1,127 @@ +# Demo Walkthrough + +## Goal + +Show a recruiter or interviewer that this repo is a disciplined internal-product artifact, not a toy notebook and not fake enterprise theater. + +## Setup + +```bash +python3 -m pip install -r requirements.txt +streamlit run app.py +``` + +Optional: + +```bash +python3 -m uvicorn api:app --reload +python3 cli.py list-demo-cases +``` + +## Recommended 5-Minute Demo Flow + +### 1. Open the Streamlit app + +Start with the scope language: + +- deterministic +- synthetic only +- administrative readiness only +- no clinical judgment +- no approval prediction + +### 2. Show the quality gates + +Point out: + +- bundled synthetic regression suite status +- policy drift status +- rulebook governance with reviewed vs active snapshots +- supported procedure registry with category, rule family, and provenance labels + +This is a strong signal that the repo cares about reliability and governance, not just output formatting. + +### 3. Run the straight-line ready case + +Use `MRI-01-complete`. + +Call out: + +- evidence mapping +- requirement-level reasoning +- audit trace +- deterministic letter drafting + +### 4. Run the documented-but-not-ready case + +Use `MRI-08-edge-below-threshold`. + +Call out the difference between: + +- present but below threshold -> `NOT_READY` +- missing or unclear -> not this case + +### 5. Run the refusal-first case + +Use `CPAP-02-borderline`. + +Call out: + +- `CANNOT_DETERMINE` is deliberate +- the system refuses to infer missing documentation +- this is safer than pretending certainty + +### 6. Show the same logic through API or CLI + +Example: + +```bash +python3 cli.py evaluate --demo-case CPAP-02-borderline +curl http://127.0.0.1:8000/supported-procedures +``` + +That makes the artifact feel like a compact internal product instead of a UI-only demo. + +### 7. Show the second supported spine MRI pathway + +Use `MRI-CERV-01-ready`. + +Call out: + +- the engine was not rewritten to add it +- the same deterministic extraction contract was reused +- the procedure registry now surfaces rule family and provenance metadata +- this is depth, not platform theater + +### 8. Show the non-spine deterministic expansion + +Use `MRI-KNEE-01-ready`. + +Call out: + +- this is a different clinical domain but still a narrow administrative contract +- only one new extractor field was added +- prior imaging is now a documented threshold, not just a missingness check +- the repo still avoids medical-necessity scoring or approval prediction + +### 9. Show the governance diff + +Use: + +```bash +python3 cli.py rulebook-diff --from-release 2026-04-09-reviewed-v0.4 --to-release 2026-04-09-active-v0.5 +``` + +Call out: + +- a human can see exactly what changed between reviewed and active rulebooks +- drift monitoring remains separate from promotion +- this is a compact governance story without pretending to be a platform + +## Good Sound Bites During The Demo + +- "This checks administrative readiness, not approval likelihood." +- "The system is narrow on purpose so every output can be defended." +- "Missing data produces refusal, not fake confidence." +- "The same deterministic workflow powers the UI, API, CLI, and exported artifacts." +- "The third pass added a non-spine pathway, a versioned rulebook, and golden acceptance checks without widening the product claim." diff --git a/docs/notes/PRD.md b/docs/notes/PRD.md index 2e60f87..805b9ca 100644 --- a/docs/notes/PRD.md +++ b/docs/notes/PRD.md @@ -42,7 +42,7 @@ Is this request administratively ready to submit as documented, and if not, whic ## Current Proof - Versioned rules in YAML -- Documented extraction behavior in [EXTRACTION_CONTRACT.md](/Users/nicholasleko/projects/PriorAuthorizationCopilot/EXTRACTION_CONTRACT.md) +- Documented extraction behavior in [EXTRACTION_CONTRACT.md](../../EXTRACTION_CONTRACT.md) - Pytest coverage for core behaviors - Streamlit demo for inspection diff --git a/docs/safety_and_scope.md b/docs/safety_and_scope.md new file mode 100644 index 0000000..bc69c51 --- /dev/null +++ b/docs/safety_and_scope.md @@ -0,0 +1,84 @@ +# Safety And Scope + +## Product Boundary + +This repo determines whether a request appears administratively ready under narrow, versioned payer rules. + +It does not: + +- make clinical recommendations +- determine medical necessity +- predict approval +- recommend utilization management strategy +- submit requests autonomously +- contact payers or patients + +## Why Synthetic-Only Data + +Synthetic-only inputs keep the repo: + +- safe to share in interviews and portfolios +- easy to test repeatedly +- free from PHI handling claims +- honest about its current maturity + +## Refusal-First Behavior + +The most important safety behavior is explicit refusal when documentation is missing. + +If any required item is not documented, the result must be `CANNOT_DETERMINE`. + +That avoids: + +- hidden inference +- false precision +- accidental overclaiming + +## Drift Monitoring Boundary + +Policy drift monitoring exists to support governance. + +It does: + +- snapshot configured sources +- normalize content +- detect changes +- flag review-required situations +- flag stale monitoring baselines when checks have aged past their configured cadence + +It does not: + +- rewrite rules +- change readiness outcomes automatically +- claim the monitored source is fully production-governed + +## Rulebook Promotion Boundary + +The rulebook registry exists to separate governance from runtime behavior. + +It does: + +- keep reviewed and active rule snapshots visible +- make release-to-release diffs inspectable +- require human promotion of runtime rule changes + +It does not: + +- auto-promote draft or reviewed rule snapshots +- auto-sync runtime rules from drift signals +- replace human policy review + +## Human Review In A Real Workflow + +In a real workflow, this kind of tool would sit before submission as an administrative quality gate. + +Human reviewers would still own: + +- chart review +- policy interpretation +- edge-case escalation +- final submission decisions + +## Honest Disclaimer + +This repo is an enterprise-shaped demo artifact, not a production healthcare deployment. diff --git a/docs/testing.md b/docs/testing.md new file mode 100644 index 0000000..37e5abd --- /dev/null +++ b/docs/testing.md @@ -0,0 +1,98 @@ +# Testing + +## Test Philosophy + +This repo favors high-signal deterministic tests over broad but shallow coverage. + +The most important things to protect are: + +- frozen readiness semantics +- deterministic extraction behavior +- refusal-first behavior for missing documentation +- evidence mapping and auditability +- supported-scope boundaries +- API and CLI surfaces sharing the same core workflow + +Two regression layers exist on purpose: + +- the bundled synthetic evaluation suite checks coarse `complete` versus `incomplete` fixture expectations +- acceptance snapshots lock representative exact outputs for evaluation and governance surfaces + +## Commands + +Run the full suite: + +```bash +pytest -q +``` + +Run the acceptance snapshots only: + +```bash +pytest -q test/test_acceptance_snapshots.py +``` + +Run the Streamlit sanity tests only: + +```bash +pytest -q test/test_streamlit_app.py +``` + +Run lint: + +```bash +ruff check . +``` + +Regenerate stable artifacts: + +```bash +python3 -m scripts.generate_artifacts +``` + +Regenerate golden snapshots intentionally after a reviewed product change: + +```bash +python3 -m scripts.generate_golden_outputs +``` + +## What Is Covered + +- extraction contracts and determinism +- rule loader validation +- provenance and policy trust behavior +- policy drift normalization and snapshot handling +- rulebook validation and release diffs +- letter drafting contracts +- shared service behavior +- API endpoints +- CLI workflows +- artifact generation +- acceptance snapshots for representative evaluation and governance outputs +- Streamlit AppTest sanity coverage +- bundled synthetic regression cases + +## Regression Cases + +The bundled synthetic case set intentionally includes: + +- ready cases +- documented-but-not-ready cases +- cannot-determine cases +- threshold edge cases +- unsupported or incomplete evidence patterns +- new procedure coverage for cervical MRI +- non-spine knee MRI coverage +- contradictory evidence precedence for red-flag extraction +- governance snapshot drift and rulebook integrity + +That is more useful here than adding a large quantity of low-value tests. + +## What Is Not Tested + +- real payer integrations +- browser automation +- authentication flows +- production deployment behavior + +Those are out of scope for this repo. diff --git a/engine/acceptance.py b/engine/acceptance.py new file mode 100644 index 0000000..28e43fd --- /dev/null +++ b/engine/acceptance.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from copy import deepcopy +from typing import Any, Dict + +from .rendering import export_evaluation_payload +from .service import ReadinessService + +DEFAULT_ACCEPTANCE_CASE_IDS = [ + "MRI-01-complete", + "MRI-08-edge-below-threshold", + "CPAP-02-borderline", + "MRI-KNEE-01-ready", +] + + +def _normalize_audit_trail(audit: Dict[str, Any]) -> Dict[str, Any]: + normalized = deepcopy(audit) + if normalized.get("run_id"): + normalized["run_id"] = "__RUN_ID__" + if normalized.get("timestamp_utc"): + normalized["timestamp_utc"] = "__TIMESTAMP_UTC__" + return normalized + + +def normalize_evaluation_payload(payload: Dict[str, Any]) -> Dict[str, Any]: + normalized = deepcopy(payload) + if isinstance(normalized.get("audit_trail"), dict): + normalized["audit_trail"] = _normalize_audit_trail(normalized["audit_trail"]) + report = normalized.get("report") + if isinstance(report, dict) and isinstance(report.get("audit_trail"), dict): + report["audit_trail"] = _normalize_audit_trail(report["audit_trail"]) + return normalized + + +def normalize_drift_payload(payload: Dict[str, Any]) -> Dict[str, Any]: + normalized = deepcopy(payload) + for source in normalized.get("sources") or []: + if source.get("days_since_last_checked") is not None: + source["days_since_last_checked"] = "__DAYS_SINCE_LAST_CHECKED__" + return normalized + + +def build_acceptance_evaluation_payload(service: ReadinessService, case_id: str) -> Dict[str, Any]: + request = service.get_demo_case_request(case_id) + evaluation = service.evaluate(request) + return normalize_evaluation_payload(export_evaluation_payload(evaluation)) + + +def build_acceptance_governance_payloads(service: ReadinessService) -> Dict[str, Dict[str, Any]]: + return { + "drift_status": normalize_drift_payload(service.get_drift_status().model_dump(mode="json")), + "rulebook_status": service.get_rulebook_status().model_dump(mode="json"), + "rulebook_diff_reviewed_vs_active": service.get_rulebook_diff( + "2026-04-09-reviewed-v0.4", + "2026-04-09-active-v0.5", + ).model_dump(mode="json"), + } diff --git a/engine/config.py b/engine/config.py new file mode 100644 index 0000000..fdd4b21 --- /dev/null +++ b/engine/config.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +import os +from pathlib import Path +from typing import List + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +VALID_LOG_LEVELS = {"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"} + + +def _repo_root_from_file() -> Path: + return Path(__file__).resolve().parents[1] + + +def _resolve_path(raw_path: str, repo_root: Path) -> Path: + candidate = Path(raw_path) + if not candidate.is_absolute(): + candidate = repo_root / candidate + return candidate.resolve() + + +class AppConfig(BaseModel): + model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) + + repo_root: Path + rules_path: Path + provenance_path: Path + policy_sources_path: Path + rulebook_manifest_path: Path + snapshot_root: Path + synthetic_cases_path: Path + docs_artifacts_dir: Path + log_level: str = "WARNING" + api_host: str = "127.0.0.1" + api_port: int = 8000 + allowed_sites: List[str] = Field(default_factory=lambda: ["outpatient", "inpatient", "ASC", "office"]) + + @field_validator("log_level") + @classmethod + def _normalize_log_level(cls, value: str) -> str: + normalized = value.strip().upper() + if normalized not in VALID_LOG_LEVELS: + raise ValueError(f"log_level must be one of {sorted(VALID_LOG_LEVELS)}") + return normalized + + @field_validator("api_host") + @classmethod + def _strip_api_host(cls, value: str) -> str: + stripped = value.strip() + if not stripped: + raise ValueError("api_host must be non-empty.") + return stripped + + @field_validator("api_port") + @classmethod + def _validate_port(cls, value: int) -> int: + if value <= 0 or value > 65535: + raise ValueError("api_port must be between 1 and 65535.") + return value + + @field_validator("allowed_sites") + @classmethod + def _validate_allowed_sites(cls, value: List[str]) -> List[str]: + sites = [site.strip() for site in value if site.strip()] + if not sites: + raise ValueError("allowed_sites must contain at least one site.") + return sites + + @model_validator(mode="after") + def _validate_required_paths(self) -> "AppConfig": + required_files = { + "rules_path": self.rules_path, + "provenance_path": self.provenance_path, + "policy_sources_path": self.policy_sources_path, + "rulebook_manifest_path": self.rulebook_manifest_path, + "synthetic_cases_path": self.synthetic_cases_path, + } + for field_name, path in required_files.items(): + if not path.exists(): + raise ValueError(f"{field_name} does not exist: {path}") + return self + + +def load_app_config(base_dir: Path | None = None) -> AppConfig: + repo_root = (base_dir or _repo_root_from_file()).resolve() + + return AppConfig( + repo_root=repo_root, + rules_path=_resolve_path(os.getenv("PA_COPILOT_RULES_PATH", "rules/payer_rules.yaml"), repo_root), + provenance_path=_resolve_path(os.getenv("PA_COPILOT_PROVENANCE_PATH", "rules/provenance.yaml"), repo_root), + policy_sources_path=_resolve_path(os.getenv("PA_COPILOT_POLICY_SOURCES_PATH", "rules/policy_sources.yaml"), repo_root), + rulebook_manifest_path=_resolve_path( + os.getenv("PA_COPILOT_RULEBOOK_MANIFEST_PATH", "rulebook/manifest.yaml"), + repo_root, + ), + snapshot_root=_resolve_path(os.getenv("PA_COPILOT_SNAPSHOT_ROOT", "policy_snapshots"), repo_root), + synthetic_cases_path=_resolve_path(os.getenv("PA_COPILOT_SYNTHETIC_CASES_PATH", "inputs/synthetic_cases.json"), repo_root), + docs_artifacts_dir=_resolve_path(os.getenv("PA_COPILOT_ARTIFACTS_DIR", "docs/artifacts"), repo_root), + log_level=os.getenv("PA_COPILOT_LOG_LEVEL", "WARNING"), + api_host=os.getenv("PA_COPILOT_API_HOST", "127.0.0.1"), + api_port=int(os.getenv("PA_COPILOT_API_PORT", "8000")), + ) diff --git a/engine/demo_cases.py b/engine/demo_cases.py new file mode 100644 index 0000000..582b603 --- /dev/null +++ b/engine/demo_cases.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import List + +from .config import AppConfig +from .schemas import DemoCase, PARequest + + +def load_demo_cases(path: str | Path) -> List[DemoCase]: + cases_path = Path(path) + with cases_path.open("r", encoding="utf-8") as handle: + payload = json.load(handle) + return [DemoCase.model_validate(item) for item in payload] + + +def list_demo_cases(config: AppConfig) -> List[DemoCase]: + return load_demo_cases(config.synthetic_cases_path) + + +def get_demo_case(case_id: str, config: AppConfig) -> DemoCase: + for case in list_demo_cases(config): + if case.id == case_id: + return case + raise KeyError(f"Demo case not found: {case_id}") + + +def featured_demo_cases(config: AppConfig) -> List[DemoCase]: + featured = [case for case in list_demo_cases(config) if bool(case.showcase.get("featured"))] + return sorted(featured, key=lambda case: int(case.showcase.get("sort_order", 999))) + + +def expected_overall_status_for_demo_case(case: DemoCase) -> str | None: + expected = str(case.showcase.get("expected_overall_status") or "").strip() + return expected or None + + +def demo_case_to_request(case: DemoCase) -> PARequest: + return PARequest( + payer=case.payer, + procedure_code=case.procedure_code, + dx_codes=case.dx_codes, + site_of_care=case.site_of_care, + specialty=case.specialty, + note_text=case.note_text, + ) diff --git a/engine/evaluate.py b/engine/evaluate.py index 07200bd..538895b 100644 --- a/engine/evaluate.py +++ b/engine/evaluate.py @@ -2,7 +2,7 @@ from typing import Any, Dict, List, Optional, Tuple -from .schemas import RequirementResult +from .schemas import EvidenceSpan, RequirementResult def _coerce_evidence_snippets(evidence_items: Any) -> List[str]: @@ -42,6 +42,26 @@ def _coerce_evidence_snippets(evidence_items: Any) -> List[str]: return dedup +def _coerce_evidence_spans(evidence_items: Any) -> List[EvidenceSpan]: + if not evidence_items: + return [] + + spans: List[EvidenceSpan] = [] + if isinstance(evidence_items, list): + for item in evidence_items: + if not isinstance(item, dict): + continue + start = item.get("start") + end = item.get("end") + text = str(item.get("text", "")).strip() + if not isinstance(start, int) or not isinstance(end, int) or not text: + continue + if end <= start or start < 0: + continue + spans.append(EvidenceSpan(start=start, end=end, text=text)) + return spans + + def _eval_number( key: str, label: str, @@ -51,7 +71,9 @@ def _eval_number( ) -> RequirementResult: val = facts.get(key) minv = req.get("min") - snippets = _coerce_evidence_snippets((evidence_map or {}).get(key)) + evidence_items = (evidence_map or {}).get(key) + snippets = _coerce_evidence_snippets(evidence_items) + spans = _coerce_evidence_spans(evidence_items) if val is None: return RequirementResult( @@ -61,6 +83,7 @@ def _eval_number( reason="Not found in note. Add explicit duration/value.", evidence=req.get("evidence"), evidence_snippets=snippets, + evidence_spans=spans, ) if minv is not None and val < minv: @@ -71,6 +94,7 @@ def _eval_number( reason=f"Documented value ({val}) below requirement (>= {minv}). Clarify or justify.", evidence=req.get("evidence"), evidence_snippets=snippets, + evidence_spans=spans, ) return RequirementResult( @@ -80,6 +104,7 @@ def _eval_number( reason=f"Documented value: {val}.", evidence=req.get("evidence"), evidence_snippets=snippets, + evidence_spans=spans, ) @@ -106,7 +131,9 @@ def _eval_boolean( - None => NOT_DOCUMENTED """ val = facts.get(key) - snippets = _coerce_evidence_snippets((evidence_map or {}).get(key)) + evidence_items = (evidence_map or {}).get(key) + snippets = _coerce_evidence_snippets(evidence_items) + spans = _coerce_evidence_spans(evidence_items) if val is True: return RequirementResult( @@ -116,6 +143,7 @@ def _eval_boolean( reason="Explicitly addressed in documentation (present/affirmed).", evidence=req.get("evidence"), evidence_snippets=snippets, + evidence_spans=spans, ) if val is False: @@ -126,6 +154,7 @@ def _eval_boolean( reason="Explicitly addressed in documentation (denied/absent).", evidence=req.get("evidence"), evidence_snippets=snippets, + evidence_spans=spans, ) return RequirementResult( @@ -135,6 +164,7 @@ def _eval_boolean( reason="Not found in note. Add explicit statement.", evidence=req.get("evidence"), evidence_snippets=snippets, + evidence_spans=spans, ) @@ -147,7 +177,9 @@ def _eval_enum( ) -> RequirementResult: val = facts.get(key) allowed = req.get("allowed", []) - snippets = _coerce_evidence_snippets((evidence_map or {}).get(key)) + evidence_items = (evidence_map or {}).get(key) + snippets = _coerce_evidence_snippets(evidence_items) + spans = _coerce_evidence_spans(evidence_items) if val is None: return RequirementResult( @@ -157,6 +189,7 @@ def _eval_enum( reason="Not found in note. Add explicit result/category.", evidence=req.get("evidence"), evidence_snippets=snippets, + evidence_spans=spans, ) if allowed and val not in allowed: @@ -167,6 +200,7 @@ def _eval_enum( reason=f"Value '{val}' not in allowed set {allowed}. Clarify wording/category.", evidence=req.get("evidence"), evidence_snippets=snippets, + evidence_spans=spans, ) return RequirementResult( @@ -176,6 +210,7 @@ def _eval_enum( reason=f"Documented: {val}.", evidence=req.get("evidence"), evidence_snippets=snippets, + evidence_spans=spans, ) @@ -246,4 +281,3 @@ def compute_overall_status(results: List[RequirementResult]) -> Dict[str, Any]: if has_not_met: return {"overall_status": "NOT_READY", "submission_readiness": False} return {"overall_status": "READY", "submission_readiness": True} - diff --git a/engine/extract.py b/engine/extract.py index 5eb2afb..e8253e0 100644 --- a/engine/extract.py +++ b/engine/extract.py @@ -101,12 +101,24 @@ def extract_facts(note_text: str) -> Tuple[Dict[str, Any], Dict[str, List[Dict[s m_months = re.search(r"\b(\d+)\s*(month|months)\b", t) if m_months: symptom_weeks = int(m_months.group(1)) * 4 - _add_span(evidence, "symptom_duration_weeks", m_months.start(), m_months.end(), raw[m_months.start(): m_months.end()]) + _add_span( + evidence, + "symptom_duration_weeks", + m_months.start(), + m_months.end(), + raw[m_months.start() : m_months.end()], + ) else: m_weeks = re.search(r"\b(\d+)\s*(week|weeks)\b", t) if m_weeks: symptom_weeks = int(m_weeks.group(1)) - _add_span(evidence, "symptom_duration_weeks", m_weeks.start(), m_weeks.end(), raw[m_weeks.start(): m_weeks.end()]) + _add_span( + evidence, + "symptom_duration_weeks", + m_weeks.start(), + m_weeks.end(), + raw[m_weeks.start() : m_weeks.end()], + ) # ---------------------------- # Neuro deficit / red flags addressed @@ -153,7 +165,7 @@ def extract_facts(note_text: str) -> Tuple[Dict[str, Any], Dict[str, List[Dict[s "neuro_red_flags_documented", positive_match.start(), positive_match.end(), - raw[positive_match.start(): positive_match.end()], + raw[positive_match.start() : positive_match.end()], ) elif denial_match: neuro_documented = True @@ -162,7 +174,7 @@ def extract_facts(note_text: str) -> Tuple[Dict[str, Any], Dict[str, List[Dict[s "neuro_red_flags_documented", denial_match.start(), denial_match.end(), - raw[denial_match.start(): denial_match.end()], + raw[denial_match.start() : denial_match.end()], ) else: neuro_documented = None @@ -176,7 +188,13 @@ def extract_facts(note_text: str) -> Tuple[Dict[str, Any], Dict[str, List[Dict[s m_no_img = re.search(r"\bno (prior )?imaging( documented| yet| to date)?\b", t) if m_no_img: prior_imaging = "none" - _add_span(evidence, "prior_imaging_result", m_no_img.start(), m_no_img.end(), raw[m_no_img.start(): m_no_img.end()]) + _add_span( + evidence, + "prior_imaging_result", + m_no_img.start(), + m_no_img.end(), + raw[m_no_img.start() : m_no_img.end()], + ) else: # Accept "prior imaging performed" / "imaging noted" even without modality/result m_any_img = re.search(r"\b(prior )?imaging\b", t) @@ -196,7 +214,7 @@ def extract_facts(note_text: str) -> Tuple[Dict[str, Any], Dict[str, List[Dict[s "prior_imaging_result", m_unclear.start(), m_unclear.end(), - raw[m_unclear.start(): m_unclear.end()], + raw[m_unclear.start() : m_unclear.end()], ) elif m_mod or m_any_img: @@ -213,18 +231,85 @@ def extract_facts(note_text: str) -> Tuple[Dict[str, Any], Dict[str, List[Dict[s "prior_imaging_result", m_norm.start(), m_norm.end(), - raw[m_norm.start(): m_norm.end()], + raw[m_norm.start() : m_norm.end()], ) else: m_abn = re.search(r"\b(abnormal|herniat|stenosis|disc bulge|fracture|degenerative)\b", t) if m_abn: prior_imaging = "abnormal" - _add_span(evidence, "prior_imaging_result", m_abn.start(), m_abn.end(), raw[m_abn.start(): m_abn.end()]) + _add_span( + evidence, + "prior_imaging_result", + m_abn.start(), + m_abn.end(), + raw[m_abn.start() : m_abn.end()], + ) else: # Imaging referenced but result not specified => documented as inconclusive prior_imaging = "inconclusive" m_span = m_mod or m_any_img - _add_span(evidence, "prior_imaging_result", m_span.start(), m_span.end(), raw[m_span.start(): m_span.end()]) + _add_span( + evidence, + "prior_imaging_result", + m_span.start(), + m_span.end(), + raw[m_span.start() : m_span.end()], + ) + + # ---------------------------- + # Mechanical symptoms addressed + # ---------------------------- + # This field is used for the narrow knee MRI pathway. + # Semantics: + # True -> explicit positive symptom wording (locking/catching/buckling/etc.) + # False -> explicit denial / absence wording + # None -> not addressed explicitly + mechanical_symptoms_documented: Optional[bool] = None + + mechanical_denial_patterns = [ + r"\bdenies\b.*\b(locking|catching|buckling|giving way|instability)\b", + r"\bno\b.*\b(locking|catching|buckling|giving way|instability)\b", + r"\bwithout\b.*\b(locking|catching|buckling|giving way|instability)\b", + ] + mechanical_positive_patterns = [ + r"\b(reports|reported|endorses|notes|noted|describes|described|with)\b.*\b(locking|catching|buckling|giving way|instability)\b", + r"\bmechanical symptoms\b", + ] + + mechanical_denial_match = None + for pat in mechanical_denial_patterns: + mm = re.search(pat, t) + if mm: + mechanical_denial_match = mm + break + + mechanical_positive_match = None + for pat in mechanical_positive_patterns: + mm = re.search(pat, t) + if mm: + text = raw[mm.start() : mm.end()].lower() + if not any(token in text for token in ("denies", "no ", "without")): + mechanical_positive_match = mm + break + + if mechanical_positive_match: + mechanical_symptoms_documented = True + _add_span( + evidence, + "mechanical_symptoms_documented", + mechanical_positive_match.start(), + mechanical_positive_match.end(), + raw[mechanical_positive_match.start() : mechanical_positive_match.end()], + ) + elif mechanical_denial_match: + mechanical_symptoms_documented = False + _add_span( + evidence, + "mechanical_symptoms_documented", + mechanical_denial_match.start(), + mechanical_denial_match.end(), + raw[mechanical_denial_match.start() : mechanical_denial_match.end()], + ) # ---------------------------- # OSA diagnosis @@ -233,7 +318,7 @@ def extract_facts(note_text: str) -> Tuple[Dict[str, Any], Dict[str, List[Dict[s m_osa = re.search(r"\b(obstructive sleep apnea|osa)\b", t) if m_osa: osa_dx = True - _add_span(evidence, "osa_diagnosis", m_osa.start(), m_osa.end(), raw[m_osa.start(): m_osa.end()]) + _add_span(evidence, "osa_diagnosis", m_osa.start(), m_osa.end(), raw[m_osa.start() : m_osa.end()]) # ---------------------------- # Sleep study date (context-gated) @@ -250,7 +335,7 @@ def extract_facts(note_text: str) -> Tuple[Dict[str, Any], Dict[str, List[Dict[s if SLEEP_CTX.search(window): sleep_study_date = True - _add_span(evidence, "sleep_study_date", m_date.start(), m_date.end(), raw[m_date.start(): m_date.end()]) + _add_span(evidence, "sleep_study_date", m_date.start(), m_date.end(), raw[m_date.start() : m_date.end()]) break # ---------------------------- @@ -261,18 +346,31 @@ def extract_facts(note_text: str) -> Tuple[Dict[str, Any], Dict[str, List[Dict[s m_ahi_missing = re.search(r"\b(ahi|rdi)\b.*\b(not documented|not stated|not available|unknown|n/?a|missing)\b", t) if m_ahi_missing: ahi_doc = None - _add_span(evidence, "ahi_documented", m_ahi_missing.start(), m_ahi_missing.end(), raw[m_ahi_missing.start(): m_ahi_missing.end()]) + _add_span( + evidence, + "ahi_documented", + m_ahi_missing.start(), + m_ahi_missing.end(), + raw[m_ahi_missing.start() : m_ahi_missing.end()], + ) else: m_ahi_val = re.search(r"\b(ahi|rdi)\b\s*[:=]?\s*(\d+(\.\d+)?)\b", t) if m_ahi_val: ahi_doc = True - _add_span(evidence, "ahi_documented", m_ahi_val.start(), m_ahi_val.end(), raw[m_ahi_val.start(): m_ahi_val.end()]) + _add_span( + evidence, + "ahi_documented", + m_ahi_val.start(), + m_ahi_val.end(), + raw[m_ahi_val.start() : m_ahi_val.end()], + ) facts: Dict[str, Any] = { "conservative_therapy_weeks": therapy_weeks, "neuro_red_flags_documented": neuro_documented, "prior_imaging_result": prior_imaging, "symptom_duration_weeks": symptom_weeks, + "mechanical_symptoms_documented": mechanical_symptoms_documented, "osa_diagnosis": osa_dx, "sleep_study_date": sleep_study_date, "ahi_documented": ahi_doc, diff --git a/engine/letter_draft.py b/engine/letter_draft.py index e4eb4cd..90385fc 100644 --- a/engine/letter_draft.py +++ b/engine/letter_draft.py @@ -5,8 +5,14 @@ from hashlib import sha256 from typing import Dict, List, Tuple -from .schemas import LetterType, OverallStatus, PARequest, PolicyTrustLevel, ReadinessReport, RequirementStatus - +from .schemas import ( + LetterType, + OverallStatus, + PARequest, + PolicyTrustLevel, + ReadinessReport, + RequirementStatus, +) ALLOWED_STATUSES: set[RequirementStatus] = {"MET", "NOT_MET", "NOT_DOCUMENTED"} ALLOWED_LETTER_TYPES: set[LetterType] = {"submission_cover_letter", "missing_info_request", "appeal_template"} @@ -120,7 +126,7 @@ def _validate_inputs( # Cross-check counts (block if inconsistent; prevents subtle downstream confusion) calc = {"MET": 0, "NOT_MET": 0, "NOT_DOCUMENTED": 0} - for r in (report.results or []): + for r in report.results or []: if r.status in calc: calc[r.status] += 1 @@ -148,10 +154,7 @@ def _policy_trust_line(policy_trust_level: str) -> str | None: Presentation-only; no logic impact. """ if policy_trust_level == "demo": - return ( - "Policy trust level: DEMO — criteria are illustrative only. " - "Verify against the official payer policy before submission." - ) + return "Policy trust level: DEMO — criteria are illustrative only. Verify against the official payer policy before submission." if policy_trust_level == "verified": return "Policy trust level: VERIFIED — criteria derived from documented payer policy sources." return None @@ -193,9 +196,7 @@ def draft_letter( if blocked_reasons: text = ( "DRAFT_BLOCKED\n\n" - "The letter could not be generated due to input validation errors:\n" - + "\n".join([f"- {r}" for r in blocked_reasons]) - + "\n" + "The letter could not be generated due to input validation errors:\n" + "\n".join([f"- {r}" for r in blocked_reasons]) + "\n" ) meta = LetterMeta( letter_version="1.1", @@ -235,13 +236,15 @@ def draft_letter( if letter_type == "missing_info_request": summary = ( "Summary:\n" - "The documentation provided is insufficient to determine administrative readiness because one or more required elements are not documented. " + "The documentation provided is insufficient to determine administrative readiness because " + "one or more required elements are not documented. " "This does not imply criteria failure and does not guarantee payer approval.\n" ) elif letter_type == "appeal_template": summary = ( "Summary:\n" - "This template summarizes documentation-based administrative criteria relevant to the request and is intended to support an appeal or reconsideration packet. " + "This template summarizes documentation-based administrative criteria relevant to the " + "request and is intended to support an appeal or reconsideration packet. " "It does not provide clinical recommendations and does not guarantee payer approval.\n" ) else: @@ -254,13 +257,15 @@ def draft_letter( elif overall == "NOT_READY": summary = ( "Summary:\n" - "The request is not administratively ready for submission because one or more documented requirements do not meet thresholds. " + "The request is not administratively ready for submission because one or more " + "documented requirements do not meet thresholds. " "This does not represent a clinical judgment and does not guarantee payer approval.\n" ) else: summary = ( "Summary:\n" - "Administrative readiness cannot be determined because one or more required elements are not documented in the record provided. " + "Administrative readiness cannot be determined because one or more required elements " + "are not documented in the record provided. " "This does not imply criteria failure and does not guarantee payer approval.\n" ) @@ -326,10 +331,7 @@ def draft_letter( if prohibited_hits: blocked_reasons = prohibited_hits text = ( - "DRAFT_BLOCKED\n\n" - "The letter was blocked due to prohibited language:\n" - + "\n".join([f"- {r}" for r in blocked_reasons]) - + "\n" + "DRAFT_BLOCKED\n\nThe letter was blocked due to prohibited language:\n" + "\n".join([f"- {r}" for r in blocked_reasons]) + "\n" ) meta = LetterMeta( letter_version="1.1", diff --git a/engine/logging_utils.py b/engine/logging_utils.py new file mode 100644 index 0000000..31bd7c0 --- /dev/null +++ b/engine/logging_utils.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +import json +import logging +from datetime import datetime, timezone +from typing import Any + + +class JsonFormatter(logging.Formatter): + def format(self, record: logging.LogRecord) -> str: + payload: dict[str, Any] = { + "timestamp_utc": datetime.fromtimestamp(record.created, tz=timezone.utc) + .replace(microsecond=0) + .isoformat() + .replace("+00:00", "Z"), + "level": record.levelname, + "logger": record.name, + "message": record.getMessage(), + } + + structured_fields = getattr(record, "structured_fields", None) + if isinstance(structured_fields, dict): + payload.update(structured_fields) + + if record.exc_info: + payload["exception"] = self.formatException(record.exc_info) + + return json.dumps(payload, sort_keys=True) + + +def configure_logging(level: str = "INFO") -> None: + root = logging.getLogger() + if getattr(root, "_pa_copilot_logging_configured", False): + root.setLevel(level) + return + + handler = logging.StreamHandler() + handler.setFormatter(JsonFormatter()) + + root.handlers.clear() + root.addHandler(handler) + root.setLevel(level) + root._pa_copilot_logging_configured = True # type: ignore[attr-defined] + + +def get_logger(name: str) -> logging.Logger: + return logging.getLogger(name) + + +def log_event(logger: logging.Logger, level: int, message: str, **fields: Any) -> None: + logger.log(level, message, extra={"structured_fields": fields}) diff --git a/engine/policy_monitor.py b/engine/policy_monitor.py index e2e85b1..e70060d 100644 --- a/engine/policy_monitor.py +++ b/engine/policy_monitor.py @@ -5,12 +5,10 @@ import difflib import hashlib import json -import os import re -import sys from dataclasses import dataclass from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional # Governance-only module: # - Detects drift in official policy sources @@ -29,6 +27,7 @@ class PolicySource: payer: str procedure_code: str url: str + source_name: str source_type: str trust_level: str check_frequency: str @@ -52,10 +51,7 @@ def load_policy_sources(path: Path = DEFAULT_SOURCES_YAML) -> List[PolicySource] try: import yaml # type: ignore except Exception as e: - raise RuntimeError( - "PyYAML is required to load rules/policy_sources.yaml. " - "Install pyyaml or vendor a minimal YAML loader." - ) from e + raise RuntimeError("PyYAML is required to load rules/policy_sources.yaml. Install pyyaml or vendor a minimal YAML loader.") from e data = yaml.safe_load(path.read_text(encoding="utf-8")) if not isinstance(data, dict) or "sources" not in data: @@ -69,7 +65,21 @@ def load_policy_sources(path: Path = DEFAULT_SOURCES_YAML) -> List[PolicySource] for s in sources_raw: if not isinstance(s, dict): continue - missing = [k for k in ("id", "payer", "procedure_code", "url", "source_type", "trust_level", "check_frequency", "owner") if k not in s] + missing = [ + k + for k in ( + "id", + "payer", + "procedure_code", + "url", + "source_name", + "source_type", + "trust_level", + "check_frequency", + "owner", + ) + if k not in s + ] if missing: raise ValueError(f"Source entry missing fields {missing}: {s}") out.append( @@ -78,6 +88,7 @@ def load_policy_sources(path: Path = DEFAULT_SOURCES_YAML) -> List[PolicySource] payer=str(s["payer"]), procedure_code=str(s["procedure_code"]), url=str(s["url"]), + source_name=str(s["source_name"]), source_type=str(s["source_type"]), trust_level=str(s["trust_level"]), check_frequency=str(s["check_frequency"]), @@ -98,9 +109,7 @@ def fetch_policy(url: str, timeout_s: int = 15) -> str: except Exception as e: raise RuntimeError("requests is required for live fetches (not used in tests).") from e - headers = { - "User-Agent": "PriorAuthorizationCopilot/PolicyMonitor (+governance; contact owner in policy_sources.yaml)" - } + headers = {"User-Agent": "PriorAuthorizationCopilot/PolicyMonitor (+governance; contact owner in policy_sources.yaml)"} resp = requests.get(url, headers=headers, timeout=timeout_s) resp.raise_for_status() # Keep as text; normalization will reduce noise. @@ -130,7 +139,14 @@ def feed(self, html: str) -> None: # 4) strip remaining tags html = re.sub(r"(?is)<[^>]+>", " ", html) # 5) unescape basic entities (minimal; deterministic) - html = html.replace(" ", " ").replace("&", "&").replace("<", "<").replace(">", ">").replace(""", '"').replace("'", "'") + html = ( + html.replace(" ", " ") + .replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace(""", '"') + .replace("'", "'") + ) self._chunks.append(html) def text(self) -> str: diff --git a/engine/provenance.py b/engine/provenance.py index dcd3bc0..c168749 100644 --- a/engine/provenance.py +++ b/engine/provenance.py @@ -7,6 +7,17 @@ from .schemas import PARequest, PolicyTrustLevel +PROVENANCE_STRING_FIELDS = { + "source_type", + "source_name", + "source_url", + "rule_source_label", + "last_reviewed", + "rule_last_updated", + "monitored_source_id", + "notes", +} + def load_provenance(path: str | Path) -> Dict[str, Any]: provenance_path = Path(path) @@ -27,6 +38,19 @@ def load_provenance(path: str | Path) -> Dict[str, Any]: if not isinstance(sources, dict): raise ValueError("Invalid provenance file: 'sources' must be a mapping.") + for payer, payer_entries in sources.items(): + if not isinstance(payer_entries, dict): + raise ValueError(f"Invalid provenance file: '{payer}' entries must be a mapping.") + for procedure_code, entry in payer_entries.items(): + if not isinstance(entry, dict): + raise ValueError(f"Invalid provenance file: '{payer}.{procedure_code}' must be a mapping.") + for field_name, value in entry.items(): + if field_name in PROVENANCE_STRING_FIELDS and value is not None: + if not isinstance(value, str) or not value.strip(): + raise ValueError( + f"Invalid provenance file: '{payer}.{procedure_code}.{field_name}' must be a non-empty string." + ) + return data diff --git a/engine/rendering.py b/engine/rendering.py new file mode 100644 index 0000000..b35493d --- /dev/null +++ b/engine/rendering.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Dict, Optional + +from .schemas import DriftStatusReport, EvaluationResult, RulebookDiffResponse, RulebookStatusResponse + + +def export_evaluation_payload( + evaluation: EvaluationResult, + letter_text: Optional[str] = None, + letter_meta: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + payload = evaluation.model_dump(mode="json") + if letter_text is not None or letter_meta is not None: + payload["letter"] = { + "text": letter_text or "", + "metadata": letter_meta or {}, + } + return payload + + +def write_json_artifact(payload: Dict[str, Any], output_path: str | Path) -> Path: + artifact_path = Path(output_path) + artifact_path.parent.mkdir(parents=True, exist_ok=True) + artifact_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return artifact_path + + +def render_cli_evaluation(evaluation: EvaluationResult) -> str: + source_label = ( + evaluation.supported_procedure.provenance.rule_source_label + or evaluation.supported_procedure.provenance.source_name + or "n/a" + ) + lines = [ + "Prior Authorization Readiness Copilot", + f"Payer: {evaluation.request.payer}", + f"Procedure: {evaluation.request.procedure_code} ({evaluation.supported_procedure.display_name})", + f"Category: {evaluation.supported_procedure.metadata.category}", + f"Rule family: {evaluation.supported_procedure.metadata.rule_family}", + f"Rule source: {source_label}", + f"Rulebook release: {evaluation.audit_trail.rulebook_active_release_id or 'n/a'}", + f"Overall status: {evaluation.overall_status}", + f"Submission readiness: {'YES' if evaluation.submission_readiness else 'NO'}", + f"Readiness score: {evaluation.readiness_score}/100", + f"Policy trust level: {evaluation.policy_trust_level.upper()}", + "", + "Blocking summary:", + f"- Missing requirements: {len(evaluation.blockers.not_documented)}", + f"- Documented but not met: {len(evaluation.blockers.not_met)}", + ] + + if evaluation.warnings: + lines.extend(["", "Warnings:"]) + lines.extend([f"- {warning}" for warning in evaluation.warnings]) + + lines.extend(["", "Requirement results:"]) + for result in evaluation.results: + lines.append(f"- {result.label}: {result.status} | {result.reason}") + + return "\n".join(lines) + + +def render_drift_status(report: DriftStatusReport) -> str: + lines = [ + "Policy Drift Status", + f"Review required: {'YES' if report.any_review_required else 'NO'}", + f"Stale monitored sources: {report.stale_source_count}", + "", + ] + for source in report.sources: + lines.append( + f"- {source.payer} {source.procedure_code} | {source.source_name or 'unnamed source'} | {source.status} | " + f"trust={source.trust_level} | check={source.check_frequency} | " + f"freshness={source.freshness_status or 'UNKNOWN'} | last_checked={source.last_checked_utc or 'n/a'}" + ) + return "\n".join(lines) + + +def render_rulebook_status(report: RulebookStatusResponse) -> str: + lines = [ + "Rulebook Status", + f"Active release: {report.active_release_id or 'n/a'}", + f"Runtime rules version: {report.runtime_rules_version or 'n/a'}", + "", + ] + for release in report.releases: + runtime = ( + f" | runtime_match={'yes' if release.runtime_matches else 'no'}" + if release.runtime_matches is not None + else "" + ) + lines.append( + f"- {release.release_id} | stage={release.stage or 'unassigned'} | " + f"rules_version={release.rules_version or 'n/a'} | procedures={len(release.procedures)}{runtime}" + ) + if report.validation_errors: + lines.extend(["", "Validation errors:"]) + lines.extend([f"- {item}" for item in report.validation_errors]) + return "\n".join(lines) + + +def render_rulebook_diff(report: RulebookDiffResponse) -> str: + lines = [ + "Rulebook Diff", + f"From: {report.from_release_id} ({report.from_stage or 'unassigned'})", + f"To: {report.to_release_id} ({report.to_stage or 'unassigned'})", + "", + ] + lines.extend([f"- {line}" for line in report.summary_lines]) + return "\n".join(lines) + + +def render_drift_markdown(report: DriftStatusReport) -> str: + lines = [ + "# Drift Report", + "", + f"- Review required: {'YES' if report.any_review_required else 'NO'}", + f"- Stale monitored sources: {report.stale_source_count}", + "", + "## Sources", + ] + for source in report.sources: + lines.extend( + [ + "", + f"### {source.payer} {source.procedure_code}", + f"- Source: {source.source_name or 'unnamed source'}", + f"- Status: {source.status}", + f"- Freshness: {source.freshness_status or 'UNKNOWN'}", + f"- Last checked: {source.last_checked_utc or 'n/a'}", + f"- Rule source label: {source.rule_source_label or 'n/a'}", + f"- Last rule reviewed: {source.last_rule_reviewed or 'n/a'}", + f"- Review reason: {source.review_reason or 'n/a'}", + f"- Snapshot path: {source.latest_snapshot_path or 'n/a'}", + f"- Diff path: {source.latest_diff_path or 'n/a'}", + ] + ) + return "\n".join(lines) + "\n" + + +def render_rulebook_diff_markdown(report: RulebookDiffResponse) -> str: + lines = [ + "# Rulebook Diff", + "", + f"- From: `{report.from_release_id}` ({report.from_stage or 'unassigned'})", + f"- To: `{report.to_release_id}` ({report.to_stage or 'unassigned'})", + "", + "## Summary", + ] + lines.extend([f"- {line}" for line in report.summary_lines]) + return "\n".join(lines) + "\n" diff --git a/engine/rulebook.py b/engine/rulebook.py new file mode 100644 index 0000000..6cbe234 --- /dev/null +++ b/engine/rulebook.py @@ -0,0 +1,258 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, Tuple + +import yaml + +from .schemas import RulebookDiffResponse, RulebookFileSet, RulebookRelease, RulebookStatusResponse + + +class RulebookError(Exception): + pass + + +def _load_yaml(path: Path) -> Dict[str, Any]: + payload = yaml.safe_load(path.read_text(encoding="utf-8")) + if payload is None: + return {} + if not isinstance(payload, dict): + raise RulebookError(f"YAML file must contain a mapping: {path.as_posix()}") + return payload + + +def _resolve_path(repo_root: Path, raw_path: str) -> Path: + candidate = Path(raw_path) + if not candidate.is_absolute(): + candidate = repo_root / candidate + return candidate.resolve() + + +def _display_path(repo_root: Path, raw_path: str) -> str: + resolved = _resolve_path(repo_root, raw_path) + try: + return resolved.relative_to(repo_root).as_posix() + except ValueError: + return resolved.as_posix() + + +def load_rulebook_manifest(path: Path) -> Dict[str, Any]: + manifest = _load_yaml(path) + if "stages" not in manifest or "releases" not in manifest: + raise RulebookError("rulebook manifest must include 'stages' and 'releases'") + if not isinstance(manifest["stages"], dict): + raise RulebookError("rulebook manifest 'stages' must be a mapping") + if not isinstance(manifest["releases"], dict): + raise RulebookError("rulebook manifest 'releases' must be a mapping") + return manifest + + +def _extract_procedure_map(rules_data: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: + out: Dict[str, Dict[str, Any]] = {} + for payer_config in (rules_data.get("payers") or {}).values(): + for procedure_code, procedure in (payer_config.get("procedures") or {}).items(): + out[str(procedure_code)] = procedure + return out + + +def _extract_procedure_codes(rules_data: Dict[str, Any]) -> list[str]: + return sorted(_extract_procedure_map(rules_data)) + + +def _extract_provenance_map(provenance_data: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: + out: Dict[str, Dict[str, Any]] = {} + for payer_entries in (provenance_data.get("sources") or {}).values(): + for procedure_code, entry in (payer_entries or {}).items(): + out[str(procedure_code)] = entry + return out + + +def _extract_policy_source_map(policy_sources_data: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: + out: Dict[str, Dict[str, Any]] = {} + for source in policy_sources_data.get("sources") or []: + if not isinstance(source, dict): + continue + out[str(source.get("id") or "")] = source + return {key: value for key, value in out.items() if key} + + +def _load_release_bundle( + repo_root: Path, + raw_release: Dict[str, Any], +) -> Tuple[RulebookFileSet, Dict[str, Any], Dict[str, Any], Dict[str, Any]]: + files = raw_release.get("files") or {} + if not isinstance(files, dict): + raise RulebookError("rulebook release 'files' must be a mapping") + + rules_path = _resolve_path(repo_root, str(files.get("rules") or "")) + provenance_path = _resolve_path(repo_root, str(files.get("provenance") or "")) + policy_sources_path = _resolve_path(repo_root, str(files.get("policy_sources") or "")) + bundle = RulebookFileSet( + rules_path=_display_path(repo_root, str(files.get("rules") or "")), + provenance_path=_display_path(repo_root, str(files.get("provenance") or "")), + policy_sources_path=_display_path(repo_root, str(files.get("policy_sources") or "")), + ) + + return ( + bundle, + _load_yaml(rules_path), + _load_yaml(provenance_path), + _load_yaml(policy_sources_path), + ) + + +def get_rulebook_status(repo_root: Path, manifest_path: Path, runtime_files: RulebookFileSet) -> RulebookStatusResponse: + manifest = load_rulebook_manifest(manifest_path) + stages = manifest.get("stages") or {} + releases_raw = manifest.get("releases") or {} + validation_errors: list[str] = [] + + for required_stage in ("draft", "reviewed", "active"): + if required_stage not in stages: + validation_errors.append(f"Missing stage assignment for '{required_stage}'.") + + runtime_rules = _load_yaml(Path(runtime_files.rules_path)) + runtime_provenance = _load_yaml(Path(runtime_files.provenance_path)) + runtime_policy_sources = _load_yaml(Path(runtime_files.policy_sources_path)) + + releases: list[RulebookRelease] = [] + active_release_id = stages.get("active") + + for release_id, raw_release in sorted(releases_raw.items()): + if not isinstance(raw_release, dict): + validation_errors.append(f"Release '{release_id}' must be a mapping.") + continue + + try: + file_set, rules_data, provenance_data, policy_sources_data = _load_release_bundle(repo_root, raw_release) + except Exception as exc: + validation_errors.append(f"Release '{release_id}' could not be loaded: {exc}") + continue + + procedures = _extract_procedure_codes(rules_data) + declared_procedures = sorted(str(item) for item in raw_release.get("procedures") or []) + if declared_procedures and declared_procedures != procedures: + validation_errors.append( + f"Release '{release_id}' declared procedures {declared_procedures} but files contain {procedures}." + ) + + file_rules_version = str(rules_data.get("version")) if rules_data.get("version") is not None else None + declared_rules_version = str(raw_release.get("rules_version")) if raw_release.get("rules_version") is not None else None + if declared_rules_version and declared_rules_version != file_rules_version: + validation_errors.append( + f"Release '{release_id}' declared rules_version={declared_rules_version} but files contain {file_rules_version}." + ) + + runtime_matches = None + if release_id == active_release_id: + runtime_matches = ( + rules_data == runtime_rules + and provenance_data == runtime_provenance + and policy_sources_data == runtime_policy_sources + ) + if not runtime_matches: + validation_errors.append( + f"Active release '{release_id}' does not match the runtime files under rules/." + ) + + releases.append( + RulebookRelease( + release_id=str(release_id), + stage=raw_release.get("stage"), + summary=str(raw_release.get("summary") or release_id), + created_at=raw_release.get("created_at"), + based_on_release_id=raw_release.get("based_on_release_id"), + rules_version=declared_rules_version or file_rules_version, + procedures=declared_procedures or procedures, + files=file_set, + reviewer=raw_release.get("reviewer"), + reviewed_at=raw_release.get("reviewed_at"), + runtime_matches=runtime_matches, + notes=[str(note) for note in raw_release.get("notes") or []], + ) + ) + + for stage_name, release_id in stages.items(): + if release_id and release_id not in releases_raw: + validation_errors.append(f"Stage '{stage_name}' points to unknown release '{release_id}'.") + + return RulebookStatusResponse( + manifest_version=str(manifest.get("version")) if manifest.get("version") is not None else None, + active_release_id=str(active_release_id) if active_release_id else None, + stage_assignments={str(key): (str(value) if value else None) for key, value in stages.items()}, + runtime_rules_version=str(runtime_rules.get("version")) if runtime_rules.get("version") is not None else None, + releases=releases, + validation_errors=validation_errors, + ) + + +def get_rulebook_diff(repo_root: Path, manifest_path: Path, from_release_id: str, to_release_id: str) -> RulebookDiffResponse: + manifest = load_rulebook_manifest(manifest_path) + releases_raw = manifest.get("releases") or {} + + if from_release_id not in releases_raw: + raise RulebookError(f"Unknown rulebook release: {from_release_id}") + if to_release_id not in releases_raw: + raise RulebookError(f"Unknown rulebook release: {to_release_id}") + + _, from_rules, from_provenance, from_policy_sources = _load_release_bundle(repo_root, releases_raw[from_release_id]) + _, to_rules, to_provenance, to_policy_sources = _load_release_bundle(repo_root, releases_raw[to_release_id]) + + from_procedure_map = _extract_procedure_map(from_rules) + to_procedure_map = _extract_procedure_map(to_rules) + from_procedures = set(from_procedure_map) + to_procedures = set(to_procedure_map) + + added_procedures = sorted(to_procedures - from_procedures) + removed_procedures = sorted(from_procedures - to_procedures) + changed_procedures = sorted( + procedure_code + for procedure_code in (from_procedures & to_procedures) + if from_procedure_map[procedure_code] != to_procedure_map[procedure_code] + ) + + from_provenance_map = _extract_provenance_map(from_provenance) + to_provenance_map = _extract_provenance_map(to_provenance) + changed_provenance = sorted( + procedure_code + for procedure_code in set(from_provenance_map) | set(to_provenance_map) + if from_provenance_map.get(procedure_code) != to_provenance_map.get(procedure_code) + ) + + from_policy_map = _extract_policy_source_map(from_policy_sources) + to_policy_map = _extract_policy_source_map(to_policy_sources) + changed_policy_sources = sorted( + source_id + for source_id in set(from_policy_map) | set(to_policy_map) + if from_policy_map.get(source_id) != to_policy_map.get(source_id) + ) + + rules_version_from = str(from_rules.get("version")) if from_rules.get("version") is not None else None + rules_version_to = str(to_rules.get("version")) if to_rules.get("version") is not None else None + + def _format_summary_line(label: str, values: list[str]) -> str: + return f"{label}: {', '.join(values)}" if values else f"{label}: none" + + summary_lines = [ + f"Rules version: {rules_version_from or 'n/a'} -> {rules_version_to or 'n/a'}", + _format_summary_line("Added procedures", added_procedures), + _format_summary_line("Removed procedures", removed_procedures), + _format_summary_line("Changed procedures", changed_procedures), + _format_summary_line("Changed provenance entries", changed_provenance), + _format_summary_line("Changed policy source entries", changed_policy_sources), + ] + + return RulebookDiffResponse( + from_release_id=from_release_id, + to_release_id=to_release_id, + from_stage=releases_raw[from_release_id].get("stage"), + to_stage=releases_raw[to_release_id].get("stage"), + rules_version_from=rules_version_from, + rules_version_to=rules_version_to, + added_procedures=added_procedures, + removed_procedures=removed_procedures, + changed_procedures=changed_procedures, + changed_provenance=changed_provenance, + changed_policy_sources=changed_policy_sources, + summary_lines=summary_lines, + ) diff --git a/engine/rules_loader.py b/engine/rules_loader.py index 0150843..dc82d1f 100644 --- a/engine/rules_loader.py +++ b/engine/rules_loader.py @@ -1,13 +1,44 @@ from __future__ import annotations + from pathlib import Path -from typing import Any, Dict, List +from typing import Any, Dict import yaml - ALLOWED_REQUIREMENT_TYPES = {"number", "boolean", "enum"} +def _validate_procedure_metadata(metadata: Dict[str, Any], payer: str, procedure_code: str) -> None: + location = f"{payer}.{procedure_code}.metadata" + + if not isinstance(metadata, dict): + raise ValueError(f"Invalid rules file: {location} must be a mapping.") + + required_string_fields = ("category", "rule_family", "summary") + for field_name in required_string_fields: + value = metadata.get(field_name) + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"Invalid rules file: {location}.{field_name} must be a non-empty string.") + + supported_sites = metadata.get("supported_sites", []) + if supported_sites is not None: + if not isinstance(supported_sites, list): + raise ValueError(f"Invalid rules file: {location}.supported_sites must be a list when provided.") + if any(not isinstance(site, str) or not site.strip() for site in supported_sites): + raise ValueError(f"Invalid rules file: {location}.supported_sites entries must be non-empty strings.") + + notes = metadata.get("notes", []) + if notes is not None: + if not isinstance(notes, list): + raise ValueError(f"Invalid rules file: {location}.notes must be a list when provided.") + if any(not isinstance(note, str) or not note.strip() for note in notes): + raise ValueError(f"Invalid rules file: {location}.notes entries must be non-empty strings.") + + last_rule_update = metadata.get("last_rule_update") + if last_rule_update is not None and (not isinstance(last_rule_update, str) or not last_rule_update.strip()): + raise ValueError(f"Invalid rules file: {location}.last_rule_update must be a non-empty string when provided.") + + def _validate_requirement(req: Dict[str, Any], payer: str, procedure_code: str, idx: int) -> None: location = f"{payer}.{procedure_code}.required[{idx}]" @@ -17,9 +48,7 @@ def _validate_requirement(req: Dict[str, Any], payer: str, procedure_code: str, req_type = req.get("type", "boolean") if req_type not in ALLOWED_REQUIREMENT_TYPES: - raise ValueError( - f"Invalid rules file: {location}.type must be one of {sorted(ALLOWED_REQUIREMENT_TYPES)}." - ) + raise ValueError(f"Invalid rules file: {location}.type must be one of {sorted(ALLOWED_REQUIREMENT_TYPES)}.") label = req.get("label", key) if not isinstance(label, str) or not label.strip(): @@ -47,21 +76,19 @@ def _validate_procedures(procedures: Dict[str, Any], payer: str) -> None: display_name = procedure.get("display_name", procedure_code) if not isinstance(display_name, str) or not display_name.strip(): - raise ValueError( - f"Invalid rules file: {payer}.{procedure_code}.display_name must be a non-empty string." - ) + raise ValueError(f"Invalid rules file: {payer}.{procedure_code}.display_name must be a non-empty string.") + + metadata = procedure.get("metadata") + if metadata is not None: + _validate_procedure_metadata(metadata, payer, procedure_code) requirements = procedure.get("required") if not isinstance(requirements, list) or not requirements: - raise ValueError( - f"Invalid rules file: {payer}.{procedure_code}.required must be a non-empty list." - ) + raise ValueError(f"Invalid rules file: {payer}.{procedure_code}.required must be a non-empty list.") for idx, requirement in enumerate(requirements): if not isinstance(requirement, dict): - raise ValueError( - f"Invalid rules file: {payer}.{procedure_code}.required[{idx}] must be a mapping." - ) + raise ValueError(f"Invalid rules file: {payer}.{procedure_code}.required[{idx}] must be a mapping.") _validate_requirement(requirement, payer, procedure_code, idx) diff --git a/engine/schemas.py b/engine/schemas.py index 46e2e64..425afe6 100644 --- a/engine/schemas.py +++ b/engine/schemas.py @@ -4,18 +4,19 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator - RequirementStatus = Literal["MET", "NOT_MET", "NOT_DOCUMENTED"] OverallStatus = Literal["READY", "NOT_READY", "CANNOT_DETERMINE", "UNKNOWN"] LetterType = Literal["submission_cover_letter", "missing_info_request", "appeal_template"] PolicyTrustLevel = Literal["demo", "verified"] +RequirementType = Literal["number", "boolean", "enum"] +RulebookStage = Literal["draft", "reviewed", "active"] class PARequest(BaseModel): model_config = ConfigDict(extra="forbid") payer: str - procedure_code: str # e.g., "MRI_LUMBAR" + procedure_code: str dx_codes: List[str] = Field(default_factory=list) site_of_care: str = "outpatient" specialty: str = "unknown" @@ -32,6 +33,55 @@ def _ensure_dx_codes_not_none(cls, value: List[str]) -> List[str]: return [str(code) for code in value if str(code).strip()] +class EvidenceSpan(BaseModel): + model_config = ConfigDict(extra="forbid") + + start: int + end: int + text: str + + @field_validator("start", "end") + @classmethod + def _validate_offsets(cls, value: int) -> int: + if value < 0: + raise ValueError("Evidence span offsets must be non-negative.") + return value + + @field_validator("text") + @classmethod + def _strip_text(cls, value: str) -> str: + return value.strip() + + +class RequirementDefinition(BaseModel): + model_config = ConfigDict(extra="forbid") + + key: str + label: str + type: RequirementType = "boolean" + min: Optional[float] = None + allowed: List[str] = Field(default_factory=list) + evidence: Optional[str] = None + + @field_validator("key", "label") + @classmethod + def _strip_required_strings(cls, value: str) -> str: + return value.strip() + + @field_validator("allowed") + @classmethod + def _normalize_allowed(cls, value: List[str]) -> List[str]: + return [str(item).strip() for item in value if str(item).strip()] + + @field_validator("evidence") + @classmethod + def _normalize_evidence(cls, value: Optional[str]) -> Optional[str]: + if value is None: + return None + stripped = value.strip() + return stripped or None + + class RequirementResult(BaseModel): model_config = ConfigDict(extra="forbid") @@ -39,8 +89,9 @@ class RequirementResult(BaseModel): label: str status: RequirementStatus reason: str - evidence: Optional[str] = None # "what to look for" hint from policy/rules - evidence_snippets: List[str] = Field(default_factory=list) # snippets from the note that triggered extraction + evidence: Optional[str] = None + evidence_snippets: List[str] = Field(default_factory=list) + evidence_spans: List[EvidenceSpan] = Field(default_factory=list) @field_validator("key", "label", "reason") @classmethod @@ -66,6 +117,180 @@ def _normalize_snippets(cls, value: List[str]) -> List[str]: return snippets +class BlockingIssue(BaseModel): + model_config = ConfigDict(extra="forbid") + + key: str + label: str + status: RequirementStatus + reason: str + + @field_validator("key", "label", "reason") + @classmethod + def _strip_strings(cls, value: str) -> str: + return value.strip() + + +class BlockingIssueSummary(BaseModel): + model_config = ConfigDict(extra="forbid") + + not_documented: List[BlockingIssue] = Field(default_factory=list) + not_met: List[BlockingIssue] = Field(default_factory=list) + + +class EvaluationMetrics(BaseModel): + model_config = ConfigDict(extra="forbid") + + extraction_success_rate: float + extraction_failure_count: int + compliance_rate: Optional[float] = None + compliant_count: int + non_compliant_count: int + + +class ProcedureMetadata(BaseModel): + model_config = ConfigDict(extra="forbid") + + category: str + rule_family: str + summary: str + supported_sites: List[str] = Field(default_factory=list) + last_rule_update: Optional[str] = None + notes: List[str] = Field(default_factory=list) + + @field_validator("category", "rule_family", "summary") + @classmethod + def _strip_strings(cls, value: str) -> str: + return value.strip() + + @field_validator("supported_sites", "notes") + @classmethod + def _normalize_lists(cls, value: List[str]) -> List[str]: + return [str(item).strip() for item in value if str(item).strip()] + + +class ProcedureProvenance(BaseModel): + model_config = ConfigDict(extra="forbid") + + source_name: Optional[str] = None + source_type: Optional[str] = None + source_url: Optional[str] = None + rule_source_label: Optional[str] = None + last_reviewed: Optional[str] = None + rule_last_updated: Optional[str] = None + monitored_source_id: Optional[str] = None + monitored_source_name: Optional[str] = None + monitored_source_url: Optional[str] = None + monitored_check_frequency: Optional[str] = None + monitored_source_owner: Optional[str] = None + notes: Optional[str] = None + + @field_validator( + "source_name", + "source_type", + "source_url", + "rule_source_label", + "last_reviewed", + "rule_last_updated", + "monitored_source_id", + "monitored_source_name", + "monitored_source_url", + "monitored_check_frequency", + "monitored_source_owner", + "notes", + ) + @classmethod + def _strip_optional_strings(cls, value: Optional[str]) -> Optional[str]: + if value is None: + return None + stripped = value.strip() + return stripped or None + + +class SupportedProcedure(BaseModel): + model_config = ConfigDict(extra="forbid") + + payer: str + procedure_code: str + display_name: str + monitored_for_drift: bool = False + policy_trust_level: PolicyTrustLevel = "demo" + required_field_keys: List[str] = Field(default_factory=list) + metadata: ProcedureMetadata + provenance: ProcedureProvenance = Field(default_factory=ProcedureProvenance) + requirements: List[RequirementDefinition] = Field(default_factory=list) + + @field_validator("payer", "procedure_code", "display_name") + @classmethod + def _strip_strings(cls, value: str) -> str: + return value.strip() + + @field_validator("required_field_keys") + @classmethod + def _normalize_required_field_keys(cls, value: List[str]) -> List[str]: + return [str(item).strip() for item in value if str(item).strip()] + + +class DemoCase(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: str + payer: str + procedure_code: str + dx_codes: List[str] = Field(default_factory=list) + site_of_care: str = "outpatient" + specialty: str = "unknown" + note_text: str = "" + expected_label: Optional[str] = None + showcase: Dict[str, Any] = Field(default_factory=dict) + + @field_validator("id", "payer", "procedure_code", "site_of_care", "specialty", "note_text") + @classmethod + def _strip_strings(cls, value: str) -> str: + return value.strip() + + +class AuditTrace(BaseModel): + model_config = ConfigDict(extra="forbid") + + run_id: str + timestamp_utc: str + note_hash: str + note_length: int + payer: str + procedure_code: str + procedure_name: str + site_of_care: str + specialty: str + rules_version: Optional[str] = None + rulebook_active_release_id: Optional[str] = None + policy_trust_level: PolicyTrustLevel + provenance_snapshot: Dict[str, Any] = Field(default_factory=dict) + facts_extracted: Dict[str, Any] = Field(default_factory=dict) + evidence_map: Dict[str, List[EvidenceSpan]] = Field(default_factory=dict) + requirements_checked: List[str] = Field(default_factory=list) + overall_status: OverallStatus + submission_readiness: bool + blocking_issues: BlockingIssueSummary + metrics: EvaluationMetrics + invariant_errors: List[str] = Field(default_factory=list) + evaluation_warnings: List[str] = Field(default_factory=list) + + @field_validator( + "run_id", + "timestamp_utc", + "note_hash", + "payer", + "procedure_code", + "procedure_name", + "site_of_care", + "specialty", + ) + @classmethod + def _strip_strings(cls, value: str) -> str: + return value.strip() + + class ReadinessReport(BaseModel): model_config = ConfigDict(extra="forbid") @@ -96,3 +321,131 @@ def _validate_non_negative_counts(cls, value: int) -> int: @classmethod def _normalize_rule_reasons(cls, value: List[str]) -> List[str]: return [str(reason).strip() for reason in value if str(reason).strip()] + + +class EvaluationResult(BaseModel): + model_config = ConfigDict(extra="forbid") + + request: PARequest + supported_procedure: SupportedProcedure + overall_status: OverallStatus + submission_readiness: bool + readiness_score: int + results: List[RequirementResult] + rule_reasons: List[str] = Field(default_factory=list) + facts: Dict[str, Any] = Field(default_factory=dict) + evidence_map: Dict[str, List[EvidenceSpan]] = Field(default_factory=dict) + blockers: BlockingIssueSummary + metrics: EvaluationMetrics + warnings: List[str] = Field(default_factory=list) + policy_trust_level: PolicyTrustLevel + provenance: Dict[str, Any] = Field(default_factory=dict) + audit_trail: AuditTrace + report: ReadinessReport + + +class DriftSourceStatus(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: str + payer: str + procedure_code: str + source_name: Optional[str] = None + source_type: str + url: str + trust_level: str + check_frequency: str + owner: str + status: str + last_checked_utc: Optional[str] = None + days_since_last_checked: Optional[int] = None + freshness_status: Optional[str] = None + latest_hash: Optional[str] = None + latest_event: Optional[str] = None + latest_snapshot_path: Optional[str] = None + latest_diff_path: Optional[str] = None + rule_source_label: Optional[str] = None + last_rule_reviewed: Optional[str] = None + review_reason: Optional[str] = None + notes: Optional[str] = None + + +class DriftStatusReport(BaseModel): + model_config = ConfigDict(extra="forbid") + + sources: List[DriftSourceStatus] = Field(default_factory=list) + any_review_required: bool = False + stale_source_count: int = 0 + + +class RulebookFileSet(BaseModel): + model_config = ConfigDict(extra="forbid") + + rules_path: str + provenance_path: str + policy_sources_path: str + + +class RulebookRelease(BaseModel): + model_config = ConfigDict(extra="forbid") + + release_id: str + stage: Optional[RulebookStage] = None + summary: str + created_at: Optional[str] = None + based_on_release_id: Optional[str] = None + rules_version: Optional[str] = None + procedures: List[str] = Field(default_factory=list) + files: RulebookFileSet + reviewer: Optional[str] = None + reviewed_at: Optional[str] = None + runtime_matches: Optional[bool] = None + notes: List[str] = Field(default_factory=list) + + +class RulebookStatusResponse(BaseModel): + model_config = ConfigDict(extra="forbid") + + manifest_version: Optional[str] = None + active_release_id: Optional[str] = None + stage_assignments: Dict[str, Optional[str]] = Field(default_factory=dict) + runtime_rules_version: Optional[str] = None + releases: List[RulebookRelease] = Field(default_factory=list) + validation_errors: List[str] = Field(default_factory=list) + + +class RulebookDiffResponse(BaseModel): + model_config = ConfigDict(extra="forbid") + + from_release_id: str + to_release_id: str + from_stage: Optional[RulebookStage] = None + to_stage: Optional[RulebookStage] = None + rules_version_from: Optional[str] = None + rules_version_to: Optional[str] = None + added_procedures: List[str] = Field(default_factory=list) + removed_procedures: List[str] = Field(default_factory=list) + changed_procedures: List[str] = Field(default_factory=list) + changed_provenance: List[str] = Field(default_factory=list) + changed_policy_sources: List[str] = Field(default_factory=list) + summary_lines: List[str] = Field(default_factory=list) + + +class StatusResponse(BaseModel): + model_config = ConfigDict(extra="forbid") + + service: str + rules_version: Optional[str] = None + rulebook_active_release_id: Optional[str] = None + rulebook_active_rules_version: Optional[str] = None + supported_procedures: int + demo_cases: int + monitored_policy_sources: int + synthetic_only: bool = True + + +class ErrorResponse(BaseModel): + model_config = ConfigDict(extra="forbid") + + error: str + detail: str diff --git a/engine/score.py b/engine/score.py deleted file mode 100644 index 8b13789..0000000 --- a/engine/score.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/engine/service.py b/engine/service.py new file mode 100644 index 0000000..ded9d4c --- /dev/null +++ b/engine/service.py @@ -0,0 +1,531 @@ +from __future__ import annotations + +import json +import logging +import uuid +from datetime import datetime, timezone +from functools import cached_property +from hashlib import sha256 +from pathlib import Path +from typing import Any, Dict, List + +from .config import AppConfig, load_app_config +from .demo_cases import demo_case_to_request, get_demo_case, list_demo_cases +from .evaluate import compute_overall_status, compute_readiness_score, evaluate_requirements +from .extract import extract_facts +from .letter_draft import draft_letter +from .logging_utils import configure_logging, get_logger, log_event +from .policy_monitor import load_policy_sources, read_latest_snapshot +from .provenance import ( + get_provenance_entry, + load_provenance, + normalized_dx_codes, + policy_trust_from_provenance, +) +from .rulebook import RulebookError, get_rulebook_diff, get_rulebook_status +from .rules_loader import load_rules +from .schemas import ( + AuditTrace, + BlockingIssue, + BlockingIssueSummary, + DemoCase, + DriftSourceStatus, + DriftStatusReport, + EvaluationMetrics, + EvaluationResult, + EvidenceSpan, + LetterType, + PARequest, + ProcedureMetadata, + ProcedureProvenance, + ReadinessReport, + RequirementDefinition, + RequirementResult, + RulebookDiffResponse, + RulebookFileSet, + RulebookStatusResponse, + StatusResponse, + SupportedProcedure, +) + + +class ServiceError(Exception): + code = "service_error" + + +class InvalidRequestError(ServiceError): + code = "invalid_request" + + +class UnsupportedScopeError(ServiceError): + code = "unsupported_scope" + + +class GovernanceConfigError(ServiceError): + code = "governance_config_error" + + +def _hash_note(note_text: str) -> str: + return sha256((note_text or "").encode("utf-8")).hexdigest()[:16] + + +def _utc_now_iso() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def _parse_utc_iso(value: str | None) -> datetime | None: + if not value: + return None + normalized = str(value).strip() + if not normalized: + return None + if normalized.endswith("Z"): + normalized = normalized[:-1] + "+00:00" + try: + return datetime.fromisoformat(normalized) + except ValueError: + return None + + +def _freshness_window_days(check_frequency: str) -> int | None: + mapping = { + "hourly": 1, + "daily": 2, + "weekly": 8, + "monthly": 35, + } + return mapping.get(str(check_frequency or "").strip().lower()) + + +def _display_repo_relative_path(repo_root: Path, raw_path: str | Path | None) -> str | None: + if raw_path is None: + return None + path = Path(str(raw_path)) + if not path.is_absolute(): + return path.as_posix() + try: + return path.relative_to(repo_root).as_posix() + except ValueError: + return path.as_posix() + + +def _compute_metrics(score_info: Dict[str, int]) -> EvaluationMetrics: + total = int(score_info.get("total", 0) or 0) + met = int(score_info.get("met_count", 0) or 0) + not_met = int(score_info.get("not_met_count", 0) or 0) + not_doc = int(score_info.get("not_documented_count", 0) or 0) + + extraction_success_rate = round(((met + not_met) / total * 100), 1) if total else 0.0 + compliance_rate = round((met / (met + not_met) * 100), 1) if (met + not_met) > 0 else None + + return EvaluationMetrics( + extraction_success_rate=extraction_success_rate, + extraction_failure_count=not_doc, + compliance_rate=compliance_rate, + compliant_count=met, + non_compliant_count=not_met, + ) + + +def _read_drift_log(log_path: Path) -> List[Dict[str, Any]]: + if not log_path.exists(): + return [] + + events: List[Dict[str, Any]] = [] + with log_path.open("r", encoding="utf-8") as handle: + for line in handle: + line = line.strip() + if not line: + continue + try: + payload = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(payload, dict): + events.append(payload) + return events + + +class ReadinessService: + def __init__(self, config: AppConfig | None = None) -> None: + self.config = config or load_app_config() + configure_logging(self.config.log_level) + self.logger = get_logger("pa_copilot.service") + + @cached_property + def rules(self) -> Dict[str, Any]: + return load_rules(str(self.config.rules_path)) + + @cached_property + def provenance(self) -> Dict[str, Any]: + return load_provenance(self.config.provenance_path) + + @cached_property + def policy_sources(self): + return load_policy_sources(self.config.policy_sources_path) + + @cached_property + def policy_source_by_procedure(self) -> Dict[tuple[str, str], Any]: + return {(source.payer, source.procedure_code): source for source in self.policy_sources} + + @cached_property + def demo_cases(self) -> List[DemoCase]: + return list_demo_cases(self.config) + + def list_supported_procedures(self) -> List[SupportedProcedure]: + out: List[SupportedProcedure] = [] + + for payer, payer_config in self.rules["payers"].items(): + for procedure_code, procedure in payer_config["procedures"].items(): + provenance_entry = get_provenance_entry(self.provenance, payer, procedure_code) + requirements = [RequirementDefinition.model_validate(requirement) for requirement in procedure.get("required", [])] + metadata = self._build_procedure_metadata(procedure, requirements) + provenance = self._build_procedure_provenance(payer, procedure_code, provenance_entry) + out.append( + SupportedProcedure( + payer=payer, + procedure_code=procedure_code, + display_name=procedure.get("display_name", procedure_code), + monitored_for_drift=(payer, procedure_code) in self.policy_source_by_procedure, + policy_trust_level=policy_trust_from_provenance(provenance_entry), + required_field_keys=[requirement.key for requirement in requirements], + metadata=metadata, + provenance=provenance, + requirements=requirements, + ) + ) + + return sorted(out, key=lambda item: (item.payer, item.procedure_code)) + + def get_supported_procedure(self, payer: str, procedure_code: str) -> SupportedProcedure: + for supported in self.list_supported_procedures(): + if supported.payer == payer and supported.procedure_code == procedure_code: + return supported + raise UnsupportedScopeError( + f"Unsupported request scope: payer='{payer}', procedure_code='{procedure_code}'. " + "Use /supported-procedures or the CLI list command to inspect current demo support." + ) + + def list_demo_case_summaries(self) -> List[DemoCase]: + return self.demo_cases + + def get_rulebook_status(self) -> RulebookStatusResponse: + try: + return get_rulebook_status( + self.config.repo_root, + self.config.rulebook_manifest_path, + RulebookFileSet( + rules_path=self.config.rules_path.as_posix(), + provenance_path=self.config.provenance_path.as_posix(), + policy_sources_path=self.config.policy_sources_path.as_posix(), + ), + ) + except RulebookError as exc: + raise GovernanceConfigError(str(exc)) from exc + + def get_rulebook_diff(self, from_release_id: str, to_release_id: str) -> RulebookDiffResponse: + try: + return get_rulebook_diff( + self.config.repo_root, + self.config.rulebook_manifest_path, + from_release_id=from_release_id, + to_release_id=to_release_id, + ) + except RulebookError as exc: + raise GovernanceConfigError(str(exc)) from exc + + def get_demo_case_request(self, case_id: str) -> PARequest: + case = get_demo_case(case_id, self.config) + return demo_case_to_request(case) + + def validate_request(self, request: PARequest) -> List[str]: + warnings: List[str] = [] + + if request.site_of_care not in self.config.allowed_sites: + raise UnsupportedScopeError( + f"Unsupported site_of_care '{request.site_of_care}'. Supported demo sites: {', '.join(self.config.allowed_sites)}." + ) + + if not request.note_text.strip(): + warnings.append("No note text provided; missing requirements will force CANNOT_DETERMINE.") + + if not request.dx_codes: + warnings.append("No diagnosis codes supplied; readiness is evaluated from note content and rule scope only.") + + specialty = request.specialty.strip().lower() + if not specialty or specialty == "unknown": + warnings.append("Ordering specialty not supplied; retained as 'unknown' for audit trace completeness.") + + return warnings + + def evaluate(self, request: PARequest) -> EvaluationResult: + normalized_request = request.model_copy( + update={ + "dx_codes": normalized_dx_codes(request.dx_codes), + "specialty": request.specialty or "unknown", + } + ) + warnings = self.validate_request(normalized_request) + supported = self.get_supported_procedure(normalized_request.payer, normalized_request.procedure_code) + + raw_facts, raw_evidence_map = extract_facts(normalized_request.note_text) + requirement_payloads = [requirement.model_dump(exclude_none=True) for requirement in supported.requirements] + results, reasons = evaluate_requirements(requirement_payloads, raw_facts, evidence_map=raw_evidence_map) + + overall = compute_overall_status(results) + score_info = compute_readiness_score(results) + metrics = _compute_metrics(score_info) + blockers = self._build_blockers(results) + invariant_errors = self._compute_invariant_errors(blockers, overall["overall_status"]) + + provenance_entry = get_provenance_entry(self.provenance, normalized_request.payer, normalized_request.procedure_code) + policy_trust_level = policy_trust_from_provenance(provenance_entry) + if policy_trust_level != "verified": + warnings.append("Policy trust remains DEMO for this procedure. Verify against official policy before real-world use.") + + structured_provenance = supported.provenance.model_dump(mode="json") + rulebook_status = self.get_rulebook_status() + + audit = AuditTrace( + run_id=str(uuid.uuid4()), + timestamp_utc=_utc_now_iso(), + note_hash=_hash_note(normalized_request.note_text), + note_length=len(normalized_request.note_text or ""), + payer=normalized_request.payer, + procedure_code=normalized_request.procedure_code, + procedure_name=supported.display_name, + site_of_care=normalized_request.site_of_care, + specialty=normalized_request.specialty, + rules_version=str(self.rules.get("version")) if self.rules.get("version") is not None else None, + rulebook_active_release_id=rulebook_status.active_release_id, + policy_trust_level=policy_trust_level, + provenance_snapshot=structured_provenance, + facts_extracted=raw_facts, + evidence_map=self._coerce_evidence_map(raw_evidence_map), + requirements_checked=[result.key for result in results], + overall_status=overall["overall_status"], + submission_readiness=bool(overall["submission_readiness"]), + blocking_issues=blockers, + metrics=metrics, + invariant_errors=invariant_errors, + evaluation_warnings=warnings, + ) + + report = ReadinessReport( + readiness_score=int(score_info.get("readiness_score", 0) or 0), + not_documented_count=int(score_info.get("not_documented_count", 0) or 0), + not_met_count=int(score_info.get("not_met_count", 0) or 0), + met_count=int(score_info.get("met_count", 0) or 0), + results=results, + rule_reasons=reasons, + audit_trail=audit.model_dump(mode="json"), + letter_draft="", + ) + + result = EvaluationResult( + request=normalized_request, + supported_procedure=supported, + overall_status=overall["overall_status"], + submission_readiness=bool(overall["submission_readiness"]), + readiness_score=report.readiness_score, + results=results, + rule_reasons=reasons, + facts=raw_facts, + evidence_map=self._coerce_evidence_map(raw_evidence_map), + blockers=blockers, + metrics=metrics, + warnings=warnings, + policy_trust_level=policy_trust_level, + provenance=structured_provenance, + audit_trail=audit, + report=report, + ) + + log_event( + self.logger, + logging.INFO, + "readiness_evaluated", + payer=normalized_request.payer, + procedure_code=normalized_request.procedure_code, + overall_status=result.overall_status, + submission_readiness=result.submission_readiness, + note_hash=audit.note_hash, + blockers_missing=len(blockers.not_documented), + blockers_not_met=len(blockers.not_met), + ) + return result + + def generate_letter( + self, evaluation: EvaluationResult, letter_type: LetterType = "submission_cover_letter" + ) -> tuple[str, Dict[str, Any]]: + return draft_letter( + evaluation.request.model_copy(update={"note_text": ""}), + evaluation.report, + letter_type=letter_type, + policy_trust_level=evaluation.policy_trust_level, + ) + + def get_drift_status(self) -> DriftStatusReport: + events = _read_drift_log(self.config.snapshot_root / "drift_log.jsonl") + latest_event_by_id = {str(event.get("id")): event for event in events if event.get("id")} + + statuses: List[DriftSourceStatus] = [] + any_review_required = False + stale_source_count = 0 + + for source in self.policy_sources: + latest_snapshot = read_latest_snapshot(self.config.snapshot_root, source.id) + status = "NO_BASELINE" if latest_snapshot is None else "OK" + event = latest_event_by_id.get(source.id, {}) + if event.get("event") == "POLICY_DRIFT_DETECTED": + status = "REVIEW_REQUIRED" + any_review_required = True + + provenance_entry = get_provenance_entry(self.provenance, source.payer, source.procedure_code) + last_checked_utc = (latest_snapshot or {}).get("fetched_at_utc") + last_checked_dt = _parse_utc_iso(last_checked_utc) + freshness_window_days = _freshness_window_days(source.check_frequency) + days_since_last_checked = None + freshness_status = "UNKNOWN" + review_reason = None + + if last_checked_dt is not None: + age_seconds = max((datetime.now(timezone.utc) - last_checked_dt).total_seconds(), 0) + days_since_last_checked = int(age_seconds // 86400) + freshness_status = "CURRENT" + if freshness_window_days is not None and age_seconds > freshness_window_days * 86400: + freshness_status = "STALE" + stale_source_count += 1 + any_review_required = True + review_reason = f"Snapshot exceeds the configured {source.check_frequency} monitoring window." + elif latest_snapshot is None: + any_review_required = True + review_reason = "No baseline snapshot exists yet for this monitored source." + + if status == "REVIEW_REQUIRED": + review_reason = "Detected policy drift requires human rule review before the related rule should be trusted." + + statuses.append( + DriftSourceStatus( + id=source.id, + payer=source.payer, + procedure_code=source.procedure_code, + source_name=source.source_name, + source_type=source.source_type, + url=source.url, + trust_level=source.trust_level, + check_frequency=source.check_frequency, + owner=source.owner, + status=status, + last_checked_utc=last_checked_utc, + days_since_last_checked=days_since_last_checked, + freshness_status=freshness_status, + latest_hash=(latest_snapshot or {}).get("content_hash_sha256"), + latest_event=event.get("event"), + latest_snapshot_path=_display_repo_relative_path( + self.config.repo_root, + self.config.snapshot_root / source.id / "latest.json", + ), + latest_diff_path=_display_repo_relative_path(self.config.repo_root, event.get("diff_path")), + rule_source_label=str( + provenance_entry.get("rule_source_label") or provenance_entry.get("source_name") or "" + ) + or None, + last_rule_reviewed=str(provenance_entry.get("last_reviewed") or "") or None, + review_reason=review_reason, + notes=str(source.notes or provenance_entry.get("notes") or "").strip() or None, + ) + ) + + return DriftStatusReport( + sources=sorted(statuses, key=lambda item: (item.payer, item.procedure_code)), + any_review_required=any_review_required, + stale_source_count=stale_source_count, + ) + + def get_status(self) -> StatusResponse: + rulebook_status = self.get_rulebook_status() + return StatusResponse( + service="Prior Authorization Readiness Copilot", + rules_version=str(self.rules.get("version")) if self.rules.get("version") is not None else None, + rulebook_active_release_id=rulebook_status.active_release_id, + rulebook_active_rules_version=rulebook_status.runtime_rules_version, + supported_procedures=len(self.list_supported_procedures()), + demo_cases=len(self.demo_cases), + monitored_policy_sources=len(self.policy_sources), + synthetic_only=True, + ) + + @staticmethod + def _coerce_evidence_map(raw_evidence_map: Dict[str, Any]) -> Dict[str, List[EvidenceSpan]]: + out: Dict[str, List[EvidenceSpan]] = {} + for key, spans in (raw_evidence_map or {}).items(): + out[key] = [] + for span in spans or []: + if not isinstance(span, dict): + continue + start = span.get("start") + end = span.get("end") + text = str(span.get("text", "")).strip() + if not isinstance(start, int) or not isinstance(end, int) or end <= start or start < 0 or not text: + continue + out[key].append(EvidenceSpan(start=start, end=end, text=text)) + return out + + @staticmethod + def _build_blockers(results: List[RequirementResult]) -> BlockingIssueSummary: + missing = [ + BlockingIssue(key=result.key, label=result.label, status=result.status, reason=result.reason) + for result in results + if result.status == "NOT_DOCUMENTED" + ] + not_met = [ + BlockingIssue(key=result.key, label=result.label, status=result.status, reason=result.reason) + for result in results + if result.status == "NOT_MET" + ] + return BlockingIssueSummary(not_documented=missing, not_met=not_met) + + @staticmethod + def _compute_invariant_errors(blockers: BlockingIssueSummary, overall_status: str) -> List[str]: + errors: List[str] = [] + if blockers.not_documented and overall_status != "CANNOT_DETERMINE": + errors.append("Invariant violation: NOT_DOCUMENTED blockers exist but overall_status is not CANNOT_DETERMINE.") + if (not blockers.not_documented) and blockers.not_met and overall_status == "READY": + errors.append("Invariant violation: NOT_MET blockers exist but overall_status is READY.") + if (not blockers.not_documented) and (not blockers.not_met) and overall_status != "READY": + errors.append("Invariant violation: no blockers exist but overall_status is not READY.") + return errors + + @staticmethod + def _build_procedure_metadata( + procedure: Dict[str, Any], requirements: List[RequirementDefinition] + ) -> ProcedureMetadata: + raw_metadata = procedure.get("metadata") or {} + return ProcedureMetadata( + category=str(raw_metadata.get("category") or "administrative_review"), + rule_family=str(raw_metadata.get("rule_family") or "deterministic_readiness"), + summary=str(raw_metadata.get("summary") or procedure.get("display_name") or "Procedure rule set"), + supported_sites=[str(site) for site in raw_metadata.get("supported_sites") or ["outpatient"]], + last_rule_update=raw_metadata.get("last_rule_update"), + notes=[str(note) for note in raw_metadata.get("notes") or []], + ) + + def _build_procedure_provenance( + self, payer: str, procedure_code: str, provenance_entry: Dict[str, Any] + ) -> ProcedureProvenance: + policy_source = self.policy_source_by_procedure.get((payer, procedure_code)) + return ProcedureProvenance( + source_name=provenance_entry.get("source_name"), + source_type=provenance_entry.get("source_type"), + source_url=provenance_entry.get("source_url"), + rule_source_label=provenance_entry.get("rule_source_label") or provenance_entry.get("source_name"), + last_reviewed=provenance_entry.get("last_reviewed"), + rule_last_updated=provenance_entry.get("rule_last_updated"), + monitored_source_id=policy_source.id if policy_source else provenance_entry.get("monitored_source_id"), + monitored_source_name=policy_source.source_name if policy_source else None, + monitored_source_url=policy_source.url if policy_source else None, + monitored_check_frequency=policy_source.check_frequency if policy_source else None, + monitored_source_owner=policy_source.owner if policy_source else None, + notes=provenance_entry.get("notes") or (policy_source.notes if policy_source else None), + ) diff --git a/engine/test_suites.py b/engine/test_suites.py index da77021..a238099 100644 --- a/engine/test_suites.py +++ b/engine/test_suites.py @@ -3,13 +3,13 @@ import json from typing import Any, Dict, List -from engine.rules_loader import load_rules -from engine.extract import extract_facts from engine.evaluate import ( - evaluate_requirements, - compute_readiness_score, compute_overall_status, + compute_readiness_score, + evaluate_requirements, ) +from engine.extract import extract_facts +from engine.rules_loader import load_rules def label_from_outputs(overall_status: str) -> str: diff --git a/inputs/synthetic_cases.json b/inputs/synthetic_cases.json index a752f75..5233986 100644 --- a/inputs/synthetic_cases.json +++ b/inputs/synthetic_cases.json @@ -14,7 +14,9 @@ "title": "Lumbar MRI ready for administrative review", "description": "A clean lumbar MRI request with therapy duration, symptom duration, imaging context, and red-flag documentation all present.", "expected_overall_status": "READY", - "why_interesting": "Shows the straight-line READY path with evidence snippets across each requirement." + "why_interesting": "Shows the straight-line READY path with evidence snippets across each requirement.", + "scenario_type": "Ready path", + "tags": ["spine MRI", "happy path", "evidence map"] } }, { @@ -92,7 +94,9 @@ "title": "CPAP request refused for missing documentation", "description": "The note mentions OSA and a past sleep study, but it leaves out the date and AHI details required to determine readiness.", "expected_overall_status": "CANNOT_DETERMINE", - "why_interesting": "Shows refusal-first behavior when required evidence is missing instead of inferred." + "why_interesting": "Shows refusal-first behavior when required evidence is missing instead of inferred.", + "scenario_type": "Refusal-first", + "tags": ["DME", "missing documentation", "cannot determine"] } }, { @@ -142,7 +146,9 @@ "title": "Documented, but not yet ready to submit", "description": "This note documents the right fields, but both symptom duration and therapy duration are still below threshold.", "expected_overall_status": "NOT_READY", - "why_interesting": "Shows the difference between documented failure and missingness." + "why_interesting": "Shows the difference between documented failure and missingness.", + "scenario_type": "Documented failure", + "tags": ["threshold miss", "not ready", "fully documented"] } }, { @@ -240,9 +246,121 @@ "title": "CPAP ready case with alternative date formatting", "description": "A CPAP request that includes OSA documentation, a slash-formatted sleep study date, and an RDI value.", "expected_overall_status": "READY", - "why_interesting": "Shows cross-domain breadth and a small extraction robustness edge case." + "why_interesting": "Shows cross-domain breadth and a small extraction robustness edge case.", + "scenario_type": "Ready path", + "tags": ["date parsing", "DME", "cross-domain"] } }, + { + "id": "MRI-CERV-01-ready", + "payer": "Aetna", + "procedure_code": "MRI_CERVICAL", + "dx_codes": ["M54.12"], + "site_of_care": "outpatient", + "specialty": "Orthopedics", + "note_text": "Neck pain with right arm radiculopathy x 8 weeks. PT for 8 weeks and NSAIDs documented with minimal improvement. Denies weakness. Denies bowel/bladder changes. Prior cervical xray inconclusive.", + "expected_label": "complete", + "showcase": { + "featured": true, + "sort_order": 5, + "title": "Cervical MRI ready case using the new supported pathway", + "description": "A second spine MRI procedure that uses the same deterministic evidence contract as lumbar MRI.", + "expected_overall_status": "READY", + "why_interesting": "Shows that the shared service and rule registry can support new deterministic procedures without changing the engine shape.", + "scenario_type": "New procedure coverage", + "tags": ["cervical MRI", "registry depth", "shared contract"] + } + }, + { + "id": "MRI-CERV-02-not-ready", + "payer": "Aetna", + "procedure_code": "MRI_CERVICAL", + "dx_codes": ["M54.2"], + "site_of_care": "outpatient", + "specialty": "Primary Care", + "note_text": "Neck pain x 4 weeks. PT x 4 weeks documented. Denies weakness. Denies bowel/bladder changes. Prior cervical xray unremarkable.", + "expected_label": "incomplete" + }, + { + "id": "MRI-CERV-03-cannot-determine", + "payer": "Aetna", + "procedure_code": "MRI_CERVICAL", + "dx_codes": ["M54.12"], + "site_of_care": "outpatient", + "specialty": "Primary Care", + "note_text": "Neck pain x 8 weeks. PT x 8 weeks documented. No prior imaging. No red flags mentioned.", + "expected_label": "incomplete" + }, + { + "id": "MRI-CERV-04-positive-redflags", + "payer": "Aetna", + "procedure_code": "MRI_CERVICAL", + "dx_codes": ["M54.12"], + "site_of_care": "outpatient", + "specialty": "Neurology", + "note_text": "Cervical radicular pain x 2 months. Completed physical therapy for 6 weeks. Reports progressive weakness in the right hand. Prior CT described as abnormal with degenerative stenosis.", + "expected_label": "complete" + }, + { + "id": "MRI-KNEE-01-ready", + "payer": "Aetna", + "procedure_code": "MRI_KNEE", + "dx_codes": ["M25.561"], + "site_of_care": "outpatient", + "specialty": "Orthopedics", + "note_text": "Right knee pain with locking and catching x 8 weeks. Completed PT for 8 weeks and NSAIDs with minimal improvement. Prior knee xray normal/unremarkable.", + "expected_label": "complete", + "showcase": { + "featured": true, + "sort_order": 6, + "title": "Knee MRI ready case with explicit mechanical symptoms", + "description": "A non-spine advanced imaging request that stays narrow: symptom duration, therapy duration, prior xray context, and explicit mechanical symptom review.", + "expected_overall_status": "READY", + "why_interesting": "Shows deeper deterministic product coverage without changing the engine shape or widening into clinical scoring.", + "scenario_type": "Non-spine coverage", + "tags": ["knee MRI", "orthopedics", "mechanical symptoms"] + } + }, + { + "id": "MRI-KNEE-02-not-ready-no-imaging", + "payer": "Aetna", + "procedure_code": "MRI_KNEE", + "dx_codes": ["M25.561"], + "site_of_care": "outpatient", + "specialty": "Orthopedics", + "note_text": "Right knee pain with locking x 8 weeks. PT x 8 weeks and activity modification documented. No prior imaging yet.", + "expected_label": "incomplete" + }, + { + "id": "MRI-KNEE-03-cannot-determine", + "payer": "Aetna", + "procedure_code": "MRI_KNEE", + "dx_codes": ["M25.561"], + "site_of_care": "outpatient", + "specialty": "Primary Care", + "note_text": "Right knee pain x 8 weeks. PT x 8 weeks documented. Prior knee xray normal/unremarkable. MRI requested.", + "expected_label": "incomplete" + }, + { + "id": "MRI-KNEE-04-ready-denied-mechanical", + "payer": "Aetna", + "procedure_code": "MRI_KNEE", + "dx_codes": ["M25.561"], + "site_of_care": "outpatient", + "specialty": "Sports Medicine", + "note_text": "Right knee pain x 10 weeks. Completed PT for 6 weeks and home exercise. Denies locking or instability. Prior knee xray findings unclear.", + "expected_label": "complete" + }, + { + "id": "MRI-KNEE-05-conflict-deny-then-positive", + "payer": "Aetna", + "procedure_code": "MRI_KNEE", + "dx_codes": ["M25.561"], + "site_of_care": "outpatient", + "specialty": "Orthopedics", + "note_text": "Right knee pain x 8 weeks. PT x 8 weeks documented. Denies locking earlier in visit. Later note: reports buckling with stairs. Prior knee xray normal.", + "expected_label": "complete" + }, { "id": "CPAP-08-doc-gap-osa-not-stated", "payer": "Aetna", diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..7176e92 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,16 @@ +[tool.ruff] +line-length = 140 +target-version = "py312" +exclude = [ + "docs/artifacts", +] + +[tool.ruff.lint] +select = [ + "E", + "F", + "I", +] + +[tool.pytest.ini_options] +testpaths = ["test"] diff --git a/requirements.txt b/requirements.txt index 327d6bb..1131932 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,7 @@ pyyaml==6.0.3 pydantic==2.12.5 requests==2.32.5 pytest==9.0.2 +fastapi==0.115.14 +uvicorn==0.35.0 +httpx==0.28.1 +ruff==0.12.0 diff --git a/rulebook/manifest.yaml b/rulebook/manifest.yaml new file mode 100644 index 0000000..39107ef --- /dev/null +++ b/rulebook/manifest.yaml @@ -0,0 +1,55 @@ +version: 1 + +runtime: + rules: "rules/payer_rules.yaml" + provenance: "rules/provenance.yaml" + policy_sources: "rules/policy_sources.yaml" + +stages: + draft: + reviewed: "2026-04-09-reviewed-v0.4" + active: "2026-04-09-active-v0.5" + +releases: + 2026-04-09-reviewed-v0.4: + stage: reviewed + summary: "Reviewed rulebook before the narrow non-spine knee MRI expansion." + created_at: "2026-04-09T10:45:00Z" + rules_version: "0.4" + procedures: + - "CPAP_DEVICE" + - "MRI_CERVICAL" + - "MRI_LUMBAR" + files: + rules: "rulebook/releases/2026-04-09-reviewed-v0.4/payer_rules.yaml" + provenance: "rulebook/releases/2026-04-09-reviewed-v0.4/provenance.yaml" + policy_sources: "rulebook/releases/2026-04-09-reviewed-v0.4/policy_sources.yaml" + reviewer: "demo-maintainer" + reviewed_at: "2026-04-09" + notes: + - "Historical reviewed snapshot retained for governance diffs." + + 2026-04-09-active-v0.5: + stage: active + summary: "Active rulebook after the narrow non-spine knee MRI expansion." + created_at: "2026-04-09T12:15:00Z" + based_on_release_id: "2026-04-09-reviewed-v0.4" + rules_version: "0.5" + procedures: + - "CPAP_DEVICE" + - "MRI_CERVICAL" + - "MRI_KNEE" + - "MRI_LUMBAR" + files: + rules: "rulebook/releases/2026-04-09-active-v0.5/payer_rules.yaml" + provenance: "rulebook/releases/2026-04-09-active-v0.5/provenance.yaml" + policy_sources: "rulebook/releases/2026-04-09-active-v0.5/policy_sources.yaml" + reviewer: "demo-maintainer" + reviewed_at: "2026-04-09" + notes: + - "This active snapshot should match the runtime files under rules/." + +promotion_workflow: + draft: "Create a candidate snapshot first. Keep it out of runtime until a human reviewer validates the evidence contract and metadata." + reviewed: "Mark a snapshot reviewed only after a human has validated rule text, provenance, and demo fixtures." + active: "Promote to active by updating the manifest stage pointer and syncing runtime rules deliberately. Drift monitoring never promotes rules automatically." diff --git a/rulebook/releases/2026-04-09-active-v0.5/payer_rules.yaml b/rulebook/releases/2026-04-09-active-v0.5/payer_rules.yaml new file mode 100644 index 0000000..0f8fdf8 --- /dev/null +++ b/rulebook/releases/2026-04-09-active-v0.5/payer_rules.yaml @@ -0,0 +1,132 @@ +version: 0.5 + +payers: + Aetna: + procedures: + MRI_LUMBAR: + display_name: "MRI Lumbar Spine (no contrast)" + metadata: + category: "advanced_imaging" + rule_family: "spine_mri_conservative_therapy" + summary: "Administrative readiness check for lumbar spine MRI requests using a narrow deterministic evidence contract." + supported_sites: ["outpatient"] + last_rule_update: "2026-04-09" + notes: + - "Demo rule set only; not a substitute for payer policy review." + - "Requires conservative therapy, symptom duration, imaging context, and explicit red-flag review." + required: + - key: conservative_therapy_weeks + label: "Conservative therapy duration (weeks)" + type: number + min: 6 + evidence: "PT/NSAIDs/activity modification trial documented" + - key: neuro_red_flags_documented + label: "Neuro red flags explicitly addressed (present or denied)" + type: boolean + evidence: "Weakness, saddle anesthesia, bowel/bladder changes explicitly documented as present or denied" + - key: prior_imaging_result + label: "Prior imaging result" + type: enum + allowed: ["none", "inconclusive", "abnormal"] + evidence: "X-ray/CT results or note of none" + - key: symptom_duration_weeks + label: "Symptom duration (weeks)" + type: number + min: 6 + evidence: "Duration supports guideline-based escalation" + + MRI_CERVICAL: + display_name: "MRI Cervical Spine (no contrast)" + metadata: + category: "advanced_imaging" + rule_family: "spine_mri_conservative_therapy" + summary: "Administrative readiness check for cervical spine MRI requests using the same narrow deterministic evidence contract as lumbar MRI." + supported_sites: ["outpatient"] + last_rule_update: "2026-04-09" + notes: + - "Intentionally mirrors the lumbar MRI demo pathway to show reusable deterministic architecture." + - "Human review remains required before any real submission." + required: + - key: conservative_therapy_weeks + label: "Conservative therapy duration (weeks)" + type: number + min: 6 + evidence: "PT/NSAIDs/activity modification trial documented" + - key: neuro_red_flags_documented + label: "Neuro red flags explicitly addressed (present or denied)" + type: boolean + evidence: "Weakness, numbness, bowel/bladder changes, or related escalation findings explicitly documented" + - key: prior_imaging_result + label: "Prior imaging result" + type: enum + allowed: ["none", "inconclusive", "abnormal"] + evidence: "X-ray/CT results or note of none" + - key: symptom_duration_weeks + label: "Symptom duration (weeks)" + type: number + min: 6 + evidence: "Duration supports guideline-based escalation" + + MRI_KNEE: + display_name: "MRI Knee (no contrast)" + metadata: + category: "advanced_imaging" + rule_family: "extremity_mri_conservative_therapy" + summary: "Administrative readiness check for knee MRI requests using a narrow deterministic documentation contract." + supported_sites: ["outpatient"] + last_rule_update: "2026-04-09" + notes: + - "This demo pathway checks only administrative completeness, not orthopedic appropriateness." + - "Requires symptom duration, conservative therapy, prior imaging context, and explicit review of mechanical symptoms." + required: + - key: conservative_therapy_weeks + label: "Conservative therapy duration (weeks)" + type: number + min: 6 + evidence: "PT/activity modification/NSAID trial documented" + - key: symptom_duration_weeks + label: "Symptom duration (weeks)" + type: number + min: 6 + evidence: "Persistent symptoms documented long enough to justify escalation" + - key: prior_imaging_result + label: "Prior imaging result" + type: enum + allowed: ["inconclusive", "abnormal"] + evidence: "Prior knee imaging such as x-ray is documented and not simply absent" + - key: mechanical_symptoms_documented + label: "Mechanical symptoms explicitly addressed (present or denied)" + type: boolean + evidence: "Locking, catching, buckling, giving way, or instability explicitly documented" + + CPAP_DEVICE: + display_name: "CPAP Device (HCPCS E0601)" + metadata: + category: "durable_medical_equipment" + rule_family: "sleep_study_documentation" + summary: "Administrative readiness check for CPAP device requests using a narrow deterministic documentation contract." + supported_sites: ["outpatient"] + last_rule_update: "2026-04-09" + notes: + - "The current demo checks for OSA diagnosis, dated sleep study evidence, and AHI/RDI documentation." + - "It does not determine clinical appropriateness or device approval likelihood." + required: + - key: osa_diagnosis + label: "OSA diagnosis documented" + type: boolean + evidence: "Diagnosis present" + - key: sleep_study_date + label: "Sleep study date documented" + type: boolean + evidence: "Date in chart" + - key: ahi_documented + label: "AHI/RDI documented" + type: boolean + evidence: "AHI value included" + + +policy_notes: + - "Conservative therapy often required prior to advanced imaging." + - "Red flags warrant escalation even without long conservative management." + - "Cervical MRI is intentionally modeled with the same narrow deterministic contract as lumbar MRI in this demo." + - "Knee MRI adds a narrow orthopedic documentation contract without introducing medical-necessity scoring." diff --git a/rulebook/releases/2026-04-09-active-v0.5/policy_sources.yaml b/rulebook/releases/2026-04-09-active-v0.5/policy_sources.yaml new file mode 100644 index 0000000..c1e3440 --- /dev/null +++ b/rulebook/releases/2026-04-09-active-v0.5/policy_sources.yaml @@ -0,0 +1,12 @@ +version: 1.0 +sources: + - id: aetna_mri_lumbar + payer: Aetna + procedure_code: MRI_LUMBAR + url: https://www.aetna.com/cpb/medical/data/100_199/0157.html + source_name: "Aetna CPB 0157" + source_type: official_policy_web + trust_level: verified + check_frequency: daily + owner: NickLeko + notes: "Monitored for drift; rules curated offline." diff --git a/rulebook/releases/2026-04-09-active-v0.5/provenance.yaml b/rulebook/releases/2026-04-09-active-v0.5/provenance.yaml new file mode 100644 index 0000000..edd6fd3 --- /dev/null +++ b/rulebook/releases/2026-04-09-active-v0.5/provenance.yaml @@ -0,0 +1,34 @@ +version: 0.2 + +sources: + Aetna: + MRI_LUMBAR: + source_type: "manual_summary" + source_name: "Aetna spine MRI policy summary" + rule_source_label: "Human-curated summary of spine MRI administrative criteria" + source_url: "https://www.aetna.com/cpb/medical/data/100_199/0157.html" + last_reviewed: "2026-04-09" + rule_last_updated: "2026-04-09" + monitored_source_id: "aetna_mri_lumbar" + notes: "Rules are illustrative for demo. Not a substitute for payer policy documents." + MRI_CERVICAL: + source_type: "manual_summary" + source_name: "Aetna cervical spine MRI policy summary" + rule_source_label: "Human-curated summary of cervical spine MRI administrative criteria" + last_reviewed: "2026-04-09" + rule_last_updated: "2026-04-09" + notes: "Rules are illustrative for demo and intentionally mirror the narrow spine MRI contract." + MRI_KNEE: + source_type: "manual_summary" + source_name: "Aetna knee MRI policy summary" + rule_source_label: "Human-curated summary of knee MRI administrative documentation criteria" + last_reviewed: "2026-04-09" + rule_last_updated: "2026-04-09" + notes: "Rules are illustrative for demo and intentionally limited to documentation completeness." + CPAP_DEVICE: + source_type: "manual_summary" + source_name: "Aetna DME policy (summary)" + rule_source_label: "Human-curated summary of CPAP administrative documentation criteria" + last_reviewed: "2026-04-09" + rule_last_updated: "2026-04-09" + notes: "Rules are illustrative for demo." diff --git a/rulebook/releases/2026-04-09-reviewed-v0.4/payer_rules.yaml b/rulebook/releases/2026-04-09-reviewed-v0.4/payer_rules.yaml new file mode 100644 index 0000000..beb5515 --- /dev/null +++ b/rulebook/releases/2026-04-09-reviewed-v0.4/payer_rules.yaml @@ -0,0 +1,99 @@ +version: 0.4 + +payers: + Aetna: + procedures: + MRI_LUMBAR: + display_name: "MRI Lumbar Spine (no contrast)" + metadata: + category: "advanced_imaging" + rule_family: "spine_mri_conservative_therapy" + summary: "Administrative readiness check for lumbar spine MRI requests using a narrow deterministic evidence contract." + supported_sites: ["outpatient"] + last_rule_update: "2026-04-09" + notes: + - "Demo rule set only; not a substitute for payer policy review." + - "Requires conservative therapy, symptom duration, imaging context, and explicit red-flag review." + required: + - key: conservative_therapy_weeks + label: "Conservative therapy duration (weeks)" + type: number + min: 6 + evidence: "PT/NSAIDs/activity modification trial documented" + - key: neuro_red_flags_documented + label: "Neuro red flags explicitly addressed (present or denied)" + type: boolean + evidence: "Weakness, saddle anesthesia, bowel/bladder changes explicitly documented as present or denied" + - key: prior_imaging_result + label: "Prior imaging result" + type: enum + allowed: ["none", "inconclusive", "abnormal"] + evidence: "X-ray/CT results or note of none" + - key: symptom_duration_weeks + label: "Symptom duration (weeks)" + type: number + min: 6 + evidence: "Duration supports guideline-based escalation" + + MRI_CERVICAL: + display_name: "MRI Cervical Spine (no contrast)" + metadata: + category: "advanced_imaging" + rule_family: "spine_mri_conservative_therapy" + summary: "Administrative readiness check for cervical spine MRI requests using the same narrow deterministic evidence contract as lumbar MRI." + supported_sites: ["outpatient"] + last_rule_update: "2026-04-09" + notes: + - "Intentionally mirrors the lumbar MRI demo pathway to show reusable deterministic architecture." + - "Human review remains required before any real submission." + required: + - key: conservative_therapy_weeks + label: "Conservative therapy duration (weeks)" + type: number + min: 6 + evidence: "PT/NSAIDs/activity modification trial documented" + - key: neuro_red_flags_documented + label: "Neuro red flags explicitly addressed (present or denied)" + type: boolean + evidence: "Weakness, numbness, bowel/bladder changes, or related escalation findings explicitly documented" + - key: prior_imaging_result + label: "Prior imaging result" + type: enum + allowed: ["none", "inconclusive", "abnormal"] + evidence: "X-ray/CT results or note of none" + - key: symptom_duration_weeks + label: "Symptom duration (weeks)" + type: number + min: 6 + evidence: "Duration supports guideline-based escalation" + + CPAP_DEVICE: + display_name: "CPAP Device (HCPCS E0601)" + metadata: + category: "durable_medical_equipment" + rule_family: "sleep_study_documentation" + summary: "Administrative readiness check for CPAP device requests using a narrow deterministic documentation contract." + supported_sites: ["outpatient"] + last_rule_update: "2026-04-09" + notes: + - "The current demo checks for OSA diagnosis, dated sleep study evidence, and AHI/RDI documentation." + - "It does not determine clinical appropriateness or device approval likelihood." + required: + - key: osa_diagnosis + label: "OSA diagnosis documented" + type: boolean + evidence: "Diagnosis present" + - key: sleep_study_date + label: "Sleep study date documented" + type: boolean + evidence: "Date in chart" + - key: ahi_documented + label: "AHI/RDI documented" + type: boolean + evidence: "AHI value included" + + +policy_notes: + - "Conservative therapy often required prior to advanced imaging." + - "Red flags warrant escalation even without long conservative management." + - "Cervical MRI is intentionally modeled with the same narrow deterministic contract as lumbar MRI in this demo." diff --git a/rulebook/releases/2026-04-09-reviewed-v0.4/policy_sources.yaml b/rulebook/releases/2026-04-09-reviewed-v0.4/policy_sources.yaml new file mode 100644 index 0000000..c1e3440 --- /dev/null +++ b/rulebook/releases/2026-04-09-reviewed-v0.4/policy_sources.yaml @@ -0,0 +1,12 @@ +version: 1.0 +sources: + - id: aetna_mri_lumbar + payer: Aetna + procedure_code: MRI_LUMBAR + url: https://www.aetna.com/cpb/medical/data/100_199/0157.html + source_name: "Aetna CPB 0157" + source_type: official_policy_web + trust_level: verified + check_frequency: daily + owner: NickLeko + notes: "Monitored for drift; rules curated offline." diff --git a/rulebook/releases/2026-04-09-reviewed-v0.4/provenance.yaml b/rulebook/releases/2026-04-09-reviewed-v0.4/provenance.yaml new file mode 100644 index 0000000..1c642e9 --- /dev/null +++ b/rulebook/releases/2026-04-09-reviewed-v0.4/provenance.yaml @@ -0,0 +1,27 @@ +version: 0.2 + +sources: + Aetna: + MRI_LUMBAR: + source_type: "manual_summary" + source_name: "Aetna spine MRI policy summary" + rule_source_label: "Human-curated summary of spine MRI administrative criteria" + source_url: "https://www.aetna.com/cpb/medical/data/100_199/0157.html" + last_reviewed: "2026-04-09" + rule_last_updated: "2026-04-09" + monitored_source_id: "aetna_mri_lumbar" + notes: "Rules are illustrative for demo. Not a substitute for payer policy documents." + MRI_CERVICAL: + source_type: "manual_summary" + source_name: "Aetna cervical spine MRI policy summary" + rule_source_label: "Human-curated summary of cervical spine MRI administrative criteria" + last_reviewed: "2026-04-09" + rule_last_updated: "2026-04-09" + notes: "Rules are illustrative for demo and intentionally mirror the narrow spine MRI contract." + CPAP_DEVICE: + source_type: "manual_summary" + source_name: "Aetna DME policy (summary)" + rule_source_label: "Human-curated summary of CPAP administrative documentation criteria" + last_reviewed: "2026-04-09" + rule_last_updated: "2026-04-09" + notes: "Rules are illustrative for demo." diff --git a/rules/payer_rules.yaml b/rules/payer_rules.yaml index 394df26..0f8fdf8 100644 --- a/rules/payer_rules.yaml +++ b/rules/payer_rules.yaml @@ -1,10 +1,19 @@ -version: 0.2 +version: 0.5 payers: Aetna: procedures: MRI_LUMBAR: display_name: "MRI Lumbar Spine (no contrast)" + metadata: + category: "advanced_imaging" + rule_family: "spine_mri_conservative_therapy" + summary: "Administrative readiness check for lumbar spine MRI requests using a narrow deterministic evidence contract." + supported_sites: ["outpatient"] + last_rule_update: "2026-04-09" + notes: + - "Demo rule set only; not a substitute for payer policy review." + - "Requires conservative therapy, symptom duration, imaging context, and explicit red-flag review." required: - key: conservative_therapy_weeks label: "Conservative therapy duration (weeks)" @@ -26,8 +35,81 @@ payers: min: 6 evidence: "Duration supports guideline-based escalation" + MRI_CERVICAL: + display_name: "MRI Cervical Spine (no contrast)" + metadata: + category: "advanced_imaging" + rule_family: "spine_mri_conservative_therapy" + summary: "Administrative readiness check for cervical spine MRI requests using the same narrow deterministic evidence contract as lumbar MRI." + supported_sites: ["outpatient"] + last_rule_update: "2026-04-09" + notes: + - "Intentionally mirrors the lumbar MRI demo pathway to show reusable deterministic architecture." + - "Human review remains required before any real submission." + required: + - key: conservative_therapy_weeks + label: "Conservative therapy duration (weeks)" + type: number + min: 6 + evidence: "PT/NSAIDs/activity modification trial documented" + - key: neuro_red_flags_documented + label: "Neuro red flags explicitly addressed (present or denied)" + type: boolean + evidence: "Weakness, numbness, bowel/bladder changes, or related escalation findings explicitly documented" + - key: prior_imaging_result + label: "Prior imaging result" + type: enum + allowed: ["none", "inconclusive", "abnormal"] + evidence: "X-ray/CT results or note of none" + - key: symptom_duration_weeks + label: "Symptom duration (weeks)" + type: number + min: 6 + evidence: "Duration supports guideline-based escalation" + + MRI_KNEE: + display_name: "MRI Knee (no contrast)" + metadata: + category: "advanced_imaging" + rule_family: "extremity_mri_conservative_therapy" + summary: "Administrative readiness check for knee MRI requests using a narrow deterministic documentation contract." + supported_sites: ["outpatient"] + last_rule_update: "2026-04-09" + notes: + - "This demo pathway checks only administrative completeness, not orthopedic appropriateness." + - "Requires symptom duration, conservative therapy, prior imaging context, and explicit review of mechanical symptoms." + required: + - key: conservative_therapy_weeks + label: "Conservative therapy duration (weeks)" + type: number + min: 6 + evidence: "PT/activity modification/NSAID trial documented" + - key: symptom_duration_weeks + label: "Symptom duration (weeks)" + type: number + min: 6 + evidence: "Persistent symptoms documented long enough to justify escalation" + - key: prior_imaging_result + label: "Prior imaging result" + type: enum + allowed: ["inconclusive", "abnormal"] + evidence: "Prior knee imaging such as x-ray is documented and not simply absent" + - key: mechanical_symptoms_documented + label: "Mechanical symptoms explicitly addressed (present or denied)" + type: boolean + evidence: "Locking, catching, buckling, giving way, or instability explicitly documented" + CPAP_DEVICE: display_name: "CPAP Device (HCPCS E0601)" + metadata: + category: "durable_medical_equipment" + rule_family: "sleep_study_documentation" + summary: "Administrative readiness check for CPAP device requests using a narrow deterministic documentation contract." + supported_sites: ["outpatient"] + last_rule_update: "2026-04-09" + notes: + - "The current demo checks for OSA diagnosis, dated sleep study evidence, and AHI/RDI documentation." + - "It does not determine clinical appropriateness or device approval likelihood." required: - key: osa_diagnosis label: "OSA diagnosis documented" @@ -46,4 +128,5 @@ payers: policy_notes: - "Conservative therapy often required prior to advanced imaging." - "Red flags warrant escalation even without long conservative management." - + - "Cervical MRI is intentionally modeled with the same narrow deterministic contract as lumbar MRI in this demo." + - "Knee MRI adds a narrow orthopedic documentation contract without introducing medical-necessity scoring." diff --git a/rules/policy_sources.yaml b/rules/policy_sources.yaml index 11c4441..c1e3440 100644 --- a/rules/policy_sources.yaml +++ b/rules/policy_sources.yaml @@ -4,6 +4,7 @@ sources: payer: Aetna procedure_code: MRI_LUMBAR url: https://www.aetna.com/cpb/medical/data/100_199/0157.html + source_name: "Aetna CPB 0157" source_type: official_policy_web trust_level: verified check_frequency: daily diff --git a/rules/provenance.yaml b/rules/provenance.yaml index bd706c4..edd6fd3 100644 --- a/rules/provenance.yaml +++ b/rules/provenance.yaml @@ -1,14 +1,34 @@ -version: 0.1 +version: 0.2 sources: Aetna: MRI_LUMBAR: source_type: "manual_summary" - source_name: "Aetna policy (summary)" - last_reviewed: "2026-02-05" + source_name: "Aetna spine MRI policy summary" + rule_source_label: "Human-curated summary of spine MRI administrative criteria" + source_url: "https://www.aetna.com/cpb/medical/data/100_199/0157.html" + last_reviewed: "2026-04-09" + rule_last_updated: "2026-04-09" + monitored_source_id: "aetna_mri_lumbar" notes: "Rules are illustrative for demo. Not a substitute for payer policy documents." + MRI_CERVICAL: + source_type: "manual_summary" + source_name: "Aetna cervical spine MRI policy summary" + rule_source_label: "Human-curated summary of cervical spine MRI administrative criteria" + last_reviewed: "2026-04-09" + rule_last_updated: "2026-04-09" + notes: "Rules are illustrative for demo and intentionally mirror the narrow spine MRI contract." + MRI_KNEE: + source_type: "manual_summary" + source_name: "Aetna knee MRI policy summary" + rule_source_label: "Human-curated summary of knee MRI administrative documentation criteria" + last_reviewed: "2026-04-09" + rule_last_updated: "2026-04-09" + notes: "Rules are illustrative for demo and intentionally limited to documentation completeness." CPAP_DEVICE: source_type: "manual_summary" source_name: "Aetna DME policy (summary)" - last_reviewed: "2026-02-05" + rule_source_label: "Human-curated summary of CPAP administrative documentation criteria" + last_reviewed: "2026-04-09" + rule_last_updated: "2026-04-09" notes: "Rules are illustrative for demo." diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..d1eefc4 --- /dev/null +++ b/scripts/__init__.py @@ -0,0 +1 @@ +# Package marker for `python -m scripts.generate_artifacts`. diff --git a/scripts/generate_artifacts.py b/scripts/generate_artifacts.py new file mode 100644 index 0000000..a725448 --- /dev/null +++ b/scripts/generate_artifacts.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +from engine.rendering import ( + export_evaluation_payload, + render_drift_markdown, + render_rulebook_diff_markdown, + write_json_artifact, +) +from engine.service import ReadinessService + +DEFAULT_CASE_IDS = [ + "MRI-01-complete", + "MRI-08-edge-below-threshold", + "MRI-CERV-01-ready", + "MRI-KNEE-01-ready", + "CPAP-02-borderline", +] + + +def main() -> int: + service = ReadinessService() + artifact_dir = service.config.docs_artifacts_dir + artifact_dir.mkdir(parents=True, exist_ok=True) + drift_report = service.get_drift_status() + supported_procedures = service.list_supported_procedures() + demo_cases = service.list_demo_case_summaries() + rulebook_status = service.get_rulebook_status() + default_rulebook_diff = service.get_rulebook_diff("2026-04-09-reviewed-v0.4", "2026-04-09-active-v0.5") + + for case_id in DEFAULT_CASE_IDS: + request = service.get_demo_case_request(case_id) + evaluation = service.evaluate(request) + letter_text, letter_meta = service.generate_letter(evaluation) + write_json_artifact( + export_evaluation_payload(evaluation, letter_text=letter_text, letter_meta=letter_meta), + artifact_dir / f"{case_id}.json", + ) + + write_json_artifact( + drift_report.model_dump(mode="json"), + artifact_dir / "drift_status.json", + ) + (artifact_dir / "drift_report.md").write_text(render_drift_markdown(drift_report), encoding="utf-8") + write_json_artifact( + [item.model_dump(mode="json") for item in supported_procedures], + artifact_dir / "supported_procedures.json", + ) + write_json_artifact( + [item.model_dump(mode="json") for item in demo_cases], + artifact_dir / "demo_cases.json", + ) + write_json_artifact( + [item.model_dump(mode="json") for item in demo_cases if item.showcase.get("featured")], + artifact_dir / "featured_demo_cases.json", + ) + write_json_artifact( + service.get_status().model_dump(mode="json"), + artifact_dir / "status.json", + ) + write_json_artifact( + rulebook_status.model_dump(mode="json"), + artifact_dir / "rulebook_status.json", + ) + write_json_artifact( + default_rulebook_diff.model_dump(mode="json"), + artifact_dir / "rulebook_diff_reviewed_vs_active.json", + ) + (artifact_dir / "rulebook_diff_reviewed_vs_active.md").write_text( + render_rulebook_diff_markdown(default_rulebook_diff), + encoding="utf-8", + ) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/generate_golden_outputs.py b/scripts/generate_golden_outputs.py new file mode 100644 index 0000000..ba62eab --- /dev/null +++ b/scripts/generate_golden_outputs.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from pathlib import Path + +from engine.acceptance import ( + DEFAULT_ACCEPTANCE_CASE_IDS, + build_acceptance_evaluation_payload, + build_acceptance_governance_payloads, +) +from engine.rendering import write_json_artifact +from engine.service import ReadinessService + +GOLDEN_ROOT = Path("test/golden") + + +def main() -> int: + service = ReadinessService() + evaluation_dir = GOLDEN_ROOT / "evaluations" + governance_dir = GOLDEN_ROOT / "governance" + + for case_id in DEFAULT_ACCEPTANCE_CASE_IDS: + write_json_artifact( + build_acceptance_evaluation_payload(service, case_id), + evaluation_dir / f"{case_id}.json", + ) + + for name, payload in build_acceptance_governance_payloads(service).items(): + write_json_artifact(payload, governance_dir / f"{name}.json") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 0000000..2659de5 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,8 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) diff --git a/test/golden/evaluations/CPAP-02-borderline.json b/test/golden/evaluations/CPAP-02-borderline.json new file mode 100644 index 0000000..5689ff0 --- /dev/null +++ b/test/golden/evaluations/CPAP-02-borderline.json @@ -0,0 +1,431 @@ +{ + "audit_trail": { + "blocking_issues": { + "not_documented": [ + { + "key": "sleep_study_date", + "label": "Sleep study date documented", + "reason": "Not found in note. Add explicit statement.", + "status": "NOT_DOCUMENTED" + }, + { + "key": "ahi_documented", + "label": "AHI/RDI documented", + "reason": "Not found in note. Add explicit statement.", + "status": "NOT_DOCUMENTED" + } + ], + "not_met": [] + }, + "evaluation_warnings": [ + "Policy trust remains DEMO for this procedure. Verify against official policy before real-world use." + ], + "evidence_map": { + "ahi_documented": [ + { + "end": 88, + "start": 70, + "text": "AHI not documented" + } + ], + "osa_diagnosis": [ + { + "end": 3, + "start": 0, + "text": "OSA" + } + ] + }, + "facts_extracted": { + "ahi_documented": null, + "conservative_therapy_weeks": null, + "mechanical_symptoms_documented": null, + "neuro_red_flags_documented": null, + "osa_diagnosis": true, + "prior_imaging_result": null, + "sleep_study_date": null, + "symptom_duration_weeks": null + }, + "invariant_errors": [], + "metrics": { + "compliance_rate": 100.0, + "compliant_count": 1, + "extraction_failure_count": 2, + "extraction_success_rate": 33.3, + "non_compliant_count": 0 + }, + "note_hash": "5ef99d034f3c708e", + "note_length": 89, + "overall_status": "CANNOT_DETERMINE", + "payer": "Aetna", + "policy_trust_level": "demo", + "procedure_code": "CPAP_DEVICE", + "procedure_name": "CPAP Device (HCPCS E0601)", + "provenance_snapshot": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": null, + "monitored_source_id": null, + "monitored_source_name": null, + "monitored_source_owner": null, + "monitored_source_url": null, + "notes": "Rules are illustrative for demo.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of CPAP administrative documentation criteria", + "source_name": "Aetna DME policy (summary)", + "source_type": "manual_summary", + "source_url": null + }, + "requirements_checked": [ + "osa_diagnosis", + "sleep_study_date", + "ahi_documented" + ], + "rulebook_active_release_id": "2026-04-09-active-v0.5", + "rules_version": "0.5", + "run_id": "__RUN_ID__", + "site_of_care": "outpatient", + "specialty": "Primary Care", + "submission_readiness": false, + "timestamp_utc": "__TIMESTAMP_UTC__" + }, + "blockers": { + "not_documented": [ + { + "key": "sleep_study_date", + "label": "Sleep study date documented", + "reason": "Not found in note. Add explicit statement.", + "status": "NOT_DOCUMENTED" + }, + { + "key": "ahi_documented", + "label": "AHI/RDI documented", + "reason": "Not found in note. Add explicit statement.", + "status": "NOT_DOCUMENTED" + } + ], + "not_met": [] + }, + "evidence_map": { + "ahi_documented": [ + { + "end": 88, + "start": 70, + "text": "AHI not documented" + } + ], + "osa_diagnosis": [ + { + "end": 3, + "start": 0, + "text": "OSA" + } + ] + }, + "facts": { + "ahi_documented": null, + "conservative_therapy_weeks": null, + "mechanical_symptoms_documented": null, + "neuro_red_flags_documented": null, + "osa_diagnosis": true, + "prior_imaging_result": null, + "sleep_study_date": null, + "symptom_duration_weeks": null + }, + "metrics": { + "compliance_rate": 100.0, + "compliant_count": 1, + "extraction_failure_count": 2, + "extraction_success_rate": 33.3, + "non_compliant_count": 0 + }, + "overall_status": "CANNOT_DETERMINE", + "policy_trust_level": "demo", + "provenance": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": null, + "monitored_source_id": null, + "monitored_source_name": null, + "monitored_source_owner": null, + "monitored_source_url": null, + "notes": "Rules are illustrative for demo.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of CPAP administrative documentation criteria", + "source_name": "Aetna DME policy (summary)", + "source_type": "manual_summary", + "source_url": null + }, + "readiness_score": 33, + "report": { + "audit_trail": { + "blocking_issues": { + "not_documented": [ + { + "key": "sleep_study_date", + "label": "Sleep study date documented", + "reason": "Not found in note. Add explicit statement.", + "status": "NOT_DOCUMENTED" + }, + { + "key": "ahi_documented", + "label": "AHI/RDI documented", + "reason": "Not found in note. Add explicit statement.", + "status": "NOT_DOCUMENTED" + } + ], + "not_met": [] + }, + "evaluation_warnings": [ + "Policy trust remains DEMO for this procedure. Verify against official policy before real-world use." + ], + "evidence_map": { + "ahi_documented": [ + { + "end": 88, + "start": 70, + "text": "AHI not documented" + } + ], + "osa_diagnosis": [ + { + "end": 3, + "start": 0, + "text": "OSA" + } + ] + }, + "facts_extracted": { + "ahi_documented": null, + "conservative_therapy_weeks": null, + "mechanical_symptoms_documented": null, + "neuro_red_flags_documented": null, + "osa_diagnosis": true, + "prior_imaging_result": null, + "sleep_study_date": null, + "symptom_duration_weeks": null + }, + "invariant_errors": [], + "metrics": { + "compliance_rate": 100.0, + "compliant_count": 1, + "extraction_failure_count": 2, + "extraction_success_rate": 33.3, + "non_compliant_count": 0 + }, + "note_hash": "5ef99d034f3c708e", + "note_length": 89, + "overall_status": "CANNOT_DETERMINE", + "payer": "Aetna", + "policy_trust_level": "demo", + "procedure_code": "CPAP_DEVICE", + "procedure_name": "CPAP Device (HCPCS E0601)", + "provenance_snapshot": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": null, + "monitored_source_id": null, + "monitored_source_name": null, + "monitored_source_owner": null, + "monitored_source_url": null, + "notes": "Rules are illustrative for demo.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of CPAP administrative documentation criteria", + "source_name": "Aetna DME policy (summary)", + "source_type": "manual_summary", + "source_url": null + }, + "requirements_checked": [ + "osa_diagnosis", + "sleep_study_date", + "ahi_documented" + ], + "rulebook_active_release_id": "2026-04-09-active-v0.5", + "rules_version": "0.5", + "run_id": "__RUN_ID__", + "site_of_care": "outpatient", + "specialty": "Primary Care", + "submission_readiness": false, + "timestamp_utc": "__TIMESTAMP_UTC__" + }, + "letter_draft": "", + "met_count": 1, + "not_documented_count": 2, + "not_met_count": 0, + "readiness_score": 33, + "results": [ + { + "evidence": "Diagnosis present", + "evidence_snippets": [ + "OSA" + ], + "evidence_spans": [ + { + "end": 3, + "start": 0, + "text": "OSA" + } + ], + "key": "osa_diagnosis", + "label": "OSA diagnosis documented", + "reason": "Explicitly addressed in documentation (present/affirmed).", + "status": "MET" + }, + { + "evidence": "Date in chart", + "evidence_snippets": [], + "evidence_spans": [], + "key": "sleep_study_date", + "label": "Sleep study date documented", + "reason": "Not found in note. Add explicit statement.", + "status": "NOT_DOCUMENTED" + }, + { + "evidence": "AHI value included", + "evidence_snippets": [ + "AHI not documented" + ], + "evidence_spans": [ + { + "end": 88, + "start": 70, + "text": "AHI not documented" + } + ], + "key": "ahi_documented", + "label": "AHI/RDI documented", + "reason": "Not found in note. Add explicit statement.", + "status": "NOT_DOCUMENTED" + } + ], + "rule_reasons": [ + "Sleep study date documented: NOT_DOCUMENTED \u2014 Not found in note. Add explicit statement.", + "AHI/RDI documented: NOT_DOCUMENTED \u2014 Not found in note. Add explicit statement." + ] + }, + "request": { + "dx_codes": [ + "G47.33" + ], + "note_text": "OSA noted in problem list. Sleep study in 2023 mentioned but no date. AHI not documented.", + "payer": "Aetna", + "procedure_code": "CPAP_DEVICE", + "site_of_care": "outpatient", + "specialty": "Primary Care" + }, + "results": [ + { + "evidence": "Diagnosis present", + "evidence_snippets": [ + "OSA" + ], + "evidence_spans": [ + { + "end": 3, + "start": 0, + "text": "OSA" + } + ], + "key": "osa_diagnosis", + "label": "OSA diagnosis documented", + "reason": "Explicitly addressed in documentation (present/affirmed).", + "status": "MET" + }, + { + "evidence": "Date in chart", + "evidence_snippets": [], + "evidence_spans": [], + "key": "sleep_study_date", + "label": "Sleep study date documented", + "reason": "Not found in note. Add explicit statement.", + "status": "NOT_DOCUMENTED" + }, + { + "evidence": "AHI value included", + "evidence_snippets": [ + "AHI not documented" + ], + "evidence_spans": [ + { + "end": 88, + "start": 70, + "text": "AHI not documented" + } + ], + "key": "ahi_documented", + "label": "AHI/RDI documented", + "reason": "Not found in note. Add explicit statement.", + "status": "NOT_DOCUMENTED" + } + ], + "rule_reasons": [ + "Sleep study date documented: NOT_DOCUMENTED \u2014 Not found in note. Add explicit statement.", + "AHI/RDI documented: NOT_DOCUMENTED \u2014 Not found in note. Add explicit statement." + ], + "submission_readiness": false, + "supported_procedure": { + "display_name": "CPAP Device (HCPCS E0601)", + "metadata": { + "category": "durable_medical_equipment", + "last_rule_update": "2026-04-09", + "notes": [ + "The current demo checks for OSA diagnosis, dated sleep study evidence, and AHI/RDI documentation.", + "It does not determine clinical appropriateness or device approval likelihood." + ], + "rule_family": "sleep_study_documentation", + "summary": "Administrative readiness check for CPAP device requests using a narrow deterministic documentation contract.", + "supported_sites": [ + "outpatient" + ] + }, + "monitored_for_drift": false, + "payer": "Aetna", + "policy_trust_level": "demo", + "procedure_code": "CPAP_DEVICE", + "provenance": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": null, + "monitored_source_id": null, + "monitored_source_name": null, + "monitored_source_owner": null, + "monitored_source_url": null, + "notes": "Rules are illustrative for demo.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of CPAP administrative documentation criteria", + "source_name": "Aetna DME policy (summary)", + "source_type": "manual_summary", + "source_url": null + }, + "required_field_keys": [ + "osa_diagnosis", + "sleep_study_date", + "ahi_documented" + ], + "requirements": [ + { + "allowed": [], + "evidence": "Diagnosis present", + "key": "osa_diagnosis", + "label": "OSA diagnosis documented", + "min": null, + "type": "boolean" + }, + { + "allowed": [], + "evidence": "Date in chart", + "key": "sleep_study_date", + "label": "Sleep study date documented", + "min": null, + "type": "boolean" + }, + { + "allowed": [], + "evidence": "AHI value included", + "key": "ahi_documented", + "label": "AHI/RDI documented", + "min": null, + "type": "boolean" + } + ] + }, + "warnings": [ + "Policy trust remains DEMO for this procedure. Verify against official policy before real-world use." + ] +} diff --git a/test/golden/evaluations/MRI-01-complete.json b/test/golden/evaluations/MRI-01-complete.json new file mode 100644 index 0000000..e3b25e2 --- /dev/null +++ b/test/golden/evaluations/MRI-01-complete.json @@ -0,0 +1,493 @@ +{ + "audit_trail": { + "blocking_issues": { + "not_documented": [], + "not_met": [] + }, + "evaluation_warnings": [ + "Policy trust remains DEMO for this procedure. Verify against official policy before real-world use." + ], + "evidence_map": { + "conservative_therapy_weeks": [ + { + "end": 78, + "start": 64, + "text": "PT for 8 weeks" + } + ], + "neuro_red_flags_documented": [ + { + "end": 239, + "start": 116, + "text": "Denies bowel/bladder incontinence. No saddle anesthesia. Prior imaging: lumbar xray inconclusive. Neuro exam: mild weakness" + } + ], + "prior_imaging_result": [ + { + "end": 199, + "start": 195, + "text": "xray" + } + ], + "symptom_duration_weeks": [ + { + "end": 52, + "start": 45, + "text": "8 weeks" + } + ] + }, + "facts_extracted": { + "ahi_documented": null, + "conservative_therapy_weeks": 8, + "mechanical_symptoms_documented": null, + "neuro_red_flags_documented": true, + "osa_diagnosis": null, + "prior_imaging_result": "inconclusive", + "sleep_study_date": null, + "symptom_duration_weeks": 8 + }, + "invariant_errors": [], + "metrics": { + "compliance_rate": 100.0, + "compliant_count": 4, + "extraction_failure_count": 0, + "extraction_success_rate": 100.0, + "non_compliant_count": 0 + }, + "note_hash": "82a37cb51ea58464", + "note_length": 257, + "overall_status": "READY", + "payer": "Aetna", + "policy_trust_level": "demo", + "procedure_code": "MRI_LUMBAR", + "procedure_name": "MRI Lumbar Spine (no contrast)", + "provenance_snapshot": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": "daily", + "monitored_source_id": "aetna_mri_lumbar", + "monitored_source_name": "Aetna CPB 0157", + "monitored_source_owner": "NickLeko", + "monitored_source_url": "https://www.aetna.com/cpb/medical/data/100_199/0157.html", + "notes": "Rules are illustrative for demo. Not a substitute for payer policy documents.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of spine MRI administrative criteria", + "source_name": "Aetna spine MRI policy summary", + "source_type": "manual_summary", + "source_url": "https://www.aetna.com/cpb/medical/data/100_199/0157.html" + }, + "requirements_checked": [ + "conservative_therapy_weeks", + "neuro_red_flags_documented", + "prior_imaging_result", + "symptom_duration_weeks" + ], + "rulebook_active_release_id": "2026-04-09-active-v0.5", + "rules_version": "0.5", + "run_id": "__RUN_ID__", + "site_of_care": "outpatient", + "specialty": "Orthopedics", + "submission_readiness": true, + "timestamp_utc": "__TIMESTAMP_UTC__" + }, + "blockers": { + "not_documented": [], + "not_met": [] + }, + "evidence_map": { + "conservative_therapy_weeks": [ + { + "end": 78, + "start": 64, + "text": "PT for 8 weeks" + } + ], + "neuro_red_flags_documented": [ + { + "end": 239, + "start": 116, + "text": "Denies bowel/bladder incontinence. No saddle anesthesia. Prior imaging: lumbar xray inconclusive. Neuro exam: mild weakness" + } + ], + "prior_imaging_result": [ + { + "end": 199, + "start": 195, + "text": "xray" + } + ], + "symptom_duration_weeks": [ + { + "end": 52, + "start": 45, + "text": "8 weeks" + } + ] + }, + "facts": { + "ahi_documented": null, + "conservative_therapy_weeks": 8, + "mechanical_symptoms_documented": null, + "neuro_red_flags_documented": true, + "osa_diagnosis": null, + "prior_imaging_result": "inconclusive", + "sleep_study_date": null, + "symptom_duration_weeks": 8 + }, + "metrics": { + "compliance_rate": 100.0, + "compliant_count": 4, + "extraction_failure_count": 0, + "extraction_success_rate": 100.0, + "non_compliant_count": 0 + }, + "overall_status": "READY", + "policy_trust_level": "demo", + "provenance": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": "daily", + "monitored_source_id": "aetna_mri_lumbar", + "monitored_source_name": "Aetna CPB 0157", + "monitored_source_owner": "NickLeko", + "monitored_source_url": "https://www.aetna.com/cpb/medical/data/100_199/0157.html", + "notes": "Rules are illustrative for demo. Not a substitute for payer policy documents.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of spine MRI administrative criteria", + "source_name": "Aetna spine MRI policy summary", + "source_type": "manual_summary", + "source_url": "https://www.aetna.com/cpb/medical/data/100_199/0157.html" + }, + "readiness_score": 100, + "report": { + "audit_trail": { + "blocking_issues": { + "not_documented": [], + "not_met": [] + }, + "evaluation_warnings": [ + "Policy trust remains DEMO for this procedure. Verify against official policy before real-world use." + ], + "evidence_map": { + "conservative_therapy_weeks": [ + { + "end": 78, + "start": 64, + "text": "PT for 8 weeks" + } + ], + "neuro_red_flags_documented": [ + { + "end": 239, + "start": 116, + "text": "Denies bowel/bladder incontinence. No saddle anesthesia. Prior imaging: lumbar xray inconclusive. Neuro exam: mild weakness" + } + ], + "prior_imaging_result": [ + { + "end": 199, + "start": 195, + "text": "xray" + } + ], + "symptom_duration_weeks": [ + { + "end": 52, + "start": 45, + "text": "8 weeks" + } + ] + }, + "facts_extracted": { + "ahi_documented": null, + "conservative_therapy_weeks": 8, + "mechanical_symptoms_documented": null, + "neuro_red_flags_documented": true, + "osa_diagnosis": null, + "prior_imaging_result": "inconclusive", + "sleep_study_date": null, + "symptom_duration_weeks": 8 + }, + "invariant_errors": [], + "metrics": { + "compliance_rate": 100.0, + "compliant_count": 4, + "extraction_failure_count": 0, + "extraction_success_rate": 100.0, + "non_compliant_count": 0 + }, + "note_hash": "82a37cb51ea58464", + "note_length": 257, + "overall_status": "READY", + "payer": "Aetna", + "policy_trust_level": "demo", + "procedure_code": "MRI_LUMBAR", + "procedure_name": "MRI Lumbar Spine (no contrast)", + "provenance_snapshot": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": "daily", + "monitored_source_id": "aetna_mri_lumbar", + "monitored_source_name": "Aetna CPB 0157", + "monitored_source_owner": "NickLeko", + "monitored_source_url": "https://www.aetna.com/cpb/medical/data/100_199/0157.html", + "notes": "Rules are illustrative for demo. Not a substitute for payer policy documents.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of spine MRI administrative criteria", + "source_name": "Aetna spine MRI policy summary", + "source_type": "manual_summary", + "source_url": "https://www.aetna.com/cpb/medical/data/100_199/0157.html" + }, + "requirements_checked": [ + "conservative_therapy_weeks", + "neuro_red_flags_documented", + "prior_imaging_result", + "symptom_duration_weeks" + ], + "rulebook_active_release_id": "2026-04-09-active-v0.5", + "rules_version": "0.5", + "run_id": "__RUN_ID__", + "site_of_care": "outpatient", + "specialty": "Orthopedics", + "submission_readiness": true, + "timestamp_utc": "__TIMESTAMP_UTC__" + }, + "letter_draft": "", + "met_count": 4, + "not_documented_count": 0, + "not_met_count": 0, + "readiness_score": 100, + "results": [ + { + "evidence": "PT/NSAIDs/activity modification trial documented", + "evidence_snippets": [ + "PT for 8 weeks" + ], + "evidence_spans": [ + { + "end": 78, + "start": 64, + "text": "PT for 8 weeks" + } + ], + "key": "conservative_therapy_weeks", + "label": "Conservative therapy duration (weeks)", + "reason": "Documented value: 8.", + "status": "MET" + }, + { + "evidence": "Weakness, saddle anesthesia, bowel/bladder changes explicitly documented as present or denied", + "evidence_snippets": [ + "Denies bowel/bladder incontinence. No saddle anesthesia. Prior imaging: lumbar xray inconclusive. Neuro exam: mild weakness" + ], + "evidence_spans": [ + { + "end": 239, + "start": 116, + "text": "Denies bowel/bladder incontinence. No saddle anesthesia. Prior imaging: lumbar xray inconclusive. Neuro exam: mild weakness" + } + ], + "key": "neuro_red_flags_documented", + "label": "Neuro red flags explicitly addressed (present or denied)", + "reason": "Explicitly addressed in documentation (present/affirmed).", + "status": "MET" + }, + { + "evidence": "X-ray/CT results or note of none", + "evidence_snippets": [ + "xray" + ], + "evidence_spans": [ + { + "end": 199, + "start": 195, + "text": "xray" + } + ], + "key": "prior_imaging_result", + "label": "Prior imaging result", + "reason": "Documented: inconclusive.", + "status": "MET" + }, + { + "evidence": "Duration supports guideline-based escalation", + "evidence_snippets": [ + "8 weeks" + ], + "evidence_spans": [ + { + "end": 52, + "start": 45, + "text": "8 weeks" + } + ], + "key": "symptom_duration_weeks", + "label": "Symptom duration (weeks)", + "reason": "Documented value: 8.", + "status": "MET" + } + ], + "rule_reasons": [] + }, + "request": { + "dx_codes": [ + "M54.16" + ], + "note_text": "Low back pain with right leg radiculopathy x 8 weeks. Completed PT for 8 weeks and NSAIDs with minimal improvement. Denies bowel/bladder incontinence. No saddle anesthesia. Prior imaging: lumbar xray inconclusive. Neuro exam: mild weakness dorsiflexion 4/5.", + "payer": "Aetna", + "procedure_code": "MRI_LUMBAR", + "site_of_care": "outpatient", + "specialty": "Orthopedics" + }, + "results": [ + { + "evidence": "PT/NSAIDs/activity modification trial documented", + "evidence_snippets": [ + "PT for 8 weeks" + ], + "evidence_spans": [ + { + "end": 78, + "start": 64, + "text": "PT for 8 weeks" + } + ], + "key": "conservative_therapy_weeks", + "label": "Conservative therapy duration (weeks)", + "reason": "Documented value: 8.", + "status": "MET" + }, + { + "evidence": "Weakness, saddle anesthesia, bowel/bladder changes explicitly documented as present or denied", + "evidence_snippets": [ + "Denies bowel/bladder incontinence. No saddle anesthesia. Prior imaging: lumbar xray inconclusive. Neuro exam: mild weakness" + ], + "evidence_spans": [ + { + "end": 239, + "start": 116, + "text": "Denies bowel/bladder incontinence. No saddle anesthesia. Prior imaging: lumbar xray inconclusive. Neuro exam: mild weakness" + } + ], + "key": "neuro_red_flags_documented", + "label": "Neuro red flags explicitly addressed (present or denied)", + "reason": "Explicitly addressed in documentation (present/affirmed).", + "status": "MET" + }, + { + "evidence": "X-ray/CT results or note of none", + "evidence_snippets": [ + "xray" + ], + "evidence_spans": [ + { + "end": 199, + "start": 195, + "text": "xray" + } + ], + "key": "prior_imaging_result", + "label": "Prior imaging result", + "reason": "Documented: inconclusive.", + "status": "MET" + }, + { + "evidence": "Duration supports guideline-based escalation", + "evidence_snippets": [ + "8 weeks" + ], + "evidence_spans": [ + { + "end": 52, + "start": 45, + "text": "8 weeks" + } + ], + "key": "symptom_duration_weeks", + "label": "Symptom duration (weeks)", + "reason": "Documented value: 8.", + "status": "MET" + } + ], + "rule_reasons": [], + "submission_readiness": true, + "supported_procedure": { + "display_name": "MRI Lumbar Spine (no contrast)", + "metadata": { + "category": "advanced_imaging", + "last_rule_update": "2026-04-09", + "notes": [ + "Demo rule set only; not a substitute for payer policy review.", + "Requires conservative therapy, symptom duration, imaging context, and explicit red-flag review." + ], + "rule_family": "spine_mri_conservative_therapy", + "summary": "Administrative readiness check for lumbar spine MRI requests using a narrow deterministic evidence contract.", + "supported_sites": [ + "outpatient" + ] + }, + "monitored_for_drift": true, + "payer": "Aetna", + "policy_trust_level": "demo", + "procedure_code": "MRI_LUMBAR", + "provenance": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": "daily", + "monitored_source_id": "aetna_mri_lumbar", + "monitored_source_name": "Aetna CPB 0157", + "monitored_source_owner": "NickLeko", + "monitored_source_url": "https://www.aetna.com/cpb/medical/data/100_199/0157.html", + "notes": "Rules are illustrative for demo. Not a substitute for payer policy documents.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of spine MRI administrative criteria", + "source_name": "Aetna spine MRI policy summary", + "source_type": "manual_summary", + "source_url": "https://www.aetna.com/cpb/medical/data/100_199/0157.html" + }, + "required_field_keys": [ + "conservative_therapy_weeks", + "neuro_red_flags_documented", + "prior_imaging_result", + "symptom_duration_weeks" + ], + "requirements": [ + { + "allowed": [], + "evidence": "PT/NSAIDs/activity modification trial documented", + "key": "conservative_therapy_weeks", + "label": "Conservative therapy duration (weeks)", + "min": 6.0, + "type": "number" + }, + { + "allowed": [], + "evidence": "Weakness, saddle anesthesia, bowel/bladder changes explicitly documented as present or denied", + "key": "neuro_red_flags_documented", + "label": "Neuro red flags explicitly addressed (present or denied)", + "min": null, + "type": "boolean" + }, + { + "allowed": [ + "none", + "inconclusive", + "abnormal" + ], + "evidence": "X-ray/CT results or note of none", + "key": "prior_imaging_result", + "label": "Prior imaging result", + "min": null, + "type": "enum" + }, + { + "allowed": [], + "evidence": "Duration supports guideline-based escalation", + "key": "symptom_duration_weeks", + "label": "Symptom duration (weeks)", + "min": 6.0, + "type": "number" + } + ] + }, + "warnings": [ + "Policy trust remains DEMO for this procedure. Verify against official policy before real-world use." + ] +} diff --git a/test/golden/evaluations/MRI-08-edge-below-threshold.json b/test/golden/evaluations/MRI-08-edge-below-threshold.json new file mode 100644 index 0000000..c3e6d75 --- /dev/null +++ b/test/golden/evaluations/MRI-08-edge-below-threshold.json @@ -0,0 +1,538 @@ +{ + "audit_trail": { + "blocking_issues": { + "not_documented": [], + "not_met": [ + { + "key": "conservative_therapy_weeks", + "label": "Conservative therapy duration (weeks)", + "reason": "Documented value (5) below requirement (>= 6.0). Clarify or justify.", + "status": "NOT_MET" + }, + { + "key": "symptom_duration_weeks", + "label": "Symptom duration (weeks)", + "reason": "Documented value (5) below requirement (>= 6.0). Clarify or justify.", + "status": "NOT_MET" + } + ] + }, + "evaluation_warnings": [ + "Policy trust remains DEMO for this procedure. Verify against official policy before real-world use." + ], + "evidence_map": { + "conservative_therapy_weeks": [ + { + "end": 37, + "start": 25, + "text": "PT x 5 weeks" + } + ], + "neuro_red_flags_documented": [ + { + "end": 87, + "start": 50, + "text": "Denies weakness. Denies bowel/bladder" + } + ], + "prior_imaging_result": [ + { + "end": 113, + "start": 97, + "text": "No prior imaging" + } + ], + "symptom_duration_weeks": [ + { + "end": 23, + "start": 16, + "text": "5 weeks" + } + ] + }, + "facts_extracted": { + "ahi_documented": null, + "conservative_therapy_weeks": 5, + "mechanical_symptoms_documented": null, + "neuro_red_flags_documented": true, + "osa_diagnosis": null, + "prior_imaging_result": "none", + "sleep_study_date": null, + "symptom_duration_weeks": 5 + }, + "invariant_errors": [], + "metrics": { + "compliance_rate": 50.0, + "compliant_count": 2, + "extraction_failure_count": 0, + "extraction_success_rate": 100.0, + "non_compliant_count": 2 + }, + "note_hash": "94ee8ba7171f7dea", + "note_length": 114, + "overall_status": "NOT_READY", + "payer": "Aetna", + "policy_trust_level": "demo", + "procedure_code": "MRI_LUMBAR", + "procedure_name": "MRI Lumbar Spine (no contrast)", + "provenance_snapshot": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": "daily", + "monitored_source_id": "aetna_mri_lumbar", + "monitored_source_name": "Aetna CPB 0157", + "monitored_source_owner": "NickLeko", + "monitored_source_url": "https://www.aetna.com/cpb/medical/data/100_199/0157.html", + "notes": "Rules are illustrative for demo. Not a substitute for payer policy documents.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of spine MRI administrative criteria", + "source_name": "Aetna spine MRI policy summary", + "source_type": "manual_summary", + "source_url": "https://www.aetna.com/cpb/medical/data/100_199/0157.html" + }, + "requirements_checked": [ + "conservative_therapy_weeks", + "neuro_red_flags_documented", + "prior_imaging_result", + "symptom_duration_weeks" + ], + "rulebook_active_release_id": "2026-04-09-active-v0.5", + "rules_version": "0.5", + "run_id": "__RUN_ID__", + "site_of_care": "outpatient", + "specialty": "Primary Care", + "submission_readiness": false, + "timestamp_utc": "__TIMESTAMP_UTC__" + }, + "blockers": { + "not_documented": [], + "not_met": [ + { + "key": "conservative_therapy_weeks", + "label": "Conservative therapy duration (weeks)", + "reason": "Documented value (5) below requirement (>= 6.0). Clarify or justify.", + "status": "NOT_MET" + }, + { + "key": "symptom_duration_weeks", + "label": "Symptom duration (weeks)", + "reason": "Documented value (5) below requirement (>= 6.0). Clarify or justify.", + "status": "NOT_MET" + } + ] + }, + "evidence_map": { + "conservative_therapy_weeks": [ + { + "end": 37, + "start": 25, + "text": "PT x 5 weeks" + } + ], + "neuro_red_flags_documented": [ + { + "end": 87, + "start": 50, + "text": "Denies weakness. Denies bowel/bladder" + } + ], + "prior_imaging_result": [ + { + "end": 113, + "start": 97, + "text": "No prior imaging" + } + ], + "symptom_duration_weeks": [ + { + "end": 23, + "start": 16, + "text": "5 weeks" + } + ] + }, + "facts": { + "ahi_documented": null, + "conservative_therapy_weeks": 5, + "mechanical_symptoms_documented": null, + "neuro_red_flags_documented": true, + "osa_diagnosis": null, + "prior_imaging_result": "none", + "sleep_study_date": null, + "symptom_duration_weeks": 5 + }, + "metrics": { + "compliance_rate": 50.0, + "compliant_count": 2, + "extraction_failure_count": 0, + "extraction_success_rate": 100.0, + "non_compliant_count": 2 + }, + "overall_status": "NOT_READY", + "policy_trust_level": "demo", + "provenance": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": "daily", + "monitored_source_id": "aetna_mri_lumbar", + "monitored_source_name": "Aetna CPB 0157", + "monitored_source_owner": "NickLeko", + "monitored_source_url": "https://www.aetna.com/cpb/medical/data/100_199/0157.html", + "notes": "Rules are illustrative for demo. Not a substitute for payer policy documents.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of spine MRI administrative criteria", + "source_name": "Aetna spine MRI policy summary", + "source_type": "manual_summary", + "source_url": "https://www.aetna.com/cpb/medical/data/100_199/0157.html" + }, + "readiness_score": 75, + "report": { + "audit_trail": { + "blocking_issues": { + "not_documented": [], + "not_met": [ + { + "key": "conservative_therapy_weeks", + "label": "Conservative therapy duration (weeks)", + "reason": "Documented value (5) below requirement (>= 6.0). Clarify or justify.", + "status": "NOT_MET" + }, + { + "key": "symptom_duration_weeks", + "label": "Symptom duration (weeks)", + "reason": "Documented value (5) below requirement (>= 6.0). Clarify or justify.", + "status": "NOT_MET" + } + ] + }, + "evaluation_warnings": [ + "Policy trust remains DEMO for this procedure. Verify against official policy before real-world use." + ], + "evidence_map": { + "conservative_therapy_weeks": [ + { + "end": 37, + "start": 25, + "text": "PT x 5 weeks" + } + ], + "neuro_red_flags_documented": [ + { + "end": 87, + "start": 50, + "text": "Denies weakness. Denies bowel/bladder" + } + ], + "prior_imaging_result": [ + { + "end": 113, + "start": 97, + "text": "No prior imaging" + } + ], + "symptom_duration_weeks": [ + { + "end": 23, + "start": 16, + "text": "5 weeks" + } + ] + }, + "facts_extracted": { + "ahi_documented": null, + "conservative_therapy_weeks": 5, + "mechanical_symptoms_documented": null, + "neuro_red_flags_documented": true, + "osa_diagnosis": null, + "prior_imaging_result": "none", + "sleep_study_date": null, + "symptom_duration_weeks": 5 + }, + "invariant_errors": [], + "metrics": { + "compliance_rate": 50.0, + "compliant_count": 2, + "extraction_failure_count": 0, + "extraction_success_rate": 100.0, + "non_compliant_count": 2 + }, + "note_hash": "94ee8ba7171f7dea", + "note_length": 114, + "overall_status": "NOT_READY", + "payer": "Aetna", + "policy_trust_level": "demo", + "procedure_code": "MRI_LUMBAR", + "procedure_name": "MRI Lumbar Spine (no contrast)", + "provenance_snapshot": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": "daily", + "monitored_source_id": "aetna_mri_lumbar", + "monitored_source_name": "Aetna CPB 0157", + "monitored_source_owner": "NickLeko", + "monitored_source_url": "https://www.aetna.com/cpb/medical/data/100_199/0157.html", + "notes": "Rules are illustrative for demo. Not a substitute for payer policy documents.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of spine MRI administrative criteria", + "source_name": "Aetna spine MRI policy summary", + "source_type": "manual_summary", + "source_url": "https://www.aetna.com/cpb/medical/data/100_199/0157.html" + }, + "requirements_checked": [ + "conservative_therapy_weeks", + "neuro_red_flags_documented", + "prior_imaging_result", + "symptom_duration_weeks" + ], + "rulebook_active_release_id": "2026-04-09-active-v0.5", + "rules_version": "0.5", + "run_id": "__RUN_ID__", + "site_of_care": "outpatient", + "specialty": "Primary Care", + "submission_readiness": false, + "timestamp_utc": "__TIMESTAMP_UTC__" + }, + "letter_draft": "", + "met_count": 2, + "not_documented_count": 0, + "not_met_count": 2, + "readiness_score": 75, + "results": [ + { + "evidence": "PT/NSAIDs/activity modification trial documented", + "evidence_snippets": [ + "PT x 5 weeks" + ], + "evidence_spans": [ + { + "end": 37, + "start": 25, + "text": "PT x 5 weeks" + } + ], + "key": "conservative_therapy_weeks", + "label": "Conservative therapy duration (weeks)", + "reason": "Documented value (5) below requirement (>= 6.0). Clarify or justify.", + "status": "NOT_MET" + }, + { + "evidence": "Weakness, saddle anesthesia, bowel/bladder changes explicitly documented as present or denied", + "evidence_snippets": [ + "Denies weakness. Denies bowel/bladder" + ], + "evidence_spans": [ + { + "end": 87, + "start": 50, + "text": "Denies weakness. Denies bowel/bladder" + } + ], + "key": "neuro_red_flags_documented", + "label": "Neuro red flags explicitly addressed (present or denied)", + "reason": "Explicitly addressed in documentation (present/affirmed).", + "status": "MET" + }, + { + "evidence": "X-ray/CT results or note of none", + "evidence_snippets": [ + "No prior imaging" + ], + "evidence_spans": [ + { + "end": 113, + "start": 97, + "text": "No prior imaging" + } + ], + "key": "prior_imaging_result", + "label": "Prior imaging result", + "reason": "Documented: none.", + "status": "MET" + }, + { + "evidence": "Duration supports guideline-based escalation", + "evidence_snippets": [ + "5 weeks" + ], + "evidence_spans": [ + { + "end": 23, + "start": 16, + "text": "5 weeks" + } + ], + "key": "symptom_duration_weeks", + "label": "Symptom duration (weeks)", + "reason": "Documented value (5) below requirement (>= 6.0). Clarify or justify.", + "status": "NOT_MET" + } + ], + "rule_reasons": [ + "Conservative therapy duration (weeks): NOT_MET \u2014 Documented value (5) below requirement (>= 6.0). Clarify or justify.", + "Symptom duration (weeks): NOT_MET \u2014 Documented value (5) below requirement (>= 6.0). Clarify or justify." + ] + }, + "request": { + "dx_codes": [ + "M54.5" + ], + "note_text": "Low back pain x 5 weeks. PT x 5 weeks documented. Denies weakness. Denies bowel/bladder changes. No prior imaging.", + "payer": "Aetna", + "procedure_code": "MRI_LUMBAR", + "site_of_care": "outpatient", + "specialty": "Primary Care" + }, + "results": [ + { + "evidence": "PT/NSAIDs/activity modification trial documented", + "evidence_snippets": [ + "PT x 5 weeks" + ], + "evidence_spans": [ + { + "end": 37, + "start": 25, + "text": "PT x 5 weeks" + } + ], + "key": "conservative_therapy_weeks", + "label": "Conservative therapy duration (weeks)", + "reason": "Documented value (5) below requirement (>= 6.0). Clarify or justify.", + "status": "NOT_MET" + }, + { + "evidence": "Weakness, saddle anesthesia, bowel/bladder changes explicitly documented as present or denied", + "evidence_snippets": [ + "Denies weakness. Denies bowel/bladder" + ], + "evidence_spans": [ + { + "end": 87, + "start": 50, + "text": "Denies weakness. Denies bowel/bladder" + } + ], + "key": "neuro_red_flags_documented", + "label": "Neuro red flags explicitly addressed (present or denied)", + "reason": "Explicitly addressed in documentation (present/affirmed).", + "status": "MET" + }, + { + "evidence": "X-ray/CT results or note of none", + "evidence_snippets": [ + "No prior imaging" + ], + "evidence_spans": [ + { + "end": 113, + "start": 97, + "text": "No prior imaging" + } + ], + "key": "prior_imaging_result", + "label": "Prior imaging result", + "reason": "Documented: none.", + "status": "MET" + }, + { + "evidence": "Duration supports guideline-based escalation", + "evidence_snippets": [ + "5 weeks" + ], + "evidence_spans": [ + { + "end": 23, + "start": 16, + "text": "5 weeks" + } + ], + "key": "symptom_duration_weeks", + "label": "Symptom duration (weeks)", + "reason": "Documented value (5) below requirement (>= 6.0). Clarify or justify.", + "status": "NOT_MET" + } + ], + "rule_reasons": [ + "Conservative therapy duration (weeks): NOT_MET \u2014 Documented value (5) below requirement (>= 6.0). Clarify or justify.", + "Symptom duration (weeks): NOT_MET \u2014 Documented value (5) below requirement (>= 6.0). Clarify or justify." + ], + "submission_readiness": false, + "supported_procedure": { + "display_name": "MRI Lumbar Spine (no contrast)", + "metadata": { + "category": "advanced_imaging", + "last_rule_update": "2026-04-09", + "notes": [ + "Demo rule set only; not a substitute for payer policy review.", + "Requires conservative therapy, symptom duration, imaging context, and explicit red-flag review." + ], + "rule_family": "spine_mri_conservative_therapy", + "summary": "Administrative readiness check for lumbar spine MRI requests using a narrow deterministic evidence contract.", + "supported_sites": [ + "outpatient" + ] + }, + "monitored_for_drift": true, + "payer": "Aetna", + "policy_trust_level": "demo", + "procedure_code": "MRI_LUMBAR", + "provenance": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": "daily", + "monitored_source_id": "aetna_mri_lumbar", + "monitored_source_name": "Aetna CPB 0157", + "monitored_source_owner": "NickLeko", + "monitored_source_url": "https://www.aetna.com/cpb/medical/data/100_199/0157.html", + "notes": "Rules are illustrative for demo. Not a substitute for payer policy documents.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of spine MRI administrative criteria", + "source_name": "Aetna spine MRI policy summary", + "source_type": "manual_summary", + "source_url": "https://www.aetna.com/cpb/medical/data/100_199/0157.html" + }, + "required_field_keys": [ + "conservative_therapy_weeks", + "neuro_red_flags_documented", + "prior_imaging_result", + "symptom_duration_weeks" + ], + "requirements": [ + { + "allowed": [], + "evidence": "PT/NSAIDs/activity modification trial documented", + "key": "conservative_therapy_weeks", + "label": "Conservative therapy duration (weeks)", + "min": 6.0, + "type": "number" + }, + { + "allowed": [], + "evidence": "Weakness, saddle anesthesia, bowel/bladder changes explicitly documented as present or denied", + "key": "neuro_red_flags_documented", + "label": "Neuro red flags explicitly addressed (present or denied)", + "min": null, + "type": "boolean" + }, + { + "allowed": [ + "none", + "inconclusive", + "abnormal" + ], + "evidence": "X-ray/CT results or note of none", + "key": "prior_imaging_result", + "label": "Prior imaging result", + "min": null, + "type": "enum" + }, + { + "allowed": [], + "evidence": "Duration supports guideline-based escalation", + "key": "symptom_duration_weeks", + "label": "Symptom duration (weeks)", + "min": 6.0, + "type": "number" + } + ] + }, + "warnings": [ + "Policy trust remains DEMO for this procedure. Verify against official policy before real-world use." + ] +} diff --git a/test/golden/evaluations/MRI-KNEE-01-ready.json b/test/golden/evaluations/MRI-KNEE-01-ready.json new file mode 100644 index 0000000..85ccfcf --- /dev/null +++ b/test/golden/evaluations/MRI-KNEE-01-ready.json @@ -0,0 +1,492 @@ +{ + "audit_trail": { + "blocking_issues": { + "not_documented": [], + "not_met": [] + }, + "evaluation_warnings": [ + "Policy trust remains DEMO for this procedure. Verify against official policy before real-world use." + ], + "evidence_map": { + "conservative_therapy_weeks": [ + { + "end": 77, + "start": 63, + "text": "PT for 8 weeks" + } + ], + "mechanical_symptoms_documented": [ + { + "end": 41, + "start": 16, + "text": "with locking and catching" + } + ], + "prior_imaging_result": [ + { + "end": 137, + "start": 126, + "text": "xray normal" + } + ], + "symptom_duration_weeks": [ + { + "end": 51, + "start": 44, + "text": "8 weeks" + } + ] + }, + "facts_extracted": { + "ahi_documented": null, + "conservative_therapy_weeks": 8, + "mechanical_symptoms_documented": true, + "neuro_red_flags_documented": null, + "osa_diagnosis": null, + "prior_imaging_result": "inconclusive", + "sleep_study_date": null, + "symptom_duration_weeks": 8 + }, + "invariant_errors": [], + "metrics": { + "compliance_rate": 100.0, + "compliant_count": 4, + "extraction_failure_count": 0, + "extraction_success_rate": 100.0, + "non_compliant_count": 0 + }, + "note_hash": "59421e8561fe0462", + "note_length": 151, + "overall_status": "READY", + "payer": "Aetna", + "policy_trust_level": "demo", + "procedure_code": "MRI_KNEE", + "procedure_name": "MRI Knee (no contrast)", + "provenance_snapshot": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": null, + "monitored_source_id": null, + "monitored_source_name": null, + "monitored_source_owner": null, + "monitored_source_url": null, + "notes": "Rules are illustrative for demo and intentionally limited to documentation completeness.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of knee MRI administrative documentation criteria", + "source_name": "Aetna knee MRI policy summary", + "source_type": "manual_summary", + "source_url": null + }, + "requirements_checked": [ + "conservative_therapy_weeks", + "symptom_duration_weeks", + "prior_imaging_result", + "mechanical_symptoms_documented" + ], + "rulebook_active_release_id": "2026-04-09-active-v0.5", + "rules_version": "0.5", + "run_id": "__RUN_ID__", + "site_of_care": "outpatient", + "specialty": "Orthopedics", + "submission_readiness": true, + "timestamp_utc": "__TIMESTAMP_UTC__" + }, + "blockers": { + "not_documented": [], + "not_met": [] + }, + "evidence_map": { + "conservative_therapy_weeks": [ + { + "end": 77, + "start": 63, + "text": "PT for 8 weeks" + } + ], + "mechanical_symptoms_documented": [ + { + "end": 41, + "start": 16, + "text": "with locking and catching" + } + ], + "prior_imaging_result": [ + { + "end": 137, + "start": 126, + "text": "xray normal" + } + ], + "symptom_duration_weeks": [ + { + "end": 51, + "start": 44, + "text": "8 weeks" + } + ] + }, + "facts": { + "ahi_documented": null, + "conservative_therapy_weeks": 8, + "mechanical_symptoms_documented": true, + "neuro_red_flags_documented": null, + "osa_diagnosis": null, + "prior_imaging_result": "inconclusive", + "sleep_study_date": null, + "symptom_duration_weeks": 8 + }, + "metrics": { + "compliance_rate": 100.0, + "compliant_count": 4, + "extraction_failure_count": 0, + "extraction_success_rate": 100.0, + "non_compliant_count": 0 + }, + "overall_status": "READY", + "policy_trust_level": "demo", + "provenance": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": null, + "monitored_source_id": null, + "monitored_source_name": null, + "monitored_source_owner": null, + "monitored_source_url": null, + "notes": "Rules are illustrative for demo and intentionally limited to documentation completeness.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of knee MRI administrative documentation criteria", + "source_name": "Aetna knee MRI policy summary", + "source_type": "manual_summary", + "source_url": null + }, + "readiness_score": 100, + "report": { + "audit_trail": { + "blocking_issues": { + "not_documented": [], + "not_met": [] + }, + "evaluation_warnings": [ + "Policy trust remains DEMO for this procedure. Verify against official policy before real-world use." + ], + "evidence_map": { + "conservative_therapy_weeks": [ + { + "end": 77, + "start": 63, + "text": "PT for 8 weeks" + } + ], + "mechanical_symptoms_documented": [ + { + "end": 41, + "start": 16, + "text": "with locking and catching" + } + ], + "prior_imaging_result": [ + { + "end": 137, + "start": 126, + "text": "xray normal" + } + ], + "symptom_duration_weeks": [ + { + "end": 51, + "start": 44, + "text": "8 weeks" + } + ] + }, + "facts_extracted": { + "ahi_documented": null, + "conservative_therapy_weeks": 8, + "mechanical_symptoms_documented": true, + "neuro_red_flags_documented": null, + "osa_diagnosis": null, + "prior_imaging_result": "inconclusive", + "sleep_study_date": null, + "symptom_duration_weeks": 8 + }, + "invariant_errors": [], + "metrics": { + "compliance_rate": 100.0, + "compliant_count": 4, + "extraction_failure_count": 0, + "extraction_success_rate": 100.0, + "non_compliant_count": 0 + }, + "note_hash": "59421e8561fe0462", + "note_length": 151, + "overall_status": "READY", + "payer": "Aetna", + "policy_trust_level": "demo", + "procedure_code": "MRI_KNEE", + "procedure_name": "MRI Knee (no contrast)", + "provenance_snapshot": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": null, + "monitored_source_id": null, + "monitored_source_name": null, + "monitored_source_owner": null, + "monitored_source_url": null, + "notes": "Rules are illustrative for demo and intentionally limited to documentation completeness.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of knee MRI administrative documentation criteria", + "source_name": "Aetna knee MRI policy summary", + "source_type": "manual_summary", + "source_url": null + }, + "requirements_checked": [ + "conservative_therapy_weeks", + "symptom_duration_weeks", + "prior_imaging_result", + "mechanical_symptoms_documented" + ], + "rulebook_active_release_id": "2026-04-09-active-v0.5", + "rules_version": "0.5", + "run_id": "__RUN_ID__", + "site_of_care": "outpatient", + "specialty": "Orthopedics", + "submission_readiness": true, + "timestamp_utc": "__TIMESTAMP_UTC__" + }, + "letter_draft": "", + "met_count": 4, + "not_documented_count": 0, + "not_met_count": 0, + "readiness_score": 100, + "results": [ + { + "evidence": "PT/activity modification/NSAID trial documented", + "evidence_snippets": [ + "PT for 8 weeks" + ], + "evidence_spans": [ + { + "end": 77, + "start": 63, + "text": "PT for 8 weeks" + } + ], + "key": "conservative_therapy_weeks", + "label": "Conservative therapy duration (weeks)", + "reason": "Documented value: 8.", + "status": "MET" + }, + { + "evidence": "Persistent symptoms documented long enough to justify escalation", + "evidence_snippets": [ + "8 weeks" + ], + "evidence_spans": [ + { + "end": 51, + "start": 44, + "text": "8 weeks" + } + ], + "key": "symptom_duration_weeks", + "label": "Symptom duration (weeks)", + "reason": "Documented value: 8.", + "status": "MET" + }, + { + "evidence": "Prior knee imaging such as x-ray is documented and not simply absent", + "evidence_snippets": [ + "xray normal" + ], + "evidence_spans": [ + { + "end": 137, + "start": 126, + "text": "xray normal" + } + ], + "key": "prior_imaging_result", + "label": "Prior imaging result", + "reason": "Documented: inconclusive.", + "status": "MET" + }, + { + "evidence": "Locking, catching, buckling, giving way, or instability explicitly documented", + "evidence_snippets": [ + "with locking and catching" + ], + "evidence_spans": [ + { + "end": 41, + "start": 16, + "text": "with locking and catching" + } + ], + "key": "mechanical_symptoms_documented", + "label": "Mechanical symptoms explicitly addressed (present or denied)", + "reason": "Explicitly addressed in documentation (present/affirmed).", + "status": "MET" + } + ], + "rule_reasons": [] + }, + "request": { + "dx_codes": [ + "M25.561" + ], + "note_text": "Right knee pain with locking and catching x 8 weeks. Completed PT for 8 weeks and NSAIDs with minimal improvement. Prior knee xray normal/unremarkable.", + "payer": "Aetna", + "procedure_code": "MRI_KNEE", + "site_of_care": "outpatient", + "specialty": "Orthopedics" + }, + "results": [ + { + "evidence": "PT/activity modification/NSAID trial documented", + "evidence_snippets": [ + "PT for 8 weeks" + ], + "evidence_spans": [ + { + "end": 77, + "start": 63, + "text": "PT for 8 weeks" + } + ], + "key": "conservative_therapy_weeks", + "label": "Conservative therapy duration (weeks)", + "reason": "Documented value: 8.", + "status": "MET" + }, + { + "evidence": "Persistent symptoms documented long enough to justify escalation", + "evidence_snippets": [ + "8 weeks" + ], + "evidence_spans": [ + { + "end": 51, + "start": 44, + "text": "8 weeks" + } + ], + "key": "symptom_duration_weeks", + "label": "Symptom duration (weeks)", + "reason": "Documented value: 8.", + "status": "MET" + }, + { + "evidence": "Prior knee imaging such as x-ray is documented and not simply absent", + "evidence_snippets": [ + "xray normal" + ], + "evidence_spans": [ + { + "end": 137, + "start": 126, + "text": "xray normal" + } + ], + "key": "prior_imaging_result", + "label": "Prior imaging result", + "reason": "Documented: inconclusive.", + "status": "MET" + }, + { + "evidence": "Locking, catching, buckling, giving way, or instability explicitly documented", + "evidence_snippets": [ + "with locking and catching" + ], + "evidence_spans": [ + { + "end": 41, + "start": 16, + "text": "with locking and catching" + } + ], + "key": "mechanical_symptoms_documented", + "label": "Mechanical symptoms explicitly addressed (present or denied)", + "reason": "Explicitly addressed in documentation (present/affirmed).", + "status": "MET" + } + ], + "rule_reasons": [], + "submission_readiness": true, + "supported_procedure": { + "display_name": "MRI Knee (no contrast)", + "metadata": { + "category": "advanced_imaging", + "last_rule_update": "2026-04-09", + "notes": [ + "This demo pathway checks only administrative completeness, not orthopedic appropriateness.", + "Requires symptom duration, conservative therapy, prior imaging context, and explicit review of mechanical symptoms." + ], + "rule_family": "extremity_mri_conservative_therapy", + "summary": "Administrative readiness check for knee MRI requests using a narrow deterministic documentation contract.", + "supported_sites": [ + "outpatient" + ] + }, + "monitored_for_drift": false, + "payer": "Aetna", + "policy_trust_level": "demo", + "procedure_code": "MRI_KNEE", + "provenance": { + "last_reviewed": "2026-04-09", + "monitored_check_frequency": null, + "monitored_source_id": null, + "monitored_source_name": null, + "monitored_source_owner": null, + "monitored_source_url": null, + "notes": "Rules are illustrative for demo and intentionally limited to documentation completeness.", + "rule_last_updated": "2026-04-09", + "rule_source_label": "Human-curated summary of knee MRI administrative documentation criteria", + "source_name": "Aetna knee MRI policy summary", + "source_type": "manual_summary", + "source_url": null + }, + "required_field_keys": [ + "conservative_therapy_weeks", + "symptom_duration_weeks", + "prior_imaging_result", + "mechanical_symptoms_documented" + ], + "requirements": [ + { + "allowed": [], + "evidence": "PT/activity modification/NSAID trial documented", + "key": "conservative_therapy_weeks", + "label": "Conservative therapy duration (weeks)", + "min": 6.0, + "type": "number" + }, + { + "allowed": [], + "evidence": "Persistent symptoms documented long enough to justify escalation", + "key": "symptom_duration_weeks", + "label": "Symptom duration (weeks)", + "min": 6.0, + "type": "number" + }, + { + "allowed": [ + "inconclusive", + "abnormal" + ], + "evidence": "Prior knee imaging such as x-ray is documented and not simply absent", + "key": "prior_imaging_result", + "label": "Prior imaging result", + "min": null, + "type": "enum" + }, + { + "allowed": [], + "evidence": "Locking, catching, buckling, giving way, or instability explicitly documented", + "key": "mechanical_symptoms_documented", + "label": "Mechanical symptoms explicitly addressed (present or denied)", + "min": null, + "type": "boolean" + } + ] + }, + "warnings": [ + "Policy trust remains DEMO for this procedure. Verify against official policy before real-world use." + ] +} diff --git a/test/golden/governance/drift_status.json b/test/golden/governance/drift_status.json new file mode 100644 index 0000000..a906b81 --- /dev/null +++ b/test/golden/governance/drift_status.json @@ -0,0 +1,29 @@ +{ + "any_review_required": true, + "sources": [ + { + "check_frequency": "daily", + "days_since_last_checked": "__DAYS_SINCE_LAST_CHECKED__", + "freshness_status": "STALE", + "id": "aetna_mri_lumbar", + "last_checked_utc": "2026-02-06T06:12:45Z", + "last_rule_reviewed": "2026-04-09", + "latest_diff_path": null, + "latest_event": "BOOTSTRAP_SNAPSHOT_CREATED", + "latest_hash": "480f08eba64b3ad166504d92675a8751c598f4c4f364c38aaf702c346e525209", + "latest_snapshot_path": "policy_snapshots/aetna_mri_lumbar/latest.json", + "notes": "Monitored for drift; rules curated offline.", + "owner": "NickLeko", + "payer": "Aetna", + "procedure_code": "MRI_LUMBAR", + "review_reason": "Snapshot exceeds the configured daily monitoring window.", + "rule_source_label": "Human-curated summary of spine MRI administrative criteria", + "source_name": "Aetna CPB 0157", + "source_type": "official_policy_web", + "status": "OK", + "trust_level": "verified", + "url": "https://www.aetna.com/cpb/medical/data/100_199/0157.html" + } + ], + "stale_source_count": 1 +} diff --git a/test/golden/governance/rulebook_diff_reviewed_vs_active.json b/test/golden/governance/rulebook_diff_reviewed_vs_active.json new file mode 100644 index 0000000..914e158 --- /dev/null +++ b/test/golden/governance/rulebook_diff_reviewed_vs_active.json @@ -0,0 +1,25 @@ +{ + "added_procedures": [ + "MRI_KNEE" + ], + "changed_policy_sources": [], + "changed_procedures": [], + "changed_provenance": [ + "MRI_KNEE" + ], + "from_release_id": "2026-04-09-reviewed-v0.4", + "from_stage": "reviewed", + "removed_procedures": [], + "rules_version_from": "0.4", + "rules_version_to": "0.5", + "summary_lines": [ + "Rules version: 0.4 -> 0.5", + "Added procedures: MRI_KNEE", + "Removed procedures: none", + "Changed procedures: none", + "Changed provenance entries: MRI_KNEE", + "Changed policy source entries: none" + ], + "to_release_id": "2026-04-09-active-v0.5", + "to_stage": "active" +} diff --git a/test/golden/governance/rulebook_status.json b/test/golden/governance/rulebook_status.json new file mode 100644 index 0000000..6bc24c7 --- /dev/null +++ b/test/golden/governance/rulebook_status.json @@ -0,0 +1,62 @@ +{ + "active_release_id": "2026-04-09-active-v0.5", + "manifest_version": "1", + "releases": [ + { + "based_on_release_id": "2026-04-09-reviewed-v0.4", + "created_at": "2026-04-09T12:15:00Z", + "files": { + "policy_sources_path": "rulebook/releases/2026-04-09-active-v0.5/policy_sources.yaml", + "provenance_path": "rulebook/releases/2026-04-09-active-v0.5/provenance.yaml", + "rules_path": "rulebook/releases/2026-04-09-active-v0.5/payer_rules.yaml" + }, + "notes": [ + "This active snapshot should match the runtime files under rules/." + ], + "procedures": [ + "CPAP_DEVICE", + "MRI_CERVICAL", + "MRI_KNEE", + "MRI_LUMBAR" + ], + "release_id": "2026-04-09-active-v0.5", + "reviewed_at": "2026-04-09", + "reviewer": "demo-maintainer", + "rules_version": "0.5", + "runtime_matches": true, + "stage": "active", + "summary": "Active rulebook after the narrow non-spine knee MRI expansion." + }, + { + "based_on_release_id": null, + "created_at": "2026-04-09T10:45:00Z", + "files": { + "policy_sources_path": "rulebook/releases/2026-04-09-reviewed-v0.4/policy_sources.yaml", + "provenance_path": "rulebook/releases/2026-04-09-reviewed-v0.4/provenance.yaml", + "rules_path": "rulebook/releases/2026-04-09-reviewed-v0.4/payer_rules.yaml" + }, + "notes": [ + "Historical reviewed snapshot retained for governance diffs." + ], + "procedures": [ + "CPAP_DEVICE", + "MRI_CERVICAL", + "MRI_LUMBAR" + ], + "release_id": "2026-04-09-reviewed-v0.4", + "reviewed_at": "2026-04-09", + "reviewer": "demo-maintainer", + "rules_version": "0.4", + "runtime_matches": null, + "stage": "reviewed", + "summary": "Reviewed rulebook before the narrow non-spine knee MRI expansion." + } + ], + "runtime_rules_version": "0.5", + "stage_assignments": { + "active": "2026-04-09-active-v0.5", + "draft": null, + "reviewed": "2026-04-09-reviewed-v0.4" + }, + "validation_errors": [] +} diff --git a/test/test_acceptance_snapshots.py b/test/test_acceptance_snapshots.py new file mode 100644 index 0000000..828c9ef --- /dev/null +++ b/test/test_acceptance_snapshots.py @@ -0,0 +1,34 @@ +import json +from pathlib import Path + +from engine.acceptance import ( + DEFAULT_ACCEPTANCE_CASE_IDS, + build_acceptance_evaluation_payload, + build_acceptance_governance_payloads, +) +from engine.service import ReadinessService + +GOLDEN_ROOT = Path("test/golden") + + +def test_acceptance_evaluation_snapshots_match_goldens(): + service = ReadinessService() + + for case_id in DEFAULT_ACCEPTANCE_CASE_IDS: + expected = json.loads((GOLDEN_ROOT / "evaluations" / f"{case_id}.json").read_text(encoding="utf-8")) + actual = build_acceptance_evaluation_payload(service, case_id) + assert actual == expected + + +def test_acceptance_governance_snapshots_match_goldens(): + service = ReadinessService() + expected_payloads = { + "drift_status": json.loads((GOLDEN_ROOT / "governance" / "drift_status.json").read_text(encoding="utf-8")), + "rulebook_status": json.loads((GOLDEN_ROOT / "governance" / "rulebook_status.json").read_text(encoding="utf-8")), + "rulebook_diff_reviewed_vs_active": json.loads( + (GOLDEN_ROOT / "governance" / "rulebook_diff_reviewed_vs_active.json").read_text(encoding="utf-8") + ), + } + + actual_payloads = build_acceptance_governance_payloads(service) + assert actual_payloads == expected_payloads diff --git a/test/test_api.py b/test/test_api.py new file mode 100644 index 0000000..e51f9b8 --- /dev/null +++ b/test/test_api.py @@ -0,0 +1,94 @@ +from fastapi.testclient import TestClient + +from api import app +from engine.service import ReadinessService + +client = TestClient(app) + + +def test_health_endpoint(): + response = client.get("/health") + + assert response.status_code == 200 + payload = response.json() + assert payload["service"] == "Prior Authorization Readiness Copilot" + assert payload["synthetic_only"] is True + + +def test_supported_procedures_endpoint(): + response = client.get("/supported-procedures") + + assert response.status_code == 200 + payload = response.json() + assert any(item["procedure_code"] == "MRI_LUMBAR" for item in payload) + assert any(item["procedure_code"] == "MRI_CERVICAL" for item in payload) + assert any(item["procedure_code"] == "MRI_KNEE" for item in payload) + cervical = next(item for item in payload if item["procedure_code"] == "MRI_CERVICAL") + assert cervical["metadata"]["category"] == "advanced_imaging" + assert cervical["provenance"]["rule_source_label"] == "Human-curated summary of cervical spine MRI administrative criteria" + knee = next(item for item in payload if item["procedure_code"] == "MRI_KNEE") + assert knee["metadata"]["rule_family"] == "extremity_mri_conservative_therapy" + + +def test_evaluate_endpoint_matches_service_behavior(): + service = ReadinessService() + request = service.get_demo_case_request("MRI-08-edge-below-threshold") + + response = client.post("/evaluate", json=request.model_dump(mode="json")) + + assert response.status_code == 200 + payload = response.json() + assert payload["overall_status"] == "NOT_READY" + assert payload["submission_readiness"] is False + assert payload["blockers"]["not_met"] + + +def test_evaluate_endpoint_rejects_unsupported_scope(): + response = client.post( + "/evaluate", + json={ + "payer": "Aetna", + "procedure_code": "UNKNOWN_PROC", + "dx_codes": ["Z00.00"], + "site_of_care": "outpatient", + "specialty": "Primary Care", + "note_text": "Synthetic note.", + }, + ) + + assert response.status_code == 422 + payload = response.json() + assert payload["error"] == "unsupported_scope" + + +def test_drift_status_endpoint(): + response = client.get("/drift-status") + + assert response.status_code == 200 + payload = response.json() + assert "sources" in payload + assert payload["sources"][0]["source_name"] == "Aetna CPB 0157" + assert "freshness_status" in payload["sources"][0] + + +def test_rulebook_status_endpoint(): + response = client.get("/rulebook") + + assert response.status_code == 200 + payload = response.json() + assert payload["active_release_id"] == "2026-04-09-active-v0.5" + assert payload["validation_errors"] == [] + + +def test_rulebook_diff_endpoint(): + response = client.get( + "/rulebook/diff", + params={ + "from_release_id": "2026-04-09-reviewed-v0.4", + "to_release_id": "2026-04-09-active-v0.5", + }, + ) + + assert response.status_code == 200 + payload = response.json() + assert payload["added_procedures"] == ["MRI_KNEE"] diff --git a/test/test_artifact_generation.py b/test/test_artifact_generation.py new file mode 100644 index 0000000..3931393 --- /dev/null +++ b/test/test_artifact_generation.py @@ -0,0 +1,29 @@ +import json + +from scripts.generate_artifacts import main + + +def test_artifact_generation_writes_enriched_outputs(tmp_path, monkeypatch): + monkeypatch.setenv("PA_COPILOT_ARTIFACTS_DIR", str(tmp_path)) + + exit_code = main() + + assert exit_code == 0 + assert (tmp_path / "MRI-CERV-01-ready.json").exists() + assert (tmp_path / "MRI-KNEE-01-ready.json").exists() + assert (tmp_path / "featured_demo_cases.json").exists() + assert (tmp_path / "status.json").exists() + assert (tmp_path / "rulebook_status.json").exists() + assert (tmp_path / "rulebook_diff_reviewed_vs_active.json").exists() + assert (tmp_path / "drift_report.md").exists() + + registry_payload = json.loads((tmp_path / "supported_procedures.json").read_text(encoding="utf-8")) + assert any(item["procedure_code"] == "MRI_CERVICAL" for item in registry_payload) + assert any(item["procedure_code"] == "MRI_KNEE" for item in registry_payload) + + featured_payload = json.loads((tmp_path / "featured_demo_cases.json").read_text(encoding="utf-8")) + assert any(item["id"] == "MRI-CERV-01-ready" for item in featured_payload) + assert any(item["id"] == "MRI-KNEE-01-ready" for item in featured_payload) + + rulebook_payload = json.loads((tmp_path / "rulebook_status.json").read_text(encoding="utf-8")) + assert rulebook_payload["active_release_id"] == "2026-04-09-active-v0.5" diff --git a/test/test_cli.py b/test/test_cli.py new file mode 100644 index 0000000..3ecbe27 --- /dev/null +++ b/test/test_cli.py @@ -0,0 +1,91 @@ +import json +from pathlib import Path + +from cli import main + + +def test_cli_list_procedures(capsys): + exit_code = main(["list-procedures"]) + + captured = capsys.readouterr() + assert exit_code == 0 + assert "MRI_LUMBAR" in captured.out + assert "MRI_CERVICAL" in captured.out + assert "MRI_KNEE" in captured.out + assert "category=advanced_imaging" in captured.out + + +def test_cli_evaluate_json(capsys): + exit_code = main(["evaluate", "--demo-case", "CPAP-02-borderline", "--json"]) + + captured = capsys.readouterr() + payload = json.loads(captured.out) + assert exit_code == 0 + assert payload["overall_status"] == "CANNOT_DETERMINE" + + +def test_cli_export_report(tmp_path: Path, capsys): + output_path = tmp_path / "artifact.json" + + exit_code = main( + [ + "export-report", + "--demo-case", + "MRI-01-complete", + "--output", + str(output_path), + "--with-letter", + ] + ) + + captured = capsys.readouterr() + assert exit_code == 0 + assert str(output_path) in captured.out + payload = json.loads(output_path.read_text(encoding="utf-8")) + assert payload["overall_status"] == "READY" + assert "letter" in payload + + +def test_cli_validate_demo_case(capsys): + exit_code = main(["validate-demo-case", "--demo-case", "MRI-05-incomplete"]) + + captured = capsys.readouterr() + assert exit_code == 0 + assert "demo_case=MRI-05-incomplete" in captured.out + + +def test_cli_list_demo_cases_includes_scenario_type(capsys): + exit_code = main(["list-demo-cases"]) + + captured = capsys.readouterr() + assert exit_code == 0 + assert "MRI-CERV-01-ready" in captured.out + assert "expected_status=READY" in captured.out + assert "fixture_label=complete" in captured.out + assert "New procedure coverage" in captured.out + assert "MRI-KNEE-01-ready" in captured.out + assert "Non-spine coverage" in captured.out + + +def test_cli_rulebook_status(capsys): + exit_code = main(["rulebook-status"]) + + captured = capsys.readouterr() + assert exit_code == 0 + assert "2026-04-09-active-v0.5" in captured.out + + +def test_cli_rulebook_diff(capsys): + exit_code = main( + [ + "rulebook-diff", + "--from-release", + "2026-04-09-reviewed-v0.4", + "--to-release", + "2026-04-09-active-v0.5", + ] + ) + + captured = capsys.readouterr() + assert exit_code == 0 + assert "Added procedures: MRI_KNEE" in captured.out diff --git a/test/test_config_contracts.py b/test/test_config_contracts.py index 587c4a2..6909ddf 100644 --- a/test/test_config_contracts.py +++ b/test/test_config_contracts.py @@ -56,6 +56,33 @@ def test_rules_loader_rejects_unknown_requirement_type(tmp_path: Path): load_rules(str(rules_path)) +def test_rules_loader_rejects_invalid_metadata(tmp_path: Path): + rules_path = tmp_path / "bad_rules_metadata.yaml" + rules_path.write_text( + """ +version: 1 +payers: + Aetna: + procedures: + MRI_CERVICAL: + display_name: "MRI Cervical Spine" + metadata: + category: "" + rule_family: "spine_mri_conservative_therapy" + summary: "Test summary" + required: + - key: symptom_duration_weeks + label: "Symptom duration" + type: number + min: 6 +""".strip(), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="metadata.category must be a non-empty string"): + load_rules(str(rules_path)) + + def test_provenance_loader_defaults_missing_file_to_empty_mapping(tmp_path: Path): missing_path = tmp_path / "missing_provenance.yaml" loaded = load_provenance(missing_path) @@ -83,6 +110,17 @@ def test_official_policy_provenance_maps_to_verified(): assert policy_trust_from_provenance(entry) == "verified" +def test_bundled_provenance_contains_rule_source_metadata(): + provenance = load_provenance("rules/provenance.yaml") + entry = get_provenance_entry(provenance, "Aetna", "MRI_CERVICAL") + + assert entry["rule_source_label"] == "Human-curated summary of cervical spine MRI administrative criteria" + assert entry["rule_last_updated"] == "2026-04-09" + + knee_entry = get_provenance_entry(provenance, "Aetna", "MRI_KNEE") + assert knee_entry["rule_source_label"] == "Human-curated summary of knee MRI administrative documentation criteria" + + def test_normalized_dx_codes_are_uppercase_deduped_and_sanitized(): dx_codes = [" m54.5 ", "M54.5", "m%51.26", "", " "] diff --git a/test/test_extract_contracts.py b/test/test_extract_contracts.py index 7d139a5..4839c71 100644 --- a/test/test_extract_contracts.py +++ b/test/test_extract_contracts.py @@ -3,15 +3,11 @@ from engine.evaluate import compute_overall_status, evaluate_requirements from engine.extract import extract_facts - CONTRACT_PATH = Path("EXTRACTION_CONTRACT.md") def test_extract_facts_is_deterministic_for_same_note(): - note = ( - "Low back pain for 8 weeks. Completed PT for 6 weeks. " - "Denies weakness, bowel or bladder changes. No prior imaging documented." - ) + note = "Low back pain for 8 weeks. Completed PT for 6 weeks. Denies weakness, bowel or bladder changes. No prior imaging documented." facts_one, evidence_one = extract_facts(note) facts_two, evidence_two = extract_facts(note) @@ -66,6 +62,11 @@ def test_extraction_contract_describes_current_shapes_and_limits(): assert "The current implementation does not require explicit symptom context for this field." in contract assert 'Type: `"none" | "inconclusive" | "abnormal" | null`' in contract assert "Imaging mention without a result is treated as documented `inconclusive`, not `null`." in contract + assert "### `mechanical_symptoms_documented`" in contract + assert ( + "Positive phrasing takes precedence if the note contains both denial and later affirmative " + "mechanical-symptom language." in contract + ) assert "This field records whether a contextualized date was found. It does not return the parsed date value." in contract assert "This field records whether the numeric value is documented. It does not return the numeric value itself." in contract assert "Possible Extensions" in contract @@ -84,3 +85,42 @@ def test_extraction_contract_examples_match_current_behavior(): facts, _ = extract_facts("Back pain x 2 months. Completed PT and NSAIDs, duration not specified.") assert facts["symptom_duration_weeks"] == 8 assert facts["conservative_therapy_weeks"] is None + + facts, _ = extract_facts("Denies locking or instability. Prior knee xray normal.") + assert facts["mechanical_symptoms_documented"] is False + + +def test_positive_red_flag_evidence_takes_precedence_when_note_contains_conflict(): + note = ( + "Neck pain x 8 weeks. PT x 8 weeks. Denies weakness earlier in visit. " + "Later note: reports progressive weakness and urinary retention. Prior CT abnormal." + ) + + facts, evidence = extract_facts(note) + + assert facts["neuro_red_flags_documented"] is True + assert "neuro_red_flags_documented" in evidence + assert any("progressive weakness" in span["text"].lower() for span in evidence["neuro_red_flags_documented"]) + + +def test_mechanical_symptom_denial_is_captured_as_explicit_documentation(): + note = "Right knee pain x 8 weeks. PT x 8 weeks. Denies locking or instability. Prior knee xray normal." + + facts, evidence = extract_facts(note) + + assert facts["mechanical_symptoms_documented"] is False + assert "mechanical_symptoms_documented" in evidence + assert any("denies locking" in span["text"].lower() for span in evidence["mechanical_symptoms_documented"]) + + +def test_positive_mechanical_symptoms_take_precedence_when_note_contains_conflict(): + note = ( + "Right knee pain x 8 weeks. PT x 8 weeks. Denies locking earlier in visit. " + "Later note: reports buckling with stairs. Prior knee xray normal." + ) + + facts, evidence = extract_facts(note) + + assert facts["mechanical_symptoms_documented"] is True + assert "mechanical_symptoms_documented" in evidence + assert any("buckling" in span["text"].lower() for span in evidence["mechanical_symptoms_documented"]) diff --git a/test/test_letter_draft.py b/test/test_letter_draft.py index 6a6c8f6..411b5be 100644 --- a/test/test_letter_draft.py +++ b/test/test_letter_draft.py @@ -1,7 +1,5 @@ -import pytest - -from engine.schemas import PARequest, ReadinessReport, RequirementResult from engine.letter_draft import draft_letter +from engine.schemas import PARequest, ReadinessReport, RequirementResult def _base_pa(): diff --git a/test/test_policy_monitor.py b/test/test_policy_monitor.py index 266b768..7c6f1e2 100644 --- a/test/test_policy_monitor.py +++ b/test/test_policy_monitor.py @@ -56,6 +56,7 @@ def test_write_snapshot_and_read_latest(tmp_path: Path): payer="Aetna", procedure_code="MRI_LUMBAR", url="https://example.invalid/0157.html", + source_name="Fixture Policy Source", source_type="official_policy_web", trust_level="verified", check_frequency="daily", diff --git a/test/test_rulebook.py b/test/test_rulebook.py new file mode 100644 index 0000000..31d21bb --- /dev/null +++ b/test/test_rulebook.py @@ -0,0 +1,27 @@ +from engine.service import ReadinessService + + +def test_rulebook_status_is_valid_and_matches_runtime(): + service = ReadinessService() + + report = service.get_rulebook_status() + + assert report.active_release_id == "2026-04-09-active-v0.5" + assert report.runtime_rules_version == "0.5" + assert not report.validation_errors + active = next(item for item in report.releases if item.release_id == report.active_release_id) + assert active.runtime_matches is True + assert active.files.rules_path == "rulebook/releases/2026-04-09-active-v0.5/payer_rules.yaml" + assert active.files.provenance_path == "rulebook/releases/2026-04-09-active-v0.5/provenance.yaml" + + +def test_rulebook_diff_highlights_new_knee_pathway(): + service = ReadinessService() + + report = service.get_rulebook_diff("2026-04-09-reviewed-v0.4", "2026-04-09-active-v0.5") + + assert report.rules_version_from == "0.4" + assert report.rules_version_to == "0.5" + assert report.added_procedures == ["MRI_KNEE"] + assert not report.removed_procedures + assert "MRI_KNEE" in report.changed_provenance diff --git a/test/test_service.py b/test/test_service.py new file mode 100644 index 0000000..279d12f --- /dev/null +++ b/test/test_service.py @@ -0,0 +1,176 @@ +from engine.schemas import PARequest +from engine.service import ReadinessService, UnsupportedScopeError + + +def test_service_evaluates_known_ready_demo_case(): + service = ReadinessService() + request = service.get_demo_case_request("MRI-01-complete") + + evaluation = service.evaluate(request) + + assert evaluation.overall_status == "READY" + assert evaluation.submission_readiness is True + assert evaluation.audit_trail.note_hash + assert evaluation.audit_trail.evidence_map["conservative_therapy_weeks"] + assert evaluation.results[0].evidence_spans + + +def test_service_returns_cannot_determine_for_missing_documentation_case(): + service = ReadinessService() + request = service.get_demo_case_request("CPAP-02-borderline") + + evaluation = service.evaluate(request) + + assert evaluation.overall_status == "CANNOT_DETERMINE" + assert evaluation.blockers.not_documented + assert not evaluation.submission_readiness + + +def test_service_lists_new_cervical_procedure_with_registry_metadata(): + service = ReadinessService() + + procedures = service.list_supported_procedures() + cervical = next(item for item in procedures if item.procedure_code == "MRI_CERVICAL") + + assert cervical.metadata.category == "advanced_imaging" + assert cervical.metadata.rule_family == "spine_mri_conservative_therapy" + assert cervical.required_field_keys == [ + "conservative_therapy_weeks", + "neuro_red_flags_documented", + "prior_imaging_result", + "symptom_duration_weeks", + ] + assert cervical.provenance.rule_source_label == "Human-curated summary of cervical spine MRI administrative criteria" + assert cervical.monitored_for_drift is False + + +def test_service_lists_new_knee_procedure_with_required_metadata(): + service = ReadinessService() + + procedures = service.list_supported_procedures() + knee = next(item for item in procedures if item.procedure_code == "MRI_KNEE") + + assert knee.metadata.category == "advanced_imaging" + assert knee.metadata.rule_family == "extremity_mri_conservative_therapy" + assert knee.required_field_keys == [ + "conservative_therapy_weeks", + "symptom_duration_weeks", + "prior_imaging_result", + "mechanical_symptoms_documented", + ] + assert knee.provenance.rule_source_label == "Human-curated summary of knee MRI administrative documentation criteria" + assert knee.monitored_for_drift is False + + +def test_service_evaluates_new_cervical_demo_case(): + service = ReadinessService() + request = service.get_demo_case_request("MRI-CERV-01-ready") + + evaluation = service.evaluate(request) + + assert evaluation.overall_status == "READY" + assert evaluation.supported_procedure.procedure_code == "MRI_CERVICAL" + assert evaluation.supported_procedure.metadata.rule_family == "spine_mri_conservative_therapy" + + +def test_service_evaluates_new_knee_demo_case(): + service = ReadinessService() + request = service.get_demo_case_request("MRI-KNEE-01-ready") + + evaluation = service.evaluate(request) + + assert evaluation.overall_status == "READY" + assert evaluation.supported_procedure.procedure_code == "MRI_KNEE" + assert evaluation.supported_procedure.metadata.rule_family == "extremity_mri_conservative_therapy" + + +def test_service_returns_cannot_determine_when_knee_mechanical_symptoms_are_missing(): + service = ReadinessService() + request = service.get_demo_case_request("MRI-KNEE-03-cannot-determine") + + evaluation = service.evaluate(request) + + assert evaluation.overall_status == "CANNOT_DETERMINE" + assert any(blocker.key == "mechanical_symptoms_documented" for blocker in evaluation.blockers.not_documented) + + +def test_service_rejects_unsupported_scope(): + service = ReadinessService() + request = PARequest( + payer="Aetna", + procedure_code="NOT_A_REAL_PROC", + dx_codes=["Z00.00"], + site_of_care="outpatient", + specialty="Primary Care", + note_text="Synthetic note text.", + ) + + try: + service.evaluate(request) + except UnsupportedScopeError as exc: + assert "Unsupported request scope" in str(exc) + else: # pragma: no cover - assertion guard + raise AssertionError("Unsupported procedure should raise UnsupportedScopeError") + + +def test_service_warns_on_blank_note(): + service = ReadinessService() + request = PARequest( + payer="Aetna", + procedure_code="MRI_LUMBAR", + dx_codes=[], + site_of_care="outpatient", + specialty="unknown", + note_text="", + ) + + warnings = service.validate_request(request) + + assert any("No note text provided" in warning for warning in warnings) + assert any("No diagnosis codes supplied" in warning for warning in warnings) + + +def test_service_rejects_unsupported_site_of_care(): + service = ReadinessService() + request = PARequest( + payer="Aetna", + procedure_code="MRI_LUMBAR", + dx_codes=["M54.5"], + site_of_care="telehealth", + specialty="Primary Care", + note_text="Low back pain x 8 weeks. PT x 8 weeks. No prior imaging. Denies weakness.", + ) + + try: + service.validate_request(request) + except UnsupportedScopeError as exc: + assert "Unsupported site_of_care" in str(exc) + else: # pragma: no cover - assertion guard + raise AssertionError("Unsupported site of care should raise UnsupportedScopeError") + + +def test_drift_status_exposes_source_metadata_and_hash(): + service = ReadinessService() + + report = service.get_drift_status() + + assert report.sources + assert report.any_review_required is True + assert report.stale_source_count >= 0 + first = report.sources[0] + assert first.source_name == "Aetna CPB 0157" + assert first.latest_hash is not None + assert first.rule_source_label + assert first.freshness_status in {"CURRENT", "STALE", "UNKNOWN"} + assert first.days_since_last_checked is not None + assert first.latest_snapshot_path == "policy_snapshots/aetna_mri_lumbar/latest.json" + assert first.review_reason + + +def test_service_status_includes_rulebook_metadata(): + service = ReadinessService() + + status = service.get_status() + + assert status.rules_version == "0.5" + assert status.rulebook_active_release_id == "2026-04-09-active-v0.5" diff --git a/test/test_streamlit_app.py b/test/test_streamlit_app.py new file mode 100644 index 0000000..6b0fc5b --- /dev/null +++ b/test/test_streamlit_app.py @@ -0,0 +1,24 @@ +from streamlit.testing.v1 import AppTest + + +def test_streamlit_app_loads_without_exceptions(): + at = AppTest.from_file("app.py") + + at.run() + + assert not at.exception + assert any(metric.label == "Supported procedures" and metric.value == "4" for metric in at.metric) + assert any(subheader.value == "Supported Procedure Registry" for subheader in at.subheader) + + +def test_streamlit_featured_case_load_produces_results(): + at = AppTest.from_file("app.py") + + at.run() + if at.checkbox: + at.checkbox[0].check().run() + load_buttons = [button for button in at.button if button.label == "Load Demo Case"] + load_buttons[-1].click().run() + + assert not at.exception + assert any(metric.label == "Overall status" for metric in at.metric) From f709d5dd9d59855ff3eb5c3af98b3a9b6bafe8c0 Mon Sep 17 00:00:00 2001 From: NickLeko Date: Thu, 9 Apr 2026 09:33:09 -0700 Subject: [PATCH 2/2] Finalize deterministic prior auth copilot overhaul and public repo cleanup --- .gitignore | 15 ++ FINAL_FREEZE_AUDIT.md | 75 ------ IMPLEMENTATION_PLAN.md | 113 --------- MARATHON_PASS_PLAN.md | 101 -------- PUSH_SUMMARY.md | 73 ------ README.md | 4 + RELEASE_NOTES_LAST_3_SPRINTS.md | 144 ----------- SECOND_PASS_PLAN.md | 98 -------- WORKLOG.md | 319 ------------------------- docs/LOCAL_WORKFLOW.md | 30 --- docs/REFUSAL_IS_A_FEATURE.md | 97 -------- docs/notes/CASE_STUDY.md | 46 ---- docs/notes/DATA_AVAILABILITY_MATRIX.md | 20 -- docs/notes/INTEGRATION_ONEPAGER.md | 22 -- docs/notes/METRICS_DASHBOARD.md | 27 --- docs/notes/PA_COPILOT_CASE_PACKET.md | 29 --- docs/notes/PRD.md | 53 ---- docs/notes/STAKEHOLDER_NARRATIVE.md | 41 ---- inputs/test_plan.md | 40 ---- 19 files changed, 19 insertions(+), 1328 deletions(-) delete mode 100644 FINAL_FREEZE_AUDIT.md delete mode 100644 IMPLEMENTATION_PLAN.md delete mode 100644 MARATHON_PASS_PLAN.md delete mode 100644 PUSH_SUMMARY.md delete mode 100644 RELEASE_NOTES_LAST_3_SPRINTS.md delete mode 100644 SECOND_PASS_PLAN.md delete mode 100644 WORKLOG.md delete mode 100644 docs/LOCAL_WORKFLOW.md delete mode 100644 docs/REFUSAL_IS_A_FEATURE.md delete mode 100644 docs/notes/CASE_STUDY.md delete mode 100644 docs/notes/DATA_AVAILABILITY_MATRIX.md delete mode 100644 docs/notes/INTEGRATION_ONEPAGER.md delete mode 100644 docs/notes/METRICS_DASHBOARD.md delete mode 100644 docs/notes/PA_COPILOT_CASE_PACKET.md delete mode 100644 docs/notes/PRD.md delete mode 100644 docs/notes/STAKEHOLDER_NARRATIVE.md delete mode 100644 inputs/test_plan.md diff --git a/.gitignore b/.gitignore index 160f160..de798e6 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,18 @@ __pycache__/ policy_snapshots/**/__pycache__/ policy_snapshots/**/*.pyc policy_snapshots/**/.DS_Store + +# Internal planning / audit residue +WORKLOG.md +IMPLEMENTATION_PLAN.md +SECOND_PASS_PLAN.md +MARATHON_PASS_PLAN.md +FINAL_FREEZE_AUDIT.md +PUSH_SUMMARY.md +RELEASE_NOTES_LAST_3_SPRINTS.md +PUBLIC_DOCS_AUDIT.md +PUBLIC_REPO_CLEANUP_SUMMARY.md +inputs/test_plan.md +docs/notes/ +docs/LOCAL_WORKFLOW.md +docs/REFUSAL_IS_A_FEATURE.md diff --git a/FINAL_FREEZE_AUDIT.md b/FINAL_FREEZE_AUDIT.md deleted file mode 100644 index 8c221fc..0000000 --- a/FINAL_FREEZE_AUDIT.md +++ /dev/null @@ -1,75 +0,0 @@ -# FINAL FREEZE AUDIT - -## Freeze Recommendation - -Yes. - -## Issues Found - -1. Governance outputs exposed machine-specific absolute paths. -2. `cli.py list-demo-cases` used a single `expected=` label for both true overall-status expectations and coarse fixture labels. -3. Streamlit featured-case wording was ambiguous about whether a value represented a real overall status or only a fixture label. -4. The synthetic eval suite explanation in the UI and docs did not clearly distinguish coarse fixture checks from exact golden acceptance snapshots. -5. Several docs still used machine-specific absolute repo links. -6. Small low-value complexity remained: - - empty dead file `engine/score.py` - - duplicated featured-demo filtering in `app.py` - - repeated service calls in artifact generation -7. Drift reporting exposed a governance-summary inconsistency: - - stale monitored sources were surfaced - - but `any_review_required` still remained `false` - - Streamlit gating and governance JSON were therefore slightly out of sync - -## Fixes Applied In This Final Pass - -- Normalized governance output paths to repo-relative values in drift and rulebook surfaces. -- Clarified CLI demo-case output by splitting `expected_status` from `fixture_label`. -- Clarified Streamlit featured-case wording and synthetic-eval explanation. -- Removed `engine/score.py`. -- Reused shared featured-demo helpers in the app and reduced duplicate work in artifact generation. -- Converted remaining machine-specific doc links to relative links. -- Added regression coverage for portable governance paths and clearer CLI output labels. -- Aligned drift summary behavior so stale or missing monitored baselines now set `any_review_required=true`. -- Updated the Streamlit smoke test to follow the real governance-acknowledgement path before running a featured case. - -## Risks That Remain - -- Scope is still intentionally narrow: one payer, four procedures, and one monitored policy source. -- Drift monitoring remains governance-only and currently reflects a stale monitored source by design. -- Extraction remains regex-based and conservative; unsupported language stays unsupported. -- Rulebook promotion remains manual and human-reviewed, not automated. -- The synthetic eval suite is intentionally coarse and should be explained alongside the stricter golden acceptance tests. - -## Exact Verification Commands - -```bash -ruff check . -pytest -q -make acceptance -make verify -python3 cli.py list-demo-cases -python3 cli.py list-procedures -python3 cli.py drift-status --json -python3 cli.py rulebook-status --json -python3 cli.py evaluate --demo-case MRI-KNEE-01-ready -python3 -m scripts.generate_artifacts -python3 -m scripts.generate_golden_outputs -``` - -## Verification Results - -- `ruff check .` -> passed -- `pytest -q` -> `59 passed in 1.31s` -- `make acceptance` -> `2 passed in 0.19s` -- `make verify` -> passed -- `python3 cli.py list-demo-cases` -> passed -- `python3 cli.py list-procedures` -> passed -- `python3 cli.py drift-status --json` -> passed -- `python3 cli.py rulebook-status --json` -> passed -- `python3 cli.py evaluate --demo-case MRI-KNEE-01-ready` -> passed -- `python3 -m scripts.generate_artifacts` -> passed -- `python3 -m scripts.generate_golden_outputs` -> passed - -## Freeze Readiness Rationale - -The repo is now freeze-ready. The remaining limitations are intentional scope limits, not unfinished engineering. The final pass removed the last portability problems, clarified reviewer-facing wording, trimmed low-value complexity, and aligned the governance summary with the actual stale-source state. README, docs, app behavior, CLI behavior, generated artifacts, golden outputs, and tests now tell one coherent story: deterministic administrative readiness review, synthetic-only data, narrow auditable scope, governance-only drift monitoring, and manual rule promotion. diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md deleted file mode 100644 index 50c3a6f..0000000 --- a/IMPLEMENTATION_PLAN.md +++ /dev/null @@ -1,113 +0,0 @@ -# IMPLEMENTATION PLAN - -## Current State Summary - -The repo already has a credible deterministic core: - -- deterministic extraction in `engine/extract.py` -- deterministic requirement evaluation in `engine/evaluate.py` -- write-only administrative letter drafting in `engine/letter_draft.py` -- versioned payer rules in `rules/payer_rules.yaml` -- partial policy drift monitoring in `engine/policy_monitor.py` -- a Streamlit demo surface in `app.py` -- a small but solid pytest suite - -The repo is compact, explainable, and honest about scope. That should be preserved. - -## Biggest Weaknesses - -1. `app.py` currently owns too much orchestration logic. - - Evaluation assembly, audit construction, metrics, and UI presentation are tightly coupled. - - That makes API and CLI additions harder than they need to be. - -2. Typed contracts exist, but they are not broad enough. - - Major internal objects such as extracted facts, evidence spans, audit summaries, and supported-procedure metadata are still mostly loose dictionaries. - -3. There is no interview-friendly programmatic surface. - - The repo has a strong Streamlit demo, but no lightweight API or CLI for clean demos, curl examples, or reusable artifact generation. - -4. Error handling and unsupported-scope behavior are under-surfaced. - - The core logic is narrow by design, but unsupported procedures, malformed requests, and partial inputs should fail more explicitly and consistently. - -5. Docs are useful but fragmented. - - The repo needs a recruiter-proof top-level narrative, architecture doc, API doc, demo walkthrough, safety/scope doc, and interview talking points. - -6. Testing is good for the current core but not broad enough for a v2 artifact. - - There is little protection yet for API, CLI, structured outputs, unsupported-scope handling, or reproducible demo artifacts. - -## Highest-Leverage Improvements - -1. Add a small shared application layer. - - Create a reusable evaluation service that wraps deterministic extraction, rule lookup, scoring, audit trace creation, and output shaping. - - This should become the single source of truth used by Streamlit, API, CLI, and artifact generation. - -2. Expand typed models with Pydantic. - - Add schemas for intake requests, supported procedures, evidence spans, blockers, audit traces, status metadata, drift summaries, and exportable reports. - -3. Add central config and structured logging. - - Keep it lightweight: one config module, one logger setup, explicit paths, and deterministic defaults. - -4. Add a lightweight FastAPI layer. - - Expose health, supported procedures, readiness evaluation, demo case listing, and drift status. - - Keep the API honest: synthetic-only, deterministic, no persistence, no autonomous action. - -5. Add a simple CLI. - - Support listing procedures, listing demo cases, evaluating a demo case, exporting a report, validating synthetic input, and checking drift artifacts. - -6. Upgrade the Streamlit demo without changing the product identity. - - Keep the current strengths: blockers, evidence mapping, auditability, and refusal-first behavior. - - Improve the flow, labels, operator framing, and scope/limitations presentation. - -7. Strengthen reusable synthetic fixtures and artifacts. - - Add clearer demo case helpers, stable sample outputs, and a reproducible artifact-generation script. - -8. Expand tests and CI only where signal is high. - - Add regression tests for the shared service layer, API, CLI, unsupported scope, artifact generation, and key deterministic outputs. - -## What Will Not Be Changed - -- The repo will remain deterministic-first. -- No LLM-first architecture will be introduced. -- The core readiness semantics will stay frozen: - - `READY` - - `NOT_READY` - - `CANNOT_DETERMINE` -- The repo will remain synthetic-only. -- No real payer integrations, no EHR connectivity, no claims adjudication, no medical-necessity review, no approval prediction, and no autonomous action will be added. -- No database will be added. -- The existing extraction and evaluation logic will not be rewritten unless a concrete defect appears during refactoring. -- Policy drift monitoring will remain governance-only and will not auto-edit rules. - -## Smallest Structural Change With Highest ROI - -The smallest high-ROI structural change is: - -1. introduce a shared, typed evaluation service -2. route Streamlit through that service -3. expose the same service through FastAPI and a CLI - -That yields cleaner architecture, stronger testability, better docs, and a much stronger interview story without replacing the deterministic core. - -## Planned Execution Order - -1. Audit and baseline verification -2. Add worklog and implementation plan -3. Add shared models, config, logging, and evaluation orchestration -4. Refactor Streamlit to consume shared outputs instead of building them inline -5. Add FastAPI endpoints with typed request/response schemas -6. Add CLI commands for core demo workflows -7. Add reusable demo fixture helpers and artifact generation -8. Expand tests for service, API, CLI, unsupported scope, and artifacts -9. Overhaul docs and interview-facing materials -10. Tighten CI and local developer commands -11. Final verification pass, simplification pass, and worklog closeout - -## Scope Discipline - -If a candidate improvement starts forcing broad rewrites across the repo, the narrower fallback is: - -- keep the deterministic engine intact -- add adapters around it -- document limitations honestly - -That is the bar for every change in this overnight pass. diff --git a/MARATHON_PASS_PLAN.md b/MARATHON_PASS_PLAN.md deleted file mode 100644 index 581fee1..0000000 --- a/MARATHON_PASS_PLAN.md +++ /dev/null @@ -1,101 +0,0 @@ -# MARATHON PASS PLAN - -## Current Starting Point - -The repo already has a solid deterministic backbone and shared product surface: - -- deterministic extraction, evaluation, blocker detection, and letter drafting -- FastAPI, CLI, and Streamlit surfaces over a shared service layer -- typed schemas, central config, and structured logging -- synthetic demo cases, reproducible artifacts, and growing regression coverage -- provenance metadata and a governance-only drift surface -- three supported procedures: - - `MRI_LUMBAR` - - `MRI_CERVICAL` - - `CPAP_DEVICE` - -The repo does not need a redesign. The remaining gains are mostly product depth, governance clarity, and behavior lock-in. - -## Chosen Work Items - -### 1. Add one narrow non-spine deterministic pathway if it stays defensible - -Target candidate: -- `MRI_KNEE` - -Acceptance bar: -- no fuzzy medical-necessity logic -- no broad orthopedic abstraction -- one additional extractor field at most -- clear documented blocker and cannot-determine behavior -- synthetic fixtures, tests, artifacts, and docs all updated - -Fallback: -- if the rule contract becomes vague, skip the new procedure and deepen scenario coverage for the existing three procedures instead - -### 2. Introduce a lightweight versioned rulebook and promotion workflow - -Planned shape: -- a small governance artifact for `draft`, `reviewed`, and `active` -- validation and diff utilities -- rulebook version metadata surfaced in outputs where useful -- explicit human-review promotion steps in docs - -### 3. Deepen offline drift reporting instead of widening to weak new sources - -Planned direction: -- richer source metadata and stale-review visibility -- markdown and JSON drift report artifacts -- comparison of monitored snapshots against active rule assumptions when practical - -### 4. Add golden outputs and acceptance checks - -Planned coverage: -- representative ready, not-ready, and cannot-determine cases -- rulebook and drift artifact integrity -- cheap deterministic acceptance command wired into local workflow - -### 5. Tighten local ergonomics and recruiter-facing docs - -Planned direction: -- one-command high-signal verify and acceptance flows -- refreshed docs tied to the new procedure, rulebook, and governance story -- sharper interview narrative for the v1, v2, v3 progression - -## Skipped Or Deprioritized Work Items - -- no LLM layer -- no database -- no auth -- no Docker or deployment stack -- no broad second wave of procedures unless the first new one is clearly rigorous -- no live payer scraping requirement -- no automatic rule promotion or rule mutation from drift monitoring -- no large UI redesign that mainly changes aesthetics without improving clarity - -## Dependency Order - -1. Confirm current repo state and add this plan and worklog section -2. Implement the non-spine procedure only if the rule contract survives review -3. Build the rulebook/promotion workflow around the updated rule set -4. Improve drift artifacts and governance reporting -5. Add golden outputs and acceptance-style tests -6. Tighten ergonomics, docs, and interview materials -7. Run Ruff, full tests, artifact generation, smoke checks, and a simplification pass - -## Stop Criteria - -Stop when one of these becomes true: - -- the backlog above is substantially complete -- the next remaining work is mostly cosmetic or platform theater -- the next remaining work would widen claims faster than it improves rigor -- the repo reaches a point where additional changes add more explanation burden than credibility - -## Decision Rules For The Marathon Pass - -- Prefer one rigorous non-spine procedure over multiple shallow ones. -- Prefer governance depth over procedure count once product scope becomes harder to defend. -- Prefer offline, inspectable artifacts over network-dependent demos. -- Prefer regression lock-in over additional optional features. -- If an idea feels impressive but hard to explain in two minutes, skip it. diff --git a/PUSH_SUMMARY.md b/PUSH_SUMMARY.md deleted file mode 100644 index c18896a..0000000 --- a/PUSH_SUMMARY.md +++ /dev/null @@ -1,73 +0,0 @@ -# PUSH SUMMARY - -## Ready To Commit - -This change set is the full three-sprint upgrade plus the final freeze-polish pass. It is ready to commit as a single maturity upgrade if the final verification commands remain green. - -## File Change Summary By Area - -### Architecture / Core Engine - -- Shared typed service orchestration under `engine/service.py` -- Central config and structured logging under `engine/config.py` and `engine/logging_utils.py` -- Expanded schemas, rendering, demo-case helpers, acceptance normalization, and rulebook utilities -- Narrow deterministic extractor extension and additional supported procedures without changing the repo’s core readiness semantics - -### API / CLI / App Surfaces - -- New `api.py` FastAPI surface -- New `cli.py` demo and governance workflows -- Streamlit rebuild in `app.py` around the shared service layer with clearer operator-facing outputs - -### Governance / Rulebook / Drift - -- Structured provenance and supported-procedure metadata -- Governance-only drift reporting and markdown/JSON artifacts -- Reviewed vs active rulebook snapshots, manifest, diffing, and validation - -### Test Suite - -- Expanded unit, API, CLI, service, artifact, Streamlit sanity, rulebook, and acceptance coverage -- Golden snapshots under `test/golden/` - -### Artifacts / Scripts - -- Reproducible artifact generation under `scripts/generate_artifacts.py` -- Golden-output generation under `scripts/generate_golden_outputs.py` -- Stable demo and governance artifacts under `docs/artifacts/` - -### Docs / Interview Materials - -- Rewritten README and docs set for architecture, API, testing, demo walkthrough, and safety/scope -- Added product overview, why-this-exists narrative, limitations, next steps, and interview talking points -- Added implementation, second-pass, marathon-pass, and freeze audit records for review traceability - -## Intentionally Removed - -- `engine/score.py` - -It was empty and provided no value. - -## Files That Deserve Extra Human Review Before Commit - -- `rules/payer_rules.yaml` -- `rules/provenance.yaml` -- `rules/policy_sources.yaml` -- `engine/service.py` -- `engine/rulebook.py` -- `app.py` -- `README.md` -- `FINAL_FREEZE_AUDIT.md` -- `RELEASE_NOTES_LAST_3_SPRINTS.md` - -## Recommended Commit Message Options - -1. `Finalize deterministic prior auth copilot maturity pass and freeze audit` -2. `Ship enterprise-style v3 upgrade with governance, artifacts, and freeze polish` -3. `Complete prior auth readiness copilot overhaul and freeze-ready cleanup` - -## Safe To Push? - -Yes. - -The final freeze verification suite is green, and the remaining risks are documented scope limitations rather than unstable behavior. diff --git a/README.md b/README.md index df23311..177a394 100644 --- a/README.md +++ b/README.md @@ -163,6 +163,10 @@ python3 -m scripts.generate_golden_outputs - [docs/demo_walkthrough.md](docs/demo_walkthrough.md) - [docs/testing.md](docs/testing.md) - [docs/safety_and_scope.md](docs/safety_and_scope.md) +- [EXTRACTION_CONTRACT.md](EXTRACTION_CONTRACT.md) +- [LETTER_DRAFTING_CONTRACT.md](LETTER_DRAFTING_CONTRACT.md) +- [MODEL_CARD.md](MODEL_CARD.md) +- [FAILURE_MODES.md](FAILURE_MODES.md) - [PRODUCT_OVERVIEW.md](PRODUCT_OVERVIEW.md) - [WHY_THIS_EXISTS.md](WHY_THIS_EXISTS.md) - [DESIGN_DECISIONS.md](DESIGN_DECISIONS.md) diff --git a/RELEASE_NOTES_LAST_3_SPRINTS.md b/RELEASE_NOTES_LAST_3_SPRINTS.md deleted file mode 100644 index db54d2d..0000000 --- a/RELEASE_NOTES_LAST_3_SPRINTS.md +++ /dev/null @@ -1,144 +0,0 @@ -# RELEASE NOTES: LAST 3 SPRINTS - -Sprint boundaries below are inferred from `WORKLOG.md` and the plan files. They are trustworthy summaries of the actual upgrade sequence, even though some milestones were recorded inside broader overnight passes. - -## 1. Sprint 1 Summary - -### What Changed - -- Reframed the repo around a shared deterministic service layer instead of having orchestration live mainly inside `app.py`. -- Added typed schemas, central config, and structured logging. -- Added FastAPI and CLI surfaces over the same deterministic workflow. -- Rebuilt the Streamlit app to consume shared outputs instead of assembling evaluation state inline. -- Added reproducible artifact generation and substantially expanded tests. -- Rewrote the README and added core docs for architecture, API, testing, demo walkthrough, safety/scope, design decisions, limitations, next steps, and interview use. -- Added lightweight Ruff and CI support. - -### Why It Mattered - -Before this sprint, the repo was a strong deterministic demo with a good engine but limited product shape. After this sprint, it became a compact internal-product artifact with clearer boundaries, typed interfaces, reusable surfaces, reproducible outputs, and an interview-defensible story. - -### Key Files - -- `engine/service.py` -- `engine/schemas.py` -- `engine/config.py` -- `engine/logging_utils.py` -- `engine/rendering.py` -- `api.py` -- `cli.py` -- `app.py` -- `scripts/generate_artifacts.py` -- `README.md` -- `docs/architecture.md` -- `docs/api.md` -- `docs/testing.md` -- `.github/workflows/ci.yml` - -## 2. Sprint 2 Summary - -### What Changed - -- Added a new supported deterministic procedure, `MRI_CERVICAL`. -- Strengthened the supported-procedure registry with category, rule family, summary, supported sites, and rule-update metadata. -- Strengthened provenance output with rule source label, source URL where available, last reviewed, last updated, and monitored-source linkage. -- Improved demo-case taxonomy and featured-case metadata. -- Added Streamlit AppTest sanity coverage and stronger regression tests around metadata integrity, contradictory evidence, unsupported site handling, and artifact generation. -- Tightened Makefile ergonomics and normalized the demo walkthrough path. - -### Why It Mattered - -This sprint added depth without changing the architecture. The repo became more credible because it could show more than one deterministic imaging pathway, expose clearer rule provenance, and verify that the UI and metadata surfaces stayed intact. - -### Key Files - -- `rules/payer_rules.yaml` -- `rules/provenance.yaml` -- `rules/policy_sources.yaml` -- `inputs/synthetic_cases.json` -- `engine/service.py` -- `engine/schemas.py` -- `engine/provenance.py` -- `engine/policy_monitor.py` -- `app.py` -- `test/test_streamlit_app.py` -- `test/test_artifact_generation.py` -- `docs/demo_walkthrough.md` - -## 3. Sprint 3 Summary - -### What Changed - -- Added a narrow non-spine deterministic procedure, `MRI_KNEE`, using one carefully scoped new extractor field. -- Introduced a versioned rulebook with reviewed and active snapshots plus manifest-driven stage metadata. -- Added rulebook validation, status, and diff utilities through the service, API, and CLI. -- Deepened drift reporting with freshness metadata, stale-source signaling, and markdown/JSON governance artifacts. -- Added golden acceptance snapshots, normalization logic, and an acceptance command. -- Expanded recruiter-facing docs with clearer product and governance narrative. - -### Why It Mattered - -This sprint turned the repo from a good multi-surface demo into a more mature governance artifact. It showed not just deterministic evaluation, but also how rules are versioned, reviewed, promoted manually, and monitored for drift without pretending to auto-govern production policy logic. - -### Key Files - -- `engine/extract.py` -- `engine/rulebook.py` -- `engine/acceptance.py` -- `rulebook/manifest.yaml` -- `api.py` -- `cli.py` -- `scripts/generate_golden_outputs.py` -- `docs/artifacts/drift_report.md` -- `docs/artifacts/rulebook_diff_reviewed_vs_active.md` -- `PRODUCT_OVERVIEW.md` -- `WHY_THIS_EXISTS.md` -- `INTERVIEW_TALKING_POINTS.md` - -## 4. Final Freeze-Polish Summary - -### What Changed In This Audit Pass - -- Fixed portability problems in governance outputs by replacing machine-specific paths with repo-relative paths. -- Aligned stale-monitoring behavior so governance outputs and Streamlit gating both treat stale or missing baselines as review-required. -- Clarified demo-case expectation wording in the CLI and Streamlit UI. -- Clarified the difference between coarse synthetic fixture checks and exact acceptance snapshots. -- Removed one dead file and trimmed a few low-value duplicate paths in the app and artifact generation flow. -- Replaced remaining machine-specific markdown links with relative links. -- Added the final freeze audit, push summary, and this release-notes summary. - -### Why It Mattered - -These were small changes, but they removed the last rough edges that could undermine recruiter trust or handoff quality. The repo now reads more cleanly, travels better across machines, and tells a more consistent story. - -### Key Files - -- `engine/service.py` -- `engine/rulebook.py` -- `cli.py` -- `app.py` -- `scripts/generate_artifacts.py` -- `README.md` -- `EXTRACTION_CONTRACT.md` -- `FINAL_FREEZE_AUDIT.md` -- `PUSH_SUMMARY.md` - -## 5. Overall Repo Before Vs After - -### Before The 3 Sprints - -- Deterministic extraction and evaluation existed. -- The primary product surface was Streamlit. -- Docs, artifacts, and tests were narrower. -- Governance, provenance, rulebook, and portable product surfaces were limited or absent. - -### After The 3 Sprints - -- One deterministic workflow powers Streamlit, FastAPI, CLI, artifacts, and acceptance checks. -- The repo has four supported procedures, including a narrow non-spine case. -- Outputs expose richer provenance, rulebook stage metadata, drift artifacts, and audit traces. -- The repo includes portable docs, reproducible artifacts, golden outputs, interview materials, and a clear governance story. - -## 6. Final Narrative For Commit / Push Review - -Across the last three sprints, this repo was upgraded from a strong deterministic demo into a compact, interview-defensible internal-product artifact. The core identity stayed fixed: deterministic administrative readiness review, synthetic-only data, narrow explainable scope, refusal-first behavior, and governance-only drift monitoring. The work focused on product surfaces, provenance, rule governance, artifacts, tests, and docs rather than broad platform expansion, which is why the repo now feels substantially more mature without becoming harder to explain. diff --git a/SECOND_PASS_PLAN.md b/SECOND_PASS_PLAN.md deleted file mode 100644 index d56196b..0000000 --- a/SECOND_PASS_PLAN.md +++ /dev/null @@ -1,98 +0,0 @@ -# SECOND PASS PLAN - -## What Already Exists - -The first pass already established the repo as a compact enterprise-style artifact: - -- shared typed service layer for deterministic orchestration -- central config and structured logging -- FastAPI and CLI surfaces -- rebuilt Streamlit app using shared outputs -- reproducible JSON artifact generation -- expanded tests across service, API, CLI, extraction, rules, provenance, and drift logic -- improved README, architecture, testing, safety, and interview docs -- lightweight Ruff + CI - -The core architecture is not the problem anymore. The next gains should come from depth. - -## What Is Still Weak - -1. Supported deterministic scope is still narrow. - - Only two procedures are currently supported. - - The repo can support one more carefully chosen deterministic pathway without architecture churn. - -2. Rule registry and provenance detail are still thinner than they could be. - - Supported procedures expose only basic fields today. - - Provenance metadata exists but is not consistently normalized or surfaced as a strong registry story. - -3. Drift/governance surfaces are still fairly minimal. - - There is one monitored source. - - The repo could do a better job surfacing monitored-source metadata and rule lineage without widening scope. - -4. Demo quality can still improve. - - Artifacts exist, but demo-case taxonomy and walkthrough clarity can be stronger. - - There is still no direct Streamlit sanity coverage. - -5. Regression safety has room to grow. - - The most important remaining gaps are malformed/partial input handling, contradictory evidence behavior, registry metadata integrity, and UI startup sanity. - -## What This Pass Will Improve - -1. Add one carefully chosen new deterministic procedure if it can reuse the current extraction rigor cleanly. - - If a second new procedure starts to feel vague, stop at one. - -2. Strengthen procedure registry and provenance metadata. - - Add richer procedure-level metadata to the rule registry. - - Normalize and surface provenance fields that matter in interviews: - - rule version - - rule source label - - last reviewed / last updated - - monitored-for-drift status - - rule family / category where useful - -3. Improve governance and drift visibility without changing behavior. - - Keep drift monitoring governance-only. - - Make monitored source status easier to inspect through outputs and docs. - -4. Improve demo quality and UI sanity coverage. - - Refresh artifacts after the new procedure/metadata land. - - Add lightweight Streamlit sanity coverage rather than heavy browser infrastructure unless a minimal browser path proves low-cost. - -5. Expand regression tests around the repo’s product identity. - - new procedure coverage - - ambiguous or contradictory evidence handling - - registry/provenance integrity - - artifact generation - - Streamlit startup/smoke sanity - -6. Tighten local ergonomics where useful. - - Add only lightweight improvements, such as small Makefile target additions and clearer run/verify commands. - -## What Will Intentionally Remain Unchanged - -- no LLM layer -- no database -- no auth -- no claims workflow platform -- no payer scraping -- no approval prediction -- no medical-necessity or clinical recommendation logic -- no autonomous action -- no rewrite of the shared service architecture unless a concrete defect appears - -## Execution Order - -1. Re-audit and record second-pass scope -2. Add one carefully chosen deterministic procedure and corresponding synthetic fixtures -3. Enrich rule registry, provenance, and drift metadata surfaces -4. Add Streamlit sanity coverage and refresh demo artifacts -5. Expand regression tests and tighten local ergonomics -6. Update README, docs, interview notes, and next steps -7. Run Ruff, tests, artifact generation, and a final simplification pass - -## Decision Rules For This Pass - -- Prefer one well-finished new procedure over two shallow ones. -- If metadata and testing improvements are higher ROI than procedure count, bias toward metadata and tests. -- If browser-style screenshot automation becomes dependency-heavy, fall back to stronger artifact generation and Streamlit sanity coverage. -- Every change must remain explainable by one person in an interview. diff --git a/WORKLOG.md b/WORKLOG.md deleted file mode 100644 index bd6bc49..0000000 --- a/WORKLOG.md +++ /dev/null @@ -1,319 +0,0 @@ -# WORKLOG - -## Final Freeze Polish - 2026-04-09 - Audit, cleanup, and verification - -- What changed: - - Performed a final freeze-candidate audit across README, docs, Streamlit, CLI, governance outputs, generated artifacts, golden outputs, and tests. - - Fixed remaining portability issues by replacing machine-specific markdown links with relative links. - - Cleaned up final reviewer-facing wording: - - split `expected_status` from `fixture_label` in CLI demo-case output - - clarified featured-case wording in Streamlit - - clarified the difference between coarse synthetic fixture checks and golden acceptance snapshots - - Removed low-value dead or duplicate complexity: - - deleted empty `engine/score.py` - - reused shared featured-demo helpers in `app.py` - - removed repeated service calls in artifact generation - - Normalized governance file-path outputs to repo-relative values. - - Fixed the final governance-summary inconsistency so stale or missing monitored baselines now set `any_review_required=true`, which keeps JSON, CLI, and Streamlit gating aligned. - - Added final review artifacts: - - `FINAL_FREEZE_AUDIT.md` - - `PUSH_SUMMARY.md` - - `RELEASE_NOTES_LAST_3_SPRINTS.md` -- Why it changed: - - The repo no longer needed more product surface area. It needed a final trust pass that removed portability issues, ambiguous wording, and small inconsistencies before freeze. -- Tests run: - - `ruff check .` -> passed - - `pytest -q` -> `59 passed in 1.31s` - - `make acceptance` -> passed - - `make verify` -> passed - - `python3 cli.py list-demo-cases` -> passed - - `python3 cli.py list-procedures` -> passed - - `python3 cli.py drift-status --json` -> passed - - `python3 cli.py rulebook-status --json` -> passed - - `python3 cli.py evaluate --demo-case MRI-KNEE-01-ready` -> passed - - `python3 -m scripts.generate_artifacts` -> passed - - `python3 -m scripts.generate_golden_outputs` -> passed -- Blockers encountered: - - A final drift-summary mismatch surfaced during verification: the repo correctly flagged a stale monitored source, but the top-level `any_review_required` flag remained `false`. I fixed the summary semantics and regenerated artifacts and goldens. - - The Streamlit AppTest needed one small update to acknowledge the governance gate before loading a featured case, which now mirrors the actual UI flow. -- Follow-up items: - - Freeze the repo. - - Human reviewer can now commit and push with the generated summary files as the handoff packet. - -## Marathon Pass - 2026-04-09 - Re-audit and execution plan - -- What changed: - - Re-audited the repo after the first two upgrade passes. - - Confirmed the current baseline already includes: - - shared deterministic service surfaces - - typed procedure/provenance metadata - - three supported procedures - - governance-only drift reporting - - artifact generation - - regression and Streamlit sanity coverage - - Identified the next highest-ROI backlog for the marathon pass: - - one defensible non-spine procedure if possible - - rulebook versioning and promotion workflow - - stronger offline drift artifacts - - golden outputs and acceptance checks - - tighter docs and local ergonomics - - Created `MARATHON_PASS_PLAN.md` to keep the pass scoped and sequential. -- Why it changed: - - The repo is past the point where broad architecture work adds value. The remaining meaningful gains are depth, governance clarity, and regression safety. -- Tests run: - - Baseline assumed from the end of the prior pass: - - `ruff check .` -> passed - - `pytest -q` -> `45 passed in 0.73s` -- Blockers encountered: - - None. -- Follow-up items: - - Add a non-spine deterministic pathway only if it stays narrow and defensible. - - Build a lightweight rulebook and promotion workflow without changing runtime behavior. - - Add golden outputs, acceptance checks, and richer drift artifacts. - -## Marathon Pass - 2026-04-09 - Non-spine procedure depth and rulebook governance - -- What changed: - - Added a new supported deterministic non-spine procedure: `MRI_KNEE`. - - Introduced one narrow new extractor field, `mechanical_symptoms_documented`, to support a defensible orthopedic MRI contract without changing the engine shape. - - Added knee rule metadata, provenance, synthetic fixtures, featured demo coverage, artifacts, and regression tests. - - Built a lightweight rulebook registry under `rulebook/` with: - - reviewed snapshot `v0.4` - - active snapshot `v0.5` - - manifest-based stage assignment - - Added rulebook validation and diff utilities through the shared service, CLI, and API. - - Surfaced rulebook metadata in service status output and evaluation audit traces. -- Why it changed: - - The repo still had room for one more high-confidence deterministic pathway. After that, governance maturity was the next highest-ROI improvement because it strengthens the interview story without widening product claims. -- Tests run: - - `python3 -m json.tool inputs/synthetic_cases.json >/dev/null` -> passed - - `pytest -q test/test_extract_contracts.py test/test_service.py test/test_config_contracts.py test/test_api.py test/test_cli.py test/test_artifact_generation.py test/test_streamlit_app.py` -> `40 passed in 0.69s` - - `pytest -q test/test_rulebook.py test/test_service.py test/test_api.py test/test_cli.py` -> `28 passed in 0.51s` - - `python3 cli.py evaluate --demo-case MRI-KNEE-01-ready` -> passed - - `python3 cli.py rulebook-status` -> passed - - `python3 cli.py rulebook-diff --from-release 2026-04-09-reviewed-v0.4 --to-release 2026-04-09-active-v0.5` -> passed -- Blockers encountered: - - The first mechanical-symptom extractor pass over-matched denials as positives because of an overly broad fallback token match. I removed the fallback and kept only contextual positive patterns. -- Follow-up items: - - Deepen drift artifacts and stale-review visibility. - - Add golden outputs and an acceptance-style harness. - - Refresh docs and demo surfaces to explain the new procedure and rulebook workflow cleanly. - -## Marathon Pass - 2026-04-09 - Drift visibility, acceptance harness, docs, and final verification - -- What changed: - - Enriched drift status with freshness metadata, stale-source counts, review reasons, and diff-path visibility. - - Added markdown governance artifacts: - - `docs/artifacts/drift_report.md` - - `docs/artifacts/rulebook_diff_reviewed_vs_active.md` - - Added a golden-output acceptance harness: - - `engine/acceptance.py` - - `scripts/generate_golden_outputs.py` - - `test/golden/...` - - `test/test_acceptance_snapshots.py` - - Added lightweight Makefile ergonomics: - - `make acceptance` - - `make goldens` - - fallback to `python3` when `.venv` does not exist - - Updated README, API/architecture/testing/safety/demo docs, interview narrative, product overview, and next steps for the third pass. - - Added `PRODUCT_OVERVIEW.md` and `WHY_THIS_EXISTS.md` to make the recruiter-facing story faster to absorb. -- Why it changed: - - After scope and governance were in place, the highest-ROI work was locking behavior down and making the repo’s product narrative easier to understand quickly. -- Tests run: - - `pytest -q test/test_rulebook.py test/test_service.py test/test_api.py test/test_artifact_generation.py` -> `22 passed in 0.59s` - - `pytest -q test/test_acceptance_snapshots.py test/test_streamlit_app.py test/test_cli.py` -> `11 passed in 0.95s` - - `pytest -q` -> `59 passed in 1.48s` - - `ruff check .` -> passed - - `python3 -m py_compile app.py api.py cli.py engine/*.py scripts/*.py` -> passed - - `python3 -m scripts.generate_artifacts` -> passed - - `python3 -m scripts.generate_golden_outputs` -> passed - - `make acceptance` -> passed - - `make verify` -> passed -- Blockers encountered: - - `make acceptance` initially assumed `.venv/bin/python` existed. I changed the Makefile to fall back to `python3` automatically. - - Ruff flagged a few final line-length and import-order issues before the last verification sweep; those were cleaned up directly. -- Follow-up items: - - Optional: add screenshot automation later only if it stays lightweight and reproducible. - - Optional: add a second monitored source only with a clean offline baseline. - -## Second Pass - 2026-04-09 - Re-audit and plan - -- What changed: - - Re-audited the current second-pass starting point across rules, shared service, API, CLI, Streamlit app, artifacts, docs, and test coverage. - - Confirmed the repo is currently green before making changes. - - Identified the highest-ROI gaps for this pass: - - only two supported procedures - - limited rule/provenance metadata surfaced to users - - drift reporting is still minimal - - no direct Streamlit sanity coverage - - demo-case taxonomy and regression fixtures can be stronger - - Created `SECOND_PASS_PLAN.md` to scope the next set of improvements. -- Why it changed: - - The repo no longer needs broad architectural work. This pass should deepen credibility and regression safety around the existing deterministic backbone. -- Tests run: - - `pytest -q` -> `34 passed in 0.20s` - - `ruff check .` -> passed -- Blockers encountered: - - None. -- Follow-up items: - - Add carefully scoped new deterministic procedure coverage. - - Enrich supported-procedure and provenance metadata surfaces. - - Add lightweight Streamlit sanity coverage and refreshed artifacts. - - Expand regression fixtures and update docs accordingly. - -## Second Pass - 2026-04-09 - Deterministic coverage and metadata depth - -- What changed: - - Added a new supported deterministic procedure: `MRI_CERVICAL`. - - Extended the rule registry with procedure metadata for all supported procedures: - - category - - rule family - - summary - - supported sites - - last rule update - - rule notes - - Extended provenance metadata with structured rule source fields: - - rule source label - - source URL where available - - last reviewed - - rule last updated - - monitored source linkage - - Added a stronger monitored-source label to `rules/policy_sources.yaml`. - - Wired enriched metadata through the shared service, supported-procedure outputs, evaluation outputs, CLI summaries, Streamlit views, and drift status surfaces. - - Added cervical MRI synthetic demo cases and improved featured-case taxonomy with scenario labels and tags. -- Why it changed: - - The repo needed more depth, not more architecture. A second spine MRI pathway plus richer registry/provenance output materially improves realism and interview defensibility while preserving the deterministic core. -- Tests run: - - `python3 -m json.tool inputs/synthetic_cases.json >/dev/null` -> passed - - `pytest -q` -> `45 passed in 0.95s` - - `python3 cli.py list-procedures` -> passed -- Blockers encountered: - - None. -- Follow-up items: - - Refresh artifacts to include the new cervical pathway and featured-case registry. - - Add Streamlit sanity coverage and finish doc updates. - -## Second Pass - 2026-04-09 - Demo quality, regression hardening, and final verification - -- What changed: - - Added artifact-generation coverage and new generated outputs: - - `MRI-CERV-01-ready.json` - - `featured_demo_cases.json` - - `status.json` - - Added Streamlit AppTest sanity coverage to ensure the app loads and produces results from featured demo input. - - Expanded tests for: - - invalid rule metadata - - provenance metadata presence - - contradictory red-flag evidence precedence - - cervical procedure registry behavior - - unsupported site-of-care handling - - enriched drift surface integrity - - artifact generation output - - Added lightweight Makefile ergonomics: - - `make verify` - - `make smoke-ui` - - `make evaluate-case CASE=...` - - Normalized the walkthrough doc path to `docs/demo_walkthrough.md`. - - Updated README, architecture/API/testing/demo/interview/next-steps docs for the second pass. -- Why it changed: - - This step improves the repo’s live-demo clarity, regression safety, and local usability without widening scope into new infrastructure. -- Tests run: - - `ruff check .` -> passed - - `pytest -q` -> `45 passed in 0.73s` - - `python3 cli.py evaluate --demo-case MRI-CERV-01-ready` -> passed - - `python3 -m scripts.generate_artifacts` -> passed - - `python3 -m py_compile app.py api.py cli.py engine/*.py scripts/*.py` -> passed -- Blockers encountered: - - Two long-line lint failures in the Streamlit metadata panel were cleaned up before final verification. -- Follow-up items: - - Optional: add one more non-spine deterministic pathway only if it meets the same rigor. - - Optional: add screenshot automation later if it can remain lightweight and reproducible. - -## Milestone 0 - Audit and baseline - -- What changed: - - Audited the current repo structure, core engine modules, Streamlit app flow, rules/config files, tests, and CI. - - Established the first-pass upgrade strategy: preserve deterministic evaluation logic and add a thin shared service layer for API, CLI, UI, and artifact generation. - - Created this worklog to record overnight progress. -- Why it changed: - - The repo already has a strong deterministic core. The highest-leverage improvement is not a rewrite; it is turning the existing logic into a cleaner, typed, reusable product surface that is easier to defend in interviews. -- Tests run: - - `pytest -q` -> `21 passed in 0.10s` -- Blockers encountered: - - None. -- Follow-up items: - - Write `IMPLEMENTATION_PLAN.md`. - - Extract evaluation orchestration out of `app.py`. - - Add central config, structured logging, API, CLI, and stronger tests without widening scope. - -## Milestone 1 - Shared service foundation - -- What changed: - - Expanded `engine/schemas.py` with typed models for evidence spans, requirement definitions, supported procedures, demo cases, blockers, metrics, audit traces, drift reports, and service status. - - Added `engine/config.py` for central path/config validation and `.env.example` for lightweight local overrides. - - Added `engine/logging_utils.py` for structured JSON logging. - - Added `engine/demo_cases.py`, `engine/service.py`, and `engine/rendering.py` so Streamlit, API, CLI, and artifacts can share one deterministic orchestration path. - - Updated `engine/evaluate.py` to preserve normalized evidence span references in requirement results without changing readiness semantics. - - Added initial `api.py` and `cli.py` shells on top of the shared service. -- Why it changed: - - The repo’s main weakness was orchestration living inside `app.py`. This step creates a narrow enterprise-style application layer without rewriting the deterministic extraction or rule engine. -- Tests run: - - `pytest -q` -> `21 passed in 0.06s` -- Blockers encountered: - - None. -- Follow-up items: - - Route Streamlit through the shared service. - - Finish CLI/API behavior and add tests. - - Add reproducible artifact generation and updated docs. - -## Milestone 2 - Product surfaces and regression coverage - -- What changed: - - Rebuilt `app.py` as a thinner operator-style Streamlit UI that consumes the shared service instead of assembling evaluation state inline. - - Added a FastAPI surface in `api.py` with typed endpoints for health/status, supported procedures, demo cases, readiness evaluation, and drift status. - - Added `cli.py` for local demo workflows: status, procedure listing, demo case listing, evaluation, validation, export, and drift inspection. - - Added `scripts/generate_artifacts.py` for reproducible JSON artifacts under `docs/artifacts`. - - Expanded test coverage with new service, API, and CLI tests. - - Added minimal dependencies and Makefile targets required to run the new surfaces and lint checks. -- Why it changed: - - These changes convert the repo from a single-demo surface into a compact internal-product artifact with reusable programmatic interfaces, reproducible exports, and stronger interview defensibility. -- Tests run: - - `pytest -q` -> `34 passed in 0.19s` -- Blockers encountered: - - FastAPI was not installed locally, so I added the minimal dependencies required for the API, CLI tests, and linting. -- Follow-up items: - - Generate stable artifacts. - - Overhaul recruiter-facing docs and CI. - - Run full lint and final verification. - -## Milestone 3 - Docs, artifacts, CI, and final verification - -- What changed: - - Replaced the top-level README with a tighter product narrative, setup instructions, API/CLI examples, artifact links, and a docs index. - - Added the requested recruiter-facing docs: - - `docs/architecture.md` - - `docs/api.md` - - `docs/demo_walkthrough.md` - - `docs/testing.md` - - `docs/safety_and_scope.md` - - `DESIGN_DECISIONS.md` - - `LIMITATIONS.md` - - `NEXT_STEPS.md` - - `INTERVIEW_TALKING_POINTS.md` - - Added `pyproject.toml` for lightweight Ruff and pytest config. - - Updated GitHub Actions CI to run lint, tests, and artifact generation. - - Generated stable JSON artifacts under `docs/artifacts/`. - - Added `test/conftest.py` to make repo-root imports stable under pytest. - - Tuned the default log level to `WARNING` so CLI and artifact commands stay clean by default. -- Why it changed: - - This step turns the repo into a more defensible overnight artifact: easier to demo, easier to explain, easier to run locally, and more honest about scope and limitations. -- Tests run: - - `ruff check .` -> passed - - `pytest -q` -> `34 passed in 0.16s` - - `python3 cli.py evaluate --demo-case MRI-01-complete` -> passed - - `python3 -m scripts.generate_artifacts` -> passed -- Blockers encountered: - - `pytest` lost repo-root imports after the config pass, so I fixed that with `test/conftest.py` rather than relying on external environment behavior. -- Follow-up items: - - Optional: capture an updated Streamlit screenshot if the README needs a fresh visual. - - Optional: add one or two more supported procedures using the same deterministic discipline. diff --git a/docs/LOCAL_WORKFLOW.md b/docs/LOCAL_WORKFLOW.md deleted file mode 100644 index c0fbc1c..0000000 --- a/docs/LOCAL_WORKFLOW.md +++ /dev/null @@ -1,30 +0,0 @@ -# Local Workflow - -Current repo status: -- deterministic extraction, evaluation, and letter drafting -- no LLM implementation - -## Canonical Setup - -```bash -python3.12 -m venv .venv -source .venv/bin/activate -python -m pip install --upgrade pip -python -m pip install -r requirements.txt -``` - -## Run - -```bash -streamlit run app.py -``` - -## Test - -```bash -pytest -q -``` - -`pytest -q` is the CI path. It covers deterministic extraction, evaluation semantics, drafting constraints, rule loading, policy-monitor helpers, and a regression check over the bundled synthetic eval cases. - -The Streamlit UI separately surfaces the same bundled synthetic eval cases as a local demo gate. diff --git a/docs/REFUSAL_IS_A_FEATURE.md b/docs/REFUSAL_IS_A_FEATURE.md deleted file mode 100644 index 6ec612a..0000000 --- a/docs/REFUSAL_IS_A_FEATURE.md +++ /dev/null @@ -1,97 +0,0 @@ -# Refusal Is a Feature (Safety Narrative) - -Project: Prior Authorization Readiness Copilot -Purpose: Explain why “CANNOT_DETERMINE” is the correct, safer outcome when documentation is missing. - -Current repo status: -- deterministic implementation -- no LLM implementation - ---- - -## 1) The Core Design Choice - -This system is **administrative decision support**, not clinical judgment and not approval prediction. - -It answers: -- “Is the request administratively ready based on documented criteria?” - -It does NOT answer: -- “Is the procedure appropriate?” -- “Will this be approved?” -- “What should the clinician do?” - ---- - -## 2) Why Missing Documentation Forces Refusal - -Clinical notes are often incomplete or inconsistent. - -If a required criterion is **not documented**, the safest output is: -- **CANNOT_DETERMINE** - -Because any alternative requires guessing: -- inferring facts the note does not state -- “helpfully” assuming negatives (dangerous) -- silently filling gaps (non-auditable) - -This system refuses because it is designed to be **auditable and defensible**. - ---- - -## 3) Current Invariants (Safety Rails) - -The system enforces: - -- Any NOT_DOCUMENTED ⇒ overall must be CANNOT_DETERMINE -- Any NOT_MET (and no NOT_DOCUMENTED) ⇒ overall must be NOT_READY -- No blockers ⇒ overall must be READY - -Invariant violations are surfaced explicitly (UI + audit). - ---- - -## 4) Example: Why CANNOT_DETERMINE Is Correct - -Scenario: -- Payer requires documented AHI for OSA pathway -- Note says: “AHI not stated.” - -Outcome: -- Requirement AHI status = NOT_DOCUMENTED -- Overall status = CANNOT_DETERMINE -- Letter includes a Missing Documentation checklist item: - - “Provide numeric AHI value (e.g., ‘AHI 22’).” - -Key point: -- The system does not assume the patient fails criteria. -- The system does not invent AHI values. -- The system makes the missing documentation explicit. - ---- - -## 5) Why This Improves Trust - -This refusal behavior: -- prevents silent hallucinations -- enables reproducible outcomes -- supports payer-facing defensibility -- makes it easy for users to correct the record (checklist) - -Refusal is a feature because it prioritizes: -- accuracy over convenience -- auditability over “smooth output” -- explicit uncertainty over fabricated certainty - ---- - -## 6) Practical Boundary (Model Usage) - -- Extraction + evaluation: deterministic (rules-first) -- Letter generation: write-only, downstream of deterministic outputs -- The letter generator cannot change statuses or infer facts - -This preserves a clear separation between: -- documentation assessment -- administrative readiness -- narrative formatting diff --git a/docs/notes/CASE_STUDY.md b/docs/notes/CASE_STUDY.md deleted file mode 100644 index fc44a6c..0000000 --- a/docs/notes/CASE_STUDY.md +++ /dev/null @@ -1,46 +0,0 @@ -# Case Study -## Prior Authorization Readiness Copilot - -## Summary - -This repo demonstrates a narrow approach to prior authorization readiness review: - -- deterministic extraction -- rules-first evaluation -- explicit refusal when documentation is missing -- deterministic write-only drafting -- auditability over automation - -The current repo does not use an LLM. - -## Problem Framing - -The repo focuses on administrative readiness, not clinical appropriateness and not approval prediction. - -## Design Choices - -- Missing documentation stays missing -- `CANNOT_DETERMINE` is a first-class outcome -- Requirement logic is explicit and versioned -- Drafting is downstream of evaluated results -- Governance artifacts are part of the repo, not hidden elsewhere - -## What The Repo Proves - -- The overall status mapping is explicit and testable -- The extractor returns evidence spans when it captures supporting text -- Drafting stays within the evaluated result set -- Policy monitoring artifacts can be inspected without changing rules automatically - -## Limits - -- Synthetic cases only -- Limited payer and procedure coverage -- Regex-based extraction with narrow phrase support -- No production integration - -## Possible Extensions - -- Expand coverage with provenance updates and regression tests -- Add production integration layers outside this repo -- Evaluate optional LLM-assisted text formatting only if the scope changes and the contracts are updated diff --git a/docs/notes/DATA_AVAILABILITY_MATRIX.md b/docs/notes/DATA_AVAILABILITY_MATRIX.md deleted file mode 100644 index eddb970..0000000 --- a/docs/notes/DATA_AVAILABILITY_MATRIX.md +++ /dev/null @@ -1,20 +0,0 @@ -# Possible Production Data Notes - -This file is not a description of the current repo. It is a short note on data that a production prior authorization workflow might need. - -## Current Repo - -- Deterministic implementation -- No LLM implementation -- Inputs are limited to a few structured fields plus synthetic note text - -## If Extended Beyond This Repo - -Possible production inputs could include: - -- insurance coverage -- diagnosis and procedure coding -- supporting reports or prior treatment history -- payer-specific structured requirements - -Any such extension would need new contracts, new tests, and updated governance documentation. diff --git a/docs/notes/INTEGRATION_ONEPAGER.md b/docs/notes/INTEGRATION_ONEPAGER.md deleted file mode 100644 index a283548..0000000 --- a/docs/notes/INTEGRATION_ONEPAGER.md +++ /dev/null @@ -1,22 +0,0 @@ -# Possible Production Integration Notes - -This repo does not implement EHR or payer integration. This note exists only to mark likely integration boundaries if the scope ever expands. - -## Current Repo - -- Deterministic implementation -- No LLM implementation -- Local demo only -- Synthetic inputs only - -## If Extended Beyond This Repo - -Likely production concerns would include: - -- scoped data access -- PHI minimization -- explicit human review before write-back -- immutable audit logging -- environment separation - -Any production integration would be a separate workstream from the current repo. diff --git a/docs/notes/METRICS_DASHBOARD.md b/docs/notes/METRICS_DASHBOARD.md deleted file mode 100644 index d3b107a..0000000 --- a/docs/notes/METRICS_DASHBOARD.md +++ /dev/null @@ -1,27 +0,0 @@ -# Possible Operational Metrics - -This file describes metrics that could matter in a production deployment. None of these metrics are collected by the current repo. - -## Current Repo - -- Deterministic implementation -- No LLM implementation -- Synthetic cases only - -## If Adapted Beyond This Repo - -Possible metric groups: - -- Extraction correctness on audited samples -- Separation accuracy between `NOT_READY` and `CANNOT_DETERMINE` -- Evidence-span coverage -- Policy-drift review turnaround -- Audit artifact completeness - -## What This File Does Not Claim - -- No deployed volume -- No time-saved measurement -- No ROI measurement -- No denial-rate improvement -- No production dashboard diff --git a/docs/notes/PA_COPILOT_CASE_PACKET.md b/docs/notes/PA_COPILOT_CASE_PACKET.md deleted file mode 100644 index 2d506a8..0000000 --- a/docs/notes/PA_COPILOT_CASE_PACKET.md +++ /dev/null @@ -1,29 +0,0 @@ -# Interview Packet -## Prior Authorization Readiness Copilot - -## What This Repo Is - -- Deterministic prior authorization readiness review -- Deterministic write-only drafting -- Synthetic demo inputs -- No LLM implementation - -## What It Demonstrates - -- Scope discipline in a regulated healthcare problem -- Explicit missingness handling -- Rules-first evaluation -- Auditability and governance boundaries - -## What It Does Not Claim - -- Production deployment -- Real user research results -- Operational ROI -- EHR integration - -## Possible Extensions - -- Broader rule coverage -- Production integration layers -- Optional LLM-assisted text formatting only behind separate contracts and tests diff --git a/docs/notes/PRD.md b/docs/notes/PRD.md deleted file mode 100644 index 805b9ca..0000000 --- a/docs/notes/PRD.md +++ /dev/null @@ -1,53 +0,0 @@ -# Product Brief -## Prior Authorization Readiness Copilot - -## Current Repo - -This repo implements a deterministic prior authorization readiness review for synthetic cases. It does not use an LLM. - -## Product Question - -Is this request administratively ready to submit as documented, and if not, which required elements are missing or below threshold? - -## In Scope - -- Deterministic extraction from note text -- Rules-first evaluation -- Explicit missingness handling -- Deterministic letter drafting downstream of evaluation -- Audit output -- Local demo UI - -## Out of Scope - -- Clinical decision support -- Approval prediction -- Autonomous submission or appeals -- Runtime policy interpretation -- Production deployment claims - -## Decision Semantics - -- `READY`: all required criteria are documented and met -- `NOT_READY`: all required criteria are documented, but one or more do not meet threshold -- `CANNOT_DETERMINE`: one or more required criteria are not documented - -## Constraints - -- Deterministic behavior only -- No LLM in the current repo -- Synthetic inputs only -- Human review required for any output use - -## Current Proof - -- Versioned rules in YAML -- Documented extraction behavior in [EXTRACTION_CONTRACT.md](../../EXTRACTION_CONTRACT.md) -- Pytest coverage for core behaviors -- Streamlit demo for inspection - -## Possible Extensions - -- More payer and procedure coverage -- Production integration layers -- Optional LLM-assisted text formatting behind separate contracts and tests diff --git a/docs/notes/STAKEHOLDER_NARRATIVE.md b/docs/notes/STAKEHOLDER_NARRATIVE.md deleted file mode 100644 index 9f19f6f..0000000 --- a/docs/notes/STAKEHOLDER_NARRATIVE.md +++ /dev/null @@ -1,41 +0,0 @@ -# Design Rationale Notes - -This file records design assumptions for the demo. It is not a record of live user research, deployment feedback, or stakeholder approval. - -## Intended Reviewers - -- Prior authorization coordinators -- Utilization management reviewers -- Compliance and audit reviewers -- Engineering interviewers evaluating scope discipline - -## Design Decisions - -### Refusal - -Missing required documentation returns `CANNOT_DETERMINE` instead of a guessed answer. - -### Evidence Spans - -When extraction captures supporting text, the UI shows the span so a reviewer can inspect what the rule engine used. - -### Policy Trust Signaling - -Rules and provenance are shown explicitly so reviewers can tell whether a pathway is demo-only or tied to monitored policy sources. - -### Write-Only Drafting - -Letter drafting is downstream of evaluated results and cannot change statuses or add new facts. - -## Current Repo Boundary - -- Deterministic implementation -- No LLM implementation -- Synthetic inputs only -- No production claims - -## Possible Extensions - -- Structured user research -- Production workflow instrumentation -- Broader rule coverage and integration work outside this repo diff --git a/inputs/test_plan.md b/inputs/test_plan.md deleted file mode 100644 index eb9aa46..0000000 --- a/inputs/test_plan.md +++ /dev/null @@ -1,40 +0,0 @@ -# Synthetic Test Plan — PA Readiness Copilot - -## Purpose -Expand coverage to reduce regressions and validate conservative extraction + rules behavior. - -## Ground Rules -- Any required item NOT_DOCUMENTED => overall_status = CANNOT_DETERMINE => expected_label = incomplete -- Borderline is only allowed when: - - not_documented_count == 0 - - not_met_count <= 1 - - score >= 60 - -## Categories to Cover -1) Boundary values -- 6 weeks exactly vs 5 weeks -- 6 weeks symptom duration vs 5 weeks - -2) Negation variants (red flags) -- "denies weakness" -- "no weakness" -- "weakness absent" -- "bowel/bladder intact" -- "no bowel or bladder changes" - -3) Ambiguity / documentation gaps -- "no red flags mentioned" (should be NOT_DOCUMENTED) -- "imaging noted" (inconclusive) -- "prior imaging" without modality/result (inconclusive) - -4) Conflicts -- "denies weakness" + later "reports weakness" (positive cue wins) - -5) Noisy notes -- templated text -- copied forward sections -- irrelevant numbers ("6 weeks pregnant", etc.) - -## Expansion Targets -- 25 cases: baseline credibility -- 50 cases: strong flagship signal