feat(calibrate): add cost-sensitive min-cost objective

docs/faq.md

@@ -265,6 +265,18 @@ calibrator place the cut at the cost-aware knee:
 wayfinder-router calibrate your-data.jsonl --mode threshold --objective knee --out wayfinder-router.toml
 ```
 
+If your actual prices should steer the cut, prefer `--objective min-cost`: it minimizes expected money
+plus a `--quality-penalty` (`Q`) charged whenever the cheap model botches a prompt that belonged on the
+strong one, so the boundary moves with the real cost gap and with how much a wrong answer costs you. (The
+knee, by contrast, maximizes quality × savings — a product in which the cost *ratio* cancels, so its cut
+is the same whether the strong model is 2× or 100× pricier.) `Q` defaults to the high-arm cost — a wrong
+answer costing roughly one strong-model redo — so raise it when a bad answer costs you more than a re-run:
+
+```bash
+wayfinder-router calibrate your-data.jsonl --mode threshold --objective min-cost \
+  --costs local=0.2,cloud=1.0 --quality-penalty 3.0 --out wayfinder-router.toml
+```
+
 To switch the lexical signals on for your domain, raise their weights and (optionally) supply your own
 trigger words — ideally mined from your labels rather than hand-picked. Full walkthrough, including the
 honest caveats, in [lexical-routing.md](lexical-routing.md). A ~20-prompt bootstrap is only a smoke test;

tests/test_calibrate.py

@@ -154,6 +154,86 @@ def test_knee_is_threshold_only():
 
 
 
+# --- min-cost objective (WF-ADR-0017) ---------------------------------------
+
+
+def _cost_separated_samples():
+    """A monotone score/label split so the min-cost sweep has a real interior optimum:
+    low-labeled prompts score low, high-labeled prompts score high, with a noisy middle."""
+    from wayfinder_router.calibrate import Sample
+
+    samples = []
+    for i in range(100):
+        s = round(i / 100, 4)
+        # mostly separable at ~0.4, with a noisy band so the cut placement matters
+        is_high = s >= 0.4 if i % 5 else s < 0.2
+        samples.append(Sample({}, "cloud" if is_high else "local", s))
+    return samples
+
+
+def test_min_cost_threshold_moves_with_the_cost_ratio():
+    # The whole point of min-cost over the knee: the actual prices steer the cut.
+    samples = _cost_separated_samples()
+    cheap_gap = calibrate(  # cheap arm nearly as costly as strong -> route more to strong (low cut)
+        samples, "threshold", objective="min-cost", costs={"local": 0.9, "cloud": 1.0}
+    )
+    wide_gap = calibrate(  # strong far pricier -> tolerate cheap routing (higher cut)
+        samples, "threshold", objective="min-cost", costs={"local": 0.05, "cloud": 1.0}
+    )
+    assert wide_gap.summary["threshold"] > cheap_gap.summary["threshold"]
+
+
+def test_min_cost_penalty_raises_the_high_arm_share():
+    # A bigger quality penalty makes botched cheap-routes costlier -> route more to strong.
+    samples = _cost_separated_samples()
+    costs = {"local": 0.2, "cloud": 1.0}
+    lax = calibrate(samples, "threshold", objective="min-cost", costs=costs, quality_penalty=0.5)
+    strict = calibrate(samples, "threshold", objective="min-cost", costs=costs, quality_penalty=50.0)
+    assert strict.summary["threshold"] <= lax.summary["threshold"]
+    assert strict.summary["quality_recovered"] >= lax.summary["quality_recovered"]
+
+
+def test_min_cost_is_deterministic():
+    samples = _cost_separated_samples()
+    assert calibrate(samples, "threshold", objective="min-cost").toml == \
+        calibrate(samples, "threshold", objective="min-cost").toml
+
+
+def test_min_cost_needs_no_target_and_emits_cost(tmp_path):
+    result = calibrate(_skewed_samples(), "threshold", objective="min-cost")  # no target
+    assert result.summary["objective"] == "min-cost"
+    assert "quality_penalty" in result.summary
+    (tmp_path / "wayfinder-router.toml").write_text(result.toml, encoding="utf-8")
+    config = load_routing_config(str(tmp_path))
+    assert [t.cost for t in config.tiers] == [0.2, 1.0]
+
+
+def test_min_cost_can_route_everything_cheap_at_the_ceiling(tmp_path):
+    # With no quality penalty the optimum is pure money: route all traffic to the cheap
+    # arm. The sweep expresses that as the score-ceiling cut (1.0), which is the highest
+    # min_score the config schema allows -- an honest "at these prices the strong model
+    # isn't worth it for this traffic," and still a loadable config.
+    result = calibrate(
+        _cost_separated_samples(), "threshold", objective="min-cost",
+        costs={"local": 0.05, "cloud": 1.0}, quality_penalty=0.0,
+    )
+    assert result.summary["threshold"] == 1.0
+    assert result.summary["quality_recovered"] == 0.0  # nothing routed to the strong arm
+    (tmp_path / "wayfinder-router.toml").write_text(result.toml, encoding="utf-8")
+    config = load_routing_config(str(tmp_path))
+    assert config.tiers[-1].min_score == 1.0
+
+
+def test_min_cost_rejects_a_negative_penalty():
+    with pytest.raises(CalibrationError):
+        calibrate(_cost_separated_samples(), "threshold", objective="min-cost", quality_penalty=-1.0)
+
+
+def test_min_cost_is_threshold_only():
+    with pytest.raises(CalibrationError):
+        calibrate(_skewed_samples(), "classifier", objective="min-cost")
+
+
 def test_cost_quality_only_in_threshold_mode(tmp_path):
     samples = load_dataset(_dataset(tmp_path, _binary_rows()))
     with pytest.raises(CalibrationError):

wayfinder_router/calibrate.py

@@ -178,15 +178,26 @@ def calibrate_threshold(
     objective: str = "accuracy",
     costs: dict[str, float] | None = None,
     target_savings: float | None = None,
+    quality_penalty: float | None = None,
     weights: dict[str, float] | None = None,
 ) -> CalibrationResult:
     """Binary calibration: sweep the local/cloud-style cut between two labels.
 
     ``objective="accuracy"`` (the default) picks the most accurate cut.
-    ``objective="knee"`` (WF-ADR-0017) picks the *cost-aware knee* — the cut that
-    maximizes quality-recovered × cost-saved, with no savings target to guess. On
-    skewed labels (one model usually right) the accuracy objective collapses to
-    always-routing-high; the knee balances quality and cost on its own.
+    ``objective="min-cost"`` (WF-ADR-0017) picks the cut that minimizes *expected
+    total cost per prompt* — money spent plus a ``quality_penalty`` (``Q``) charged
+    for every high-labeled prompt the cut sends to the cheap arm (a botched answer).
+    Unlike the knee, the cost ratio does not algebraically cancel: the minimum moves
+    with ``costs`` and ``Q``, so real prices actually steer the cut. ``Q`` is the one
+    honest knob — what a wrong answer costs you, in the same units as ``costs`` — and
+    defaults to the high-arm cost (a wrong answer ≈ one strong-model redo); raise it
+    when a bad answer costs more than a re-run.
+    ``objective="knee"`` (WF-ADR-0017) picks the *cost knee* — the cut that maximizes
+    quality-recovered × cost-saved, with no savings target to guess. On skewed labels
+    (one model usually right) the accuracy objective collapses to always-routing-high;
+    the knee keeps a real share on the cheap arm. Note the knee's cut is invariant to
+    the cost *ratio* (it cancels in the product), so use ``min-cost`` when the actual
+    prices should move the boundary.
     ``objective="cost-quality"`` picks the most accurate cut that still reaches
     ``target_savings`` against always-routing-high. All cost objectives take per-arm
     ``costs`` (defaulting to the benchmark's 0.2 / 1.0 units). Cost only moves where
@@ -219,6 +230,21 @@ def calibrate_threshold(
                      "quality_recovered": round(recall, 4), "cost_savings": round(savings, 4),
                      "samples": len(samples)},
         )
+    if objective == "min-cost":
+        cost_low, cost_high, low, high = _cost_ordered_arms(labels, costs)
+        q = cost_high if quality_penalty is None else float(quality_penalty)
+        if q < 0:
+            raise CalibrationError(f"quality_penalty must be >= 0 (got {q})")
+        scored = [(s.score, s.label == high) for s in samples]
+        threshold, accuracy, savings, recall = _sweep_cut_mincost(scored, cost_low, cost_high, q)
+        tiers = (Tier(0.0, low, cost_low), Tier(threshold, high, cost_high))
+        return CalibrationResult(
+            toml=prefix + _tiers_toml(tiers),
+            summary={"mode": "threshold", "objective": "min-cost", "threshold": threshold,
+                     "models": [low, high], "accuracy": round(accuracy, 4),
+                     "quality_recovered": round(recall, 4), "cost_savings": round(savings, 4),
+                     "quality_penalty": round(q, 4), "samples": len(samples)},
+        )
     if objective == "cost-quality":
         if target_savings is None:
             raise CalibrationError("cost-quality objective needs a target_savings")
@@ -347,6 +373,48 @@ def _sweep_cut_knee(
     return chosen, accuracy, _savings_at(scored, chosen, cost_low, cost_high), recall
 
 
+def _sweep_cut_mincost(
+    scored: list[tuple[float, bool]], cost_low: float, cost_high: float, quality_penalty: float
+) -> tuple[float, float, float, float]:
+    """The cut minimizing expected total cost per prompt: money + quality penalty.
+
+    Each prompt costs ``cost_high`` when routed high (``score >= cut``) else
+    ``cost_low``; each *high*-labeled prompt the cut routes low is additionally charged
+    ``quality_penalty`` (``Q``) — the price of the cheap model botching a prompt that
+    belonged on the strong one. Raising the cut trades money saved against quality lost,
+    and — crucially, unlike the knee — the cost ratio does **not** cancel: the minimum
+    moves with ``cost_low``, ``cost_high`` and ``Q``, so real prices steer the boundary.
+    Ties break to the median cut (a stable central choice).
+
+    The candidate set includes the score ceiling ``1.0`` so the sweep can express the
+    "route everything cheap" optimum a small ``Q`` genuinely wants — at these prices the
+    strong model isn't worth it for this traffic. It stops *at* the ceiling, not above:
+    a cut > 1.0 would route every prompt low, but the config schema caps ``min_score`` at
+    1.0 (a threshold above the score domain is rejected), so the honest, loadable way to
+    say "all cheap" is the ceiling cut. The lone residue — a prompt scoring exactly 1.0
+    still routes high under ``>=`` — is the schema's boundary rule, consistent everywhere.
+
+    Returns ``(threshold, accuracy, savings, recall)``.
+    """
+    candidates = sorted({0.0, 1.0, *(round(score, 4) for score, _ in scored)})
+    total = len(scored)
+    n_high = sum(1 for _, is_high in scored if is_high) or 1
+    best_obj: float | None = None
+    best_cuts: list[float] = []
+    for cut in candidates:
+        money = sum(cost_high if score >= cut else cost_low for score, _ in scored) / total
+        botched = sum(1 for score, is_high in scored if is_high and score < cut) / total
+        obj = money + quality_penalty * botched
+        if best_obj is None or obj < best_obj - 1e-12:
+            best_obj, best_cuts = obj, [cut]
+        elif abs(obj - best_obj) <= 1e-12:
+            best_cuts.append(cut)
+    chosen = best_cuts[len(best_cuts) // 2]
+    accuracy = sum(1 for score, is_high in scored if (score >= chosen) == is_high) / total
+    recall = sum(1 for score, is_high in scored if is_high and score >= chosen) / n_high
+    return chosen, accuracy, _savings_at(scored, chosen, cost_low, cost_high), recall
+
+
 def calibrate_tiers(
     samples: list[Sample], models_order: list[str] | None = None,
     *, weights: dict[str, float] | None = None,
@@ -578,14 +646,16 @@ def calibrate(
     objective: str = "accuracy",
     costs: dict[str, float] | None = None,
     target_savings: float | None = None,
+    quality_penalty: float | None = None,
     weights: dict[str, float] | None = None,
 ) -> CalibrationResult:
     """Dispatch to the requested calibration mode.
 
-    The cost-aware objective (WF-ADR-0017) is scoped to ``threshold`` mode in v1 —
-    the binary cut is where a savings target is well defined. ``weights`` (custom
-    feature weights, e.g. the lexical opt-in) applies to the score-based modes
-    (threshold, tiers); the classifier fits its own weights and ignores it.
+    The cost-aware objectives (WF-ADR-0017) are scoped to ``threshold``
+    mode — the binary cut is where a savings target or a per-prompt cost is well
+    defined. ``weights`` (custom feature weights, e.g. the lexical opt-in) applies to
+    the score-based modes (threshold, tiers); the classifier fits its own weights and
+    ignores it.
     """
     if objective != "accuracy" and mode != "threshold":
         raise CalibrationError(
@@ -594,7 +664,7 @@ def calibrate(
     if mode == "threshold":
         return calibrate_threshold(
             samples, objective=objective, costs=costs, target_savings=target_savings,
-            weights=weights,
+            quality_penalty=quality_penalty, weights=weights,
         )
     if mode == "tiers":
         return calibrate_tiers(samples, models_order=models_order, weights=weights)

wayfinder_router/cli.py

@@ -161,6 +161,7 @@ def _cmd_calibrate(args: argparse.Namespace) -> int:
             iterations=args.iterations,
             l2=args.l2,
             objective=args.objective,
+            quality_penalty=args.quality_penalty,
             costs=costs,
             target_savings=args.target_savings,
             weights=weights,
@@ -952,18 +953,27 @@ def build_parser() -> argparse.ArgumentParser:
     )
     p_cal.add_argument(
         "--objective",
-        choices=["accuracy", "knee", "cost-quality"],
+        choices=["accuracy", "min-cost", "knee", "cost-quality"],
         default="accuracy",
-        help="threshold mode: maximize accuracy (default); 'knee' for the cost-aware "
-        "knee (quality x savings, no target needed); or 'cost-quality' for accuracy at "
-        "a --target-savings.",
+        help="threshold mode: maximize accuracy (default); 'min-cost' to minimize "
+        "expected money + a --quality-penalty for botched cheap-routes (real prices "
+        "steer the cut); 'knee' for the cost knee (quality x savings, no target, but "
+        "ratio-invariant); or 'cost-quality' for accuracy at a --target-savings.",
     )
     p_cal.add_argument(
         "--target-savings",
         type=float,
         default=None,
         help="Cost saved vs always-routing-high to hold, 0.0-1.0 (cost-quality objective).",
     )
+    p_cal.add_argument(
+        "--quality-penalty",
+        type=float,
+        default=None,
+        help="min-cost objective: cost (in --costs units) of the cheap model botching a "
+        "prompt that belonged on the strong one. Defaults to the high-arm cost (a wrong "
+        "answer ~ one strong-model redo); raise it when a bad answer costs more.",
+    )
     p_cal.add_argument(
         "--costs",
         default=None,