diff --git a/docs/faq.md b/docs/faq.md index 688a3ae6..e39cd4c3 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -265,6 +265,18 @@ calibrator place the cut at the cost-aware knee: wayfinder-router calibrate your-data.jsonl --mode threshold --objective knee --out wayfinder-router.toml ``` +If your actual prices should steer the cut, prefer `--objective min-cost`: it minimizes expected money +plus a `--quality-penalty` (`Q`) charged whenever the cheap model botches a prompt that belonged on the +strong one, so the boundary moves with the real cost gap and with how much a wrong answer costs you. (The +knee, by contrast, maximizes quality × savings — a product in which the cost *ratio* cancels, so its cut +is the same whether the strong model is 2× or 100× pricier.) `Q` defaults to the high-arm cost — a wrong +answer costing roughly one strong-model redo — so raise it when a bad answer costs you more than a re-run: + +```bash +wayfinder-router calibrate your-data.jsonl --mode threshold --objective min-cost \ + --costs local=0.2,cloud=1.0 --quality-penalty 3.0 --out wayfinder-router.toml +``` + To switch the lexical signals on for your domain, raise their weights and (optionally) supply your own trigger words — ideally mined from your labels rather than hand-picked. Full walkthrough, including the honest caveats, in [lexical-routing.md](lexical-routing.md). A ~20-prompt bootstrap is only a smoke test; diff --git a/tests/test_calibrate.py b/tests/test_calibrate.py index 2d17bf83..9c025aba 100644 --- a/tests/test_calibrate.py +++ b/tests/test_calibrate.py @@ -154,6 +154,86 @@ def test_knee_is_threshold_only(): +# --- min-cost objective (WF-ADR-0017) --------------------------------------- + + +def _cost_separated_samples(): + """A monotone score/label split so the min-cost sweep has a real interior optimum: + low-labeled prompts score low, high-labeled prompts score high, with a noisy middle.""" + from wayfinder_router.calibrate import Sample + + samples = [] + for i in range(100): + s = round(i / 100, 4) + # mostly separable at ~0.4, with a noisy band so the cut placement matters + is_high = s >= 0.4 if i % 5 else s < 0.2 + samples.append(Sample({}, "cloud" if is_high else "local", s)) + return samples + + +def test_min_cost_threshold_moves_with_the_cost_ratio(): + # The whole point of min-cost over the knee: the actual prices steer the cut. + samples = _cost_separated_samples() + cheap_gap = calibrate( # cheap arm nearly as costly as strong -> route more to strong (low cut) + samples, "threshold", objective="min-cost", costs={"local": 0.9, "cloud": 1.0} + ) + wide_gap = calibrate( # strong far pricier -> tolerate cheap routing (higher cut) + samples, "threshold", objective="min-cost", costs={"local": 0.05, "cloud": 1.0} + ) + assert wide_gap.summary["threshold"] > cheap_gap.summary["threshold"] + + +def test_min_cost_penalty_raises_the_high_arm_share(): + # A bigger quality penalty makes botched cheap-routes costlier -> route more to strong. + samples = _cost_separated_samples() + costs = {"local": 0.2, "cloud": 1.0} + lax = calibrate(samples, "threshold", objective="min-cost", costs=costs, quality_penalty=0.5) + strict = calibrate(samples, "threshold", objective="min-cost", costs=costs, quality_penalty=50.0) + assert strict.summary["threshold"] <= lax.summary["threshold"] + assert strict.summary["quality_recovered"] >= lax.summary["quality_recovered"] + + +def test_min_cost_is_deterministic(): + samples = _cost_separated_samples() + assert calibrate(samples, "threshold", objective="min-cost").toml == \ + calibrate(samples, "threshold", objective="min-cost").toml + + +def test_min_cost_needs_no_target_and_emits_cost(tmp_path): + result = calibrate(_skewed_samples(), "threshold", objective="min-cost") # no target + assert result.summary["objective"] == "min-cost" + assert "quality_penalty" in result.summary + (tmp_path / "wayfinder-router.toml").write_text(result.toml, encoding="utf-8") + config = load_routing_config(str(tmp_path)) + assert [t.cost for t in config.tiers] == [0.2, 1.0] + + +def test_min_cost_can_route_everything_cheap_at_the_ceiling(tmp_path): + # With no quality penalty the optimum is pure money: route all traffic to the cheap + # arm. The sweep expresses that as the score-ceiling cut (1.0), which is the highest + # min_score the config schema allows -- an honest "at these prices the strong model + # isn't worth it for this traffic," and still a loadable config. + result = calibrate( + _cost_separated_samples(), "threshold", objective="min-cost", + costs={"local": 0.05, "cloud": 1.0}, quality_penalty=0.0, + ) + assert result.summary["threshold"] == 1.0 + assert result.summary["quality_recovered"] == 0.0 # nothing routed to the strong arm + (tmp_path / "wayfinder-router.toml").write_text(result.toml, encoding="utf-8") + config = load_routing_config(str(tmp_path)) + assert config.tiers[-1].min_score == 1.0 + + +def test_min_cost_rejects_a_negative_penalty(): + with pytest.raises(CalibrationError): + calibrate(_cost_separated_samples(), "threshold", objective="min-cost", quality_penalty=-1.0) + + +def test_min_cost_is_threshold_only(): + with pytest.raises(CalibrationError): + calibrate(_skewed_samples(), "classifier", objective="min-cost") + + def test_cost_quality_only_in_threshold_mode(tmp_path): samples = load_dataset(_dataset(tmp_path, _binary_rows())) with pytest.raises(CalibrationError): diff --git a/wayfinder_router/calibrate.py b/wayfinder_router/calibrate.py index f49bc80a..7fb7302f 100644 --- a/wayfinder_router/calibrate.py +++ b/wayfinder_router/calibrate.py @@ -178,15 +178,26 @@ def calibrate_threshold( objective: str = "accuracy", costs: dict[str, float] | None = None, target_savings: float | None = None, + quality_penalty: float | None = None, weights: dict[str, float] | None = None, ) -> CalibrationResult: """Binary calibration: sweep the local/cloud-style cut between two labels. ``objective="accuracy"`` (the default) picks the most accurate cut. - ``objective="knee"`` (WF-ADR-0017) picks the *cost-aware knee* — the cut that - maximizes quality-recovered × cost-saved, with no savings target to guess. On - skewed labels (one model usually right) the accuracy objective collapses to - always-routing-high; the knee balances quality and cost on its own. + ``objective="min-cost"`` (WF-ADR-0017) picks the cut that minimizes *expected + total cost per prompt* — money spent plus a ``quality_penalty`` (``Q``) charged + for every high-labeled prompt the cut sends to the cheap arm (a botched answer). + Unlike the knee, the cost ratio does not algebraically cancel: the minimum moves + with ``costs`` and ``Q``, so real prices actually steer the cut. ``Q`` is the one + honest knob — what a wrong answer costs you, in the same units as ``costs`` — and + defaults to the high-arm cost (a wrong answer ≈ one strong-model redo); raise it + when a bad answer costs more than a re-run. + ``objective="knee"`` (WF-ADR-0017) picks the *cost knee* — the cut that maximizes + quality-recovered × cost-saved, with no savings target to guess. On skewed labels + (one model usually right) the accuracy objective collapses to always-routing-high; + the knee keeps a real share on the cheap arm. Note the knee's cut is invariant to + the cost *ratio* (it cancels in the product), so use ``min-cost`` when the actual + prices should move the boundary. ``objective="cost-quality"`` picks the most accurate cut that still reaches ``target_savings`` against always-routing-high. All cost objectives take per-arm ``costs`` (defaulting to the benchmark's 0.2 / 1.0 units). Cost only moves where @@ -219,6 +230,21 @@ def calibrate_threshold( "quality_recovered": round(recall, 4), "cost_savings": round(savings, 4), "samples": len(samples)}, ) + if objective == "min-cost": + cost_low, cost_high, low, high = _cost_ordered_arms(labels, costs) + q = cost_high if quality_penalty is None else float(quality_penalty) + if q < 0: + raise CalibrationError(f"quality_penalty must be >= 0 (got {q})") + scored = [(s.score, s.label == high) for s in samples] + threshold, accuracy, savings, recall = _sweep_cut_mincost(scored, cost_low, cost_high, q) + tiers = (Tier(0.0, low, cost_low), Tier(threshold, high, cost_high)) + return CalibrationResult( + toml=prefix + _tiers_toml(tiers), + summary={"mode": "threshold", "objective": "min-cost", "threshold": threshold, + "models": [low, high], "accuracy": round(accuracy, 4), + "quality_recovered": round(recall, 4), "cost_savings": round(savings, 4), + "quality_penalty": round(q, 4), "samples": len(samples)}, + ) if objective == "cost-quality": if target_savings is None: raise CalibrationError("cost-quality objective needs a target_savings") @@ -347,6 +373,48 @@ def _sweep_cut_knee( return chosen, accuracy, _savings_at(scored, chosen, cost_low, cost_high), recall +def _sweep_cut_mincost( + scored: list[tuple[float, bool]], cost_low: float, cost_high: float, quality_penalty: float +) -> tuple[float, float, float, float]: + """The cut minimizing expected total cost per prompt: money + quality penalty. + + Each prompt costs ``cost_high`` when routed high (``score >= cut``) else + ``cost_low``; each *high*-labeled prompt the cut routes low is additionally charged + ``quality_penalty`` (``Q``) — the price of the cheap model botching a prompt that + belonged on the strong one. Raising the cut trades money saved against quality lost, + and — crucially, unlike the knee — the cost ratio does **not** cancel: the minimum + moves with ``cost_low``, ``cost_high`` and ``Q``, so real prices steer the boundary. + Ties break to the median cut (a stable central choice). + + The candidate set includes the score ceiling ``1.0`` so the sweep can express the + "route everything cheap" optimum a small ``Q`` genuinely wants — at these prices the + strong model isn't worth it for this traffic. It stops *at* the ceiling, not above: + a cut > 1.0 would route every prompt low, but the config schema caps ``min_score`` at + 1.0 (a threshold above the score domain is rejected), so the honest, loadable way to + say "all cheap" is the ceiling cut. The lone residue — a prompt scoring exactly 1.0 + still routes high under ``>=`` — is the schema's boundary rule, consistent everywhere. + + Returns ``(threshold, accuracy, savings, recall)``. + """ + candidates = sorted({0.0, 1.0, *(round(score, 4) for score, _ in scored)}) + total = len(scored) + n_high = sum(1 for _, is_high in scored if is_high) or 1 + best_obj: float | None = None + best_cuts: list[float] = [] + for cut in candidates: + money = sum(cost_high if score >= cut else cost_low for score, _ in scored) / total + botched = sum(1 for score, is_high in scored if is_high and score < cut) / total + obj = money + quality_penalty * botched + if best_obj is None or obj < best_obj - 1e-12: + best_obj, best_cuts = obj, [cut] + elif abs(obj - best_obj) <= 1e-12: + best_cuts.append(cut) + chosen = best_cuts[len(best_cuts) // 2] + accuracy = sum(1 for score, is_high in scored if (score >= chosen) == is_high) / total + recall = sum(1 for score, is_high in scored if is_high and score >= chosen) / n_high + return chosen, accuracy, _savings_at(scored, chosen, cost_low, cost_high), recall + + def calibrate_tiers( samples: list[Sample], models_order: list[str] | None = None, *, weights: dict[str, float] | None = None, @@ -578,14 +646,16 @@ def calibrate( objective: str = "accuracy", costs: dict[str, float] | None = None, target_savings: float | None = None, + quality_penalty: float | None = None, weights: dict[str, float] | None = None, ) -> CalibrationResult: """Dispatch to the requested calibration mode. - The cost-aware objective (WF-ADR-0017) is scoped to ``threshold`` mode in v1 — - the binary cut is where a savings target is well defined. ``weights`` (custom - feature weights, e.g. the lexical opt-in) applies to the score-based modes - (threshold, tiers); the classifier fits its own weights and ignores it. + The cost-aware objectives (WF-ADR-0017) are scoped to ``threshold`` + mode — the binary cut is where a savings target or a per-prompt cost is well + defined. ``weights`` (custom feature weights, e.g. the lexical opt-in) applies to + the score-based modes (threshold, tiers); the classifier fits its own weights and + ignores it. """ if objective != "accuracy" and mode != "threshold": raise CalibrationError( @@ -594,7 +664,7 @@ def calibrate( if mode == "threshold": return calibrate_threshold( samples, objective=objective, costs=costs, target_savings=target_savings, - weights=weights, + quality_penalty=quality_penalty, weights=weights, ) if mode == "tiers": return calibrate_tiers(samples, models_order=models_order, weights=weights) diff --git a/wayfinder_router/cli.py b/wayfinder_router/cli.py index e61cbb23..9f6b19a0 100644 --- a/wayfinder_router/cli.py +++ b/wayfinder_router/cli.py @@ -161,6 +161,7 @@ def _cmd_calibrate(args: argparse.Namespace) -> int: iterations=args.iterations, l2=args.l2, objective=args.objective, + quality_penalty=args.quality_penalty, costs=costs, target_savings=args.target_savings, weights=weights, @@ -952,11 +953,12 @@ def build_parser() -> argparse.ArgumentParser: ) p_cal.add_argument( "--objective", - choices=["accuracy", "knee", "cost-quality"], + choices=["accuracy", "min-cost", "knee", "cost-quality"], default="accuracy", - help="threshold mode: maximize accuracy (default); 'knee' for the cost-aware " - "knee (quality x savings, no target needed); or 'cost-quality' for accuracy at " - "a --target-savings.", + help="threshold mode: maximize accuracy (default); 'min-cost' to minimize " + "expected money + a --quality-penalty for botched cheap-routes (real prices " + "steer the cut); 'knee' for the cost knee (quality x savings, no target, but " + "ratio-invariant); or 'cost-quality' for accuracy at a --target-savings.", ) p_cal.add_argument( "--target-savings", @@ -964,6 +966,14 @@ def build_parser() -> argparse.ArgumentParser: default=None, help="Cost saved vs always-routing-high to hold, 0.0-1.0 (cost-quality objective).", ) + p_cal.add_argument( + "--quality-penalty", + type=float, + default=None, + help="min-cost objective: cost (in --costs units) of the cheap model botching a " + "prompt that belonged on the strong one. Defaults to the high-arm cost (a wrong " + "answer ~ one strong-model redo); raise it when a bad answer costs more.", + ) p_cal.add_argument( "--costs", default=None,