diff --git a/optimizations/ooo_spec_lucebox5_cpu/README.md b/optimizations/ooo_spec_lucebox5_cpu/README.md new file mode 100644 index 000000000..1f18a0e8e --- /dev/null +++ b/optimizations/ooo_spec_lucebox5_cpu/README.md @@ -0,0 +1,133 @@ +# CPU-isolated tool speculation on Lucebox5 + +The engine asks a small predictor for one concrete tool call before the target +model runs. On Lucebox5, Qwen3-0.6B Q8_0 predicts on Strix, then exits its GPU +compute window; the predicted read-only tool runs on reserved CPU cores while +DeepSeek-V4-0731 decodes with DS4/DSpark on R9700 + Strix. The result stays +private unless DeepSeek emits the exact same canonical function and arguments. + +This path does not inject tokens, replace DSpark, or retry speculative decoding +with autoregressive decoding. A wrong prediction is discarded and the caller +executes the target model's authoritative call normally. + +## Production result + +The 2026-08-17 paired run compiled a recurring, side-effect-free five-step +trace into one typed workflow tool. Independent branches ran concurrently on +the isolated CPU lane. The six measured tasks covered 10, 15, and 20 leaf calls +twice each, with randomized arm order and one warmup task. + +| Metric | Result | +| --- | ---: | +| Normal stage-batched workflow, p50 | 81.030 s | +| Trace-compiled + speculative workflow, p50 | **14.597 s** | +| End-to-end speedup, paired p50 | **5.5961x** | +| End-to-end bootstrap 95% CI | **5.4577x–5.6599x** | +| Trace compilation alone | **3.2806x** | +| Early launch on top of compilation | **1.6954x** | +| Early-launch bootstrap 95% CI | **1.6760x–1.7210x** | +| Exposed tool wait, compiled / speculative p50 | 10.143 s / **0.027 ms** | +| Qwen prediction latency, p50 | 203.5 ms | +| Target model-compute slowdown, p50 / p95 | -0.458% / -0.332% | +| Target decode slowdown, p50 / p95 | -0.101% / 0.219% | +| Exact predictor-to-target hits | 6 / 6 | + +All 20 production gates passed: identical leaf calls, tool-result hashes, +macro calls, and final outputs; positive DS4 acceptance on every call turn; +correct CPU isolation; and no measurable target slowdown. The 5.60x result +combines two independent gains: four fewer model/tool synchronization barriers +from trace compilation, plus the 1.70x gained by starting the compiled graph +before target authorization completes. + +Artifact: + +- `results/trace-compiled-engine-qwen-production-6pairs-compact.json` + (`sha256:0807cca1d22453728b069a0150800fcfa9a513db6a9f25815663fc03b99285b9`) + +The compact training fixture below reproduces the artifact's compiled pattern +fingerprint (`06d95882…0645`); the artifact retains the original full-report +hash for provenance. + +## Safety and portability + +- Only explicitly allowlisted, read-only/idempotent tools are eligible. +- The external result is committed only on an exact canonical call match. +- The executor is launched directly without a shell and has a hard timeout. +- Lucebox5 reserves CPUs `14-15,30-31`; the model uses `0-13,16-29`. +- Startup fails closed if CPU masks overlap or the measured lane profile fails. +- `before-model` is the native predictor default, so shared-GPU prediction + cannot reduce target prefill/decode throughput. +- The same API works on a single GPU: run the predictor before the target and + overlap only the CPU tool. An HTTP predictor can use the same verification + and executor path on other model families. + +The speedup applies to tool-using request latency, not token throughput. Its +real-world value depends on exact predictor hit rate and on how much tool work +can overlap target generation. + +## Reproduce + +Build the deterministic sparse tool used by the single-call qualification: + +```bash +JSON_INCLUDE=/path/to/server/deps/json/include \ + ./build_cpu_sparse_executor.sh ./cpu_sparse_tool_executor +``` + +Launch the qualified single-call configuration on an otherwise idle Lucebox5: + +```bash +./run_native_cpu_server_lucebox5.sh +``` + +The launcher defaults to Qwen3-0.6B Q8_0 on predictor GPU 1. Override placement +with `PREDICTOR_MODEL`, `PREDICTOR_GPU`, `PREDICTOR_MAX_CTX`, and +`PREDICTOR_MAX_TOKENS`. The adjacent `candidate-build` symlink in the wrapper +selects a build even though the qualified launcher clears ambient variables. + +Run the single-call paired gate: + +```bash +python3 benchmark_cpu_tool_speculation.py native-qwen \ + --url http://127.0.0.1:18145/v1/chat/completions \ + --binary ./cpu_sparse_tool_executor \ + --tool-cpus 14-15,30-31 \ + --iterations 172452 \ + --max-tokens 32 \ + --pairs 20 \ + --warmups 5 \ + --bootstrap-resamples 20000 \ + --min-speedup 1.6 \ + --min-speedup-ci-low 1.5 \ + --min-speedup-p05 1.5 \ + --min-prediction-hit-rate 1.0 \ + --max-model-slowdown-percent 5 \ + --output results/qwen-auto-production-20pairs.json +``` + +For the 10–20-call workflow gate, launch with the trace executor and macro +allowlist: + +```bash +TOOL_SPEC_EXECUTOR=./trace_compiled_tool_executor.py \ +TOOL_SPEC_ALLOW=resolve_customer,list_open_orders,get_order_details,calculate_shipping,prepare_customer_summary,execute_customer_workflows \ + ./run_native_cpu_server_lucebox5.sh +``` + +Then run: + +```bash +python3 benchmark_trace_compiled_workflows.py \ + --binary ./bfcl_replay_tool_executor.py \ + --training-report results/trace-compiled-training-traces.json \ + --pairs 6 \ + --warmup-tasks 1 \ + --min-branches 2 \ + --max-branches 4 \ + --seed 814 \ + --bootstrap-resamples 20000 \ + --output results/trace-compiled-engine-qwen-production-6pairs-compact.json +``` + +The harness exits nonzero on any correctness, isolation, DS4-activity, +slowdown, hit-rate, or speed threshold failure. diff --git a/optimizations/ooo_spec_lucebox5_cpu/benchmark_cpu_tool_speculation.py b/optimizations/ooo_spec_lucebox5_cpu/benchmark_cpu_tool_speculation.py new file mode 100755 index 000000000..b2238c97d --- /dev/null +++ b/optimizations/ooo_spec_lucebox5_cpu/benchmark_cpu_tool_speculation.py @@ -0,0 +1,1409 @@ +#!/usr/bin/env python3 +"""Qualify and benchmark a disjoint Strix CPU speculative-tool lane. + +The qualification phase runs the official model server without tool +speculation, pins the real sparse-compute tool to reserved physical cores, and +measures sequential versus overlapped execution. It emits a qualified engine +profile only if model output, DS4 activity, tool results, CPU isolation, and +the slowdown gate all pass. + +The native phase then measures the engine's exact-call commit path against the +same strong sequential CPU baseline. Wrong predictions are cancelled and +their private results must never be exposed. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import random +import re +import statistics +import subprocess +import time +import urllib.request +from pathlib import Path +from typing import Any, Iterable +from urllib.parse import urlsplit, urlunsplit + + +TOOL_NAME = "benchmark_cpu_sparse" +PROTOCOL = "dflash.tool-speculation.v1" +NATIVE_PREDICTION_SOURCE = "native-qwen3" +SPARSE_ROWS = 4096 +SPARSE_NONZEROS_PER_ROW = 16 +SPARSE_THREADS = 2 +SPARSE_SEED = 731 + + +def parse_cpu_list(value: str) -> list[int]: + cpus: set[int] = set() + for item in value.split(","): + if not item: + raise argparse.ArgumentTypeError("CPU list contains an empty item") + if "-" in item: + parts = item.split("-") + if len(parts) != 2 or not all(part.isdigit() for part in parts): + raise argparse.ArgumentTypeError(f"invalid CPU range: {item}") + first, last = map(int, parts) + if first > last: + raise argparse.ArgumentTypeError(f"invalid CPU range: {item}") + cpus.update(range(first, last + 1)) + elif item.isdigit(): + cpus.add(int(item)) + else: + raise argparse.ArgumentTypeError(f"invalid CPU id: {item}") + if not cpus: + raise argparse.ArgumentTypeError("CPU list must not be empty") + return sorted(cpus) + + +def compact_cpu_list(cpus: Iterable[int]) -> str: + return ",".join(str(cpu) for cpu in cpus) + + +def expected_arguments( + rows: int, + nonzeros_per_row: int, + iterations: int, + threads: int, + seed: int, +) -> dict[str, int]: + if ( + rows != SPARSE_ROWS + or nonzeros_per_row != SPARSE_NONZEROS_PER_ROW + or threads != SPARSE_THREADS + or seed != SPARSE_SEED + ): + raise ValueError("this qualification binary has a fixed sparse shape") + # Keep the generated tool call intentionally short. The deterministic + # benchmark binary owns the qualified sparse shape; only work duration is + # request-dependent, matching real tools with a compact identifier. + return {"iterations": iterations} + + +def tool_definition() -> dict[str, Any]: + properties = {"iterations": {"type": "integer"}} + return { + "name": TOOL_NAME, + "parameters": { + "type": "object", + "properties": properties, + "required": list(properties), + "additionalProperties": False, + }, + } + + +def request_body( + arguments: dict[str, int], + max_tokens: int, + *, + prediction: dict[str, int] | None, + automatic_prediction: bool = False, + tool_choice: str | None = None, +) -> dict[str, Any]: + compact = json.dumps(arguments, separators=(",", ":")) + prompt = f"Return only this JSON object and nothing else: {compact}" + body: dict[str, Any] = { + "model": "dflash", + "stream": False, + "max_tokens": max_tokens, + "temperature": 0, + "messages": [ + { + "role": "user", + "content": prompt, + } + ], + "tools": [tool_definition()], + "automatic_tool_speculation": automatic_prediction, + } + if tool_choice is not None: + body["tool_choice"] = tool_choice + if prediction is not None: + body["tool_speculation"] = { + "call": {"name": TOOL_NAME, "arguments": prediction}, + "confidence": 1.0, + } + return body + + +def normalize_tool_call(result: dict[str, Any]) -> dict[str, Any] | None: + message = result.get("choices", [{}])[0].get("message", {}) + tool_calls = message.get("tool_calls") or [] + if len(tool_calls) == 1: + function = tool_calls[0].get("function") or {} + arguments = function.get("arguments", "{}") + if isinstance(arguments, str): + try: + arguments = json.loads(arguments) + except json.JSONDecodeError: + return None + return {"name": function.get("name"), "arguments": arguments} + content = message.get("content") + if not isinstance(content, str) or not content: + return None + bracket_call = re.fullmatch(r'\["([^"]+)"\]\((\{.*\})\)', content) + if bracket_call: + try: + return { + "name": bracket_call.group(1), + "arguments": json.loads(bracket_call.group(2)), + } + except json.JSONDecodeError: + return None + try: + parsed = json.loads(content) + except json.JSONDecodeError: + return None + if not isinstance(parsed, dict): + return None + function = parsed.get("function", parsed.get("name")) + if isinstance(function, dict): + name = function.get("name") + arguments = function.get("arguments", function.get("parameters")) + else: + name = function + arguments = parsed.get( + "params", + parsed.get( + "parameters", + parsed.get("arguments", parsed.get("function_args")), + ), + ) + if ( + arguments is None + and isinstance(parsed.get("parameter"), str) + and "parameter_value" in parsed + ): + # DeepSeek may serialize a one-argument native call as a compact + # name/value envelope. It is semantically the same function call. + arguments = {parsed["parameter"]: parsed["parameter_value"]} + if arguments is None and isinstance(name, str): + arguments = { + key: value + for key, value in parsed.items() + if key not in {"function", "name", "type"} + } + if isinstance(arguments, str): + try: + arguments = json.loads(arguments) + except json.JSONDecodeError: + return None + if not isinstance(name, str) or not isinstance(arguments, dict): + return None + return {"name": name, "arguments": arguments} + + +def post_json( + url: str, body: dict[str, Any], timeout: float +) -> tuple[dict[str, Any], float]: + request = urllib.request.Request( + url, + data=json.dumps(body, separators=(",", ":")).encode(), + headers={"Content-Type": "application/json"}, + method="POST", + ) + started = time.perf_counter() + with urllib.request.urlopen(request, timeout=timeout) as response: + result = json.load(response) + if not isinstance(result, dict): + raise RuntimeError("model response is not a JSON object") + return result, (time.perf_counter() - started) * 1000.0 + + +def get_json(url: str, timeout: float) -> dict[str, Any]: + with urllib.request.urlopen(url, timeout=timeout) as response: + result = json.load(response) + if not isinstance(result, dict): + raise RuntimeError(f"expected a JSON object from {url}") + return result + + +def props_url(completion_url: str) -> str: + parsed = urlsplit(completion_url) + return urlunsplit((parsed.scheme, parsed.netloc, "/props", "", "")) + + +def observation(result: dict[str, Any], wall_ms: float) -> dict[str, Any]: + call = normalize_tool_call(result) + usage = result.get("usage") or {} + timings = usage.get("timings") or {} + message = result.get("choices", [{}])[0].get("message", {}) + content = message.get("content") or "" + canonical_call = json.dumps(call, sort_keys=True, separators=(",", ":")) + return { + "request_wall_ms": wall_ms, + "model_compute_ms": float(timings.get("prefill_ms", 0.0)) + + float(timings.get("decode_ms", 0.0)), + "prefill_ms": float(timings.get("prefill_ms", 0.0)), + "decode_ms": float(timings.get("decode_ms", 0.0)), + "decode_tokens_per_sec": float( + timings.get("decode_tokens_per_sec", 0.0) + ), + "completion_tokens": int(usage.get("completion_tokens", 0)), + "accept_rate": float(usage.get("accept_rate", 0.0)), + "tool_call": call, + "tool_call_sha256": hashlib.sha256(canonical_call.encode()).hexdigest(), + "assistant_content_sha256": hashlib.sha256(content.encode()).hexdigest(), + "speculation": result.get("dflash_tool_speculation"), + } + + +def post_model( + url: str, + arguments: dict[str, int], + max_tokens: int, + timeout: float, + *, + prediction: dict[str, int] | None = None, + automatic_prediction: bool = False, + tool_choice: str | None = None, +) -> dict[str, Any]: + result, wall_ms = post_json( + url, + request_body( + arguments, + max_tokens, + prediction=prediction, + automatic_prediction=automatic_prediction, + tool_choice=tool_choice, + ), + timeout, + ) + return observation(result, wall_ms) + + +def executor_request( + arguments: dict[str, int], cpus: list[int], request_id: str +) -> dict[str, Any]: + return { + "protocol": PROTOCOL, + "request_id": request_id, + "mode": "authoritative-benchmark", + "resource_percentage": 100, + "accelerator_relation": "non_accelerator", + "cpu_affinity": cpus, + "cpu_affinity_isolated": True, + "call": {"name": TOOL_NAME, "arguments": arguments}, + } + + +def start_executor( + binary: Path, + arguments: dict[str, int], + cpus: list[int], + request_id: str, +) -> dict[str, Any]: + environment = os.environ.copy() + environment["DFLASH_TOOL_SPECULATION_CPU_AFFINITY"] = compact_cpu_list(cpus) + + def pin_child() -> None: + os.sched_setaffinity(0, set(cpus)) + + started = time.perf_counter() + process = subprocess.Popen( + [str(binary), "--dflash-tool-spec-v1"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=environment, + preexec_fn=pin_child, + ) + assert process.stdin is not None + process.stdin.write( + json.dumps( + executor_request(arguments, cpus, request_id), + separators=(",", ":"), + ) + + "\n" + ) + process.stdin.close() + process.stdin = None + return {"process": process, "started": started} + + +def finish_executor(handle: dict[str, Any], timeout: float) -> dict[str, Any]: + process: subprocess.Popen[str] = handle["process"] + try: + stdout, stderr = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + process.kill() + stdout, stderr = process.communicate(timeout=5) + raise RuntimeError("CPU executor timed out") + wall_ms = (time.perf_counter() - float(handle["started"])) * 1000.0 + if process.returncode != 0: + raise RuntimeError( + f"CPU executor exited {process.returncode}: {stderr.strip()}" + ) + try: + envelope = json.loads(stdout) + except json.JSONDecodeError as error: + raise RuntimeError(f"CPU executor returned invalid JSON: {stdout!r}") from error + if not isinstance(envelope, dict) or not envelope.get("ok"): + raise RuntimeError(f"CPU executor rejected request: {envelope!r}") + result = envelope.get("result") + if not isinstance(result, dict): + raise RuntimeError("CPU executor result is not an object") + return {"wall_ms": wall_ms, "result": result} + + +def stop_executor(handle: dict[str, Any]) -> None: + process: subprocess.Popen[str] = handle["process"] + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=1.0) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5.0) + + +def run_executor( + binary: Path, + arguments: dict[str, int], + cpus: list[int], + timeout: float, + request_id: str, +) -> dict[str, Any]: + return finish_executor( + start_executor(binary, arguments, cpus, request_id), timeout + ) + + +def expected_call(arguments: dict[str, int]) -> dict[str, Any]: + return {"name": TOOL_NAME, "arguments": arguments} + + +def validate_model_call(row: dict[str, Any], arguments: dict[str, int]) -> None: + expected = expected_call(arguments) + if row["tool_call"] != expected: + raise RuntimeError( + f"model emitted {row['tool_call']!r}, expected {expected!r}" + ) + + +def percentile(values: Iterable[float], quantile: float) -> float: + ordered = sorted(float(value) for value in values) + if not ordered: + raise ValueError("cannot take percentile of an empty sequence") + position = (len(ordered) - 1) * quantile + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return ordered[lower] + fraction = position - lower + return ordered[lower] * (1.0 - fraction) + ordered[upper] * fraction + + +def bootstrap_speedup_ci( + pairs: list[dict[str, Any]], resamples: int, seed: int +) -> list[float]: + generator = random.Random(seed) + ratios = [] + for _ in range(resamples): + sample = [pairs[generator.randrange(len(pairs))] for _ in pairs] + control = statistics.median( + float(pair["control"]["task_ms"]) for pair in sample + ) + speculative = statistics.median( + float(pair["speculative"]["task_ms"]) for pair in sample + ) + ratios.append(control / speculative) + return [percentile(ratios, 0.025), percentile(ratios, 0.975)] + + +def read_process_affinity(pid: int) -> list[int]: + return sorted(os.sched_getaffinity(pid)) + + +def calibrate( + args: argparse.Namespace, +) -> tuple[dict[str, int], dict[str, Any]]: + probe_arguments = expected_arguments( + args.rows, + args.nonzeros_per_row, + max(1, args.initial_iterations), + args.threads, + args.tool_seed, + ) + model_samples = [] + for _ in range(args.calibration_model_samples): + row = post_model( + args.url, probe_arguments, args.max_tokens, args.timeout + ) + validate_model_call(row, probe_arguments) + model_samples.append(row) + target_ms = statistics.median( + float(row["request_wall_ms"]) for row in model_samples + ) + + iterations = args.initial_iterations + calibration_steps = [] + for step in range(args.calibration_steps): + arguments = expected_arguments( + args.rows, + args.nonzeros_per_row, + iterations, + args.threads, + args.tool_seed, + ) + samples = [ + run_executor( + args.binary, + arguments, + args.tool_cpus, + args.timeout, + f"calibrate-{step}-{sample}", + ) + for sample in range(args.calibration_tool_samples) + ] + observed_ms = statistics.median( + float(sample["wall_ms"]) for sample in samples + ) + calibration_steps.append( + { + "iterations": iterations, + "tool_wall_p50_ms": observed_ms, + "samples": samples, + } + ) + if observed_ms <= 0: + raise RuntimeError("CPU executor calibration returned zero time") + ratio = target_ms / observed_ms + if 0.97 <= ratio <= 1.03: + break + iterations = max(1, round(iterations * ratio)) + + final_arguments = expected_arguments( + args.rows, + args.nonzeros_per_row, + iterations, + args.threads, + args.tool_seed, + ) + return final_arguments, { + "target_model_request_p50_ms": target_ms, + "model_samples": model_samples, + "steps": calibration_steps, + "selected_iterations": iterations, + } + + +def run_direct_control( + args: argparse.Namespace, + arguments: dict[str, int], + label: str, +) -> dict[str, Any]: + started = time.perf_counter() + model = post_model(args.url, arguments, args.max_tokens, args.timeout) + tool = run_executor( + args.binary, arguments, args.tool_cpus, args.timeout, f"{label}-tool" + ) + validate_model_call(model, arguments) + return { + "mode": "control", + "task_ms": (time.perf_counter() - started) * 1000.0, + "model": model, + "tool": tool, + } + + +def run_direct_overlap( + args: argparse.Namespace, + arguments: dict[str, int], + label: str, +) -> dict[str, Any]: + started = time.perf_counter() + handle = start_executor(args.binary, arguments, args.tool_cpus, label) + try: + model = post_model(args.url, arguments, args.max_tokens, args.timeout) + tool = finish_executor(handle, args.timeout) + except BaseException: + stop_executor(handle) + raise + validate_model_call(model, arguments) + return { + "mode": "speculative", + "task_ms": (time.perf_counter() - started) * 1000.0, + "model": model, + "tool": tool, + } + + +def run_direct_miss( + args: argparse.Namespace, + arguments: dict[str, int], + label: str, +) -> dict[str, Any]: + wrong = dict(arguments) + wrong["iterations"] = max(1, arguments["iterations"] - 1) + if wrong["iterations"] == arguments["iterations"]: + wrong["iterations"] += 1 + started = time.perf_counter() + private = start_executor(args.binary, wrong, args.tool_cpus, f"{label}-wrong") + model = post_model(args.url, arguments, args.max_tokens, args.timeout) + stop_executor(private) + authoritative = run_executor( + args.binary, + arguments, + args.tool_cpus, + args.timeout, + f"{label}-authoritative", + ) + validate_model_call(model, arguments) + return { + "mode": "miss", + "task_ms": (time.perf_counter() - started) * 1000.0, + "model": model, + "authoritative_tool": authoritative, + "private_result_exposed": False, + } + + +def qualify(args: argparse.Namespace) -> None: + model_affinity = read_process_affinity(args.model_pid) + overlap = sorted(set(model_affinity).intersection(args.tool_cpus)) + if overlap: + raise SystemExit(f"model/tool CPU affinity overlaps: {overlap}") + if args.threads > len(args.tool_cpus): + raise SystemExit("tool threads exceed reserved logical CPUs") + + arguments, calibration = calibrate(args) + for warmup in range(args.warmups): + run_direct_control(args, arguments, f"warmup-control-{warmup}") + run_direct_overlap(args, arguments, f"warmup-overlap-{warmup}") + + generator = random.Random(args.seed) + pairs = [] + for pair_index in range(args.pairs): + order = ["control", "speculative"] + generator.shuffle(order) + rows: dict[str, dict[str, Any]] = {} + for arm in order: + rows[arm] = ( + run_direct_control( + args, arguments, f"pair-{pair_index}-control" + ) + if arm == "control" + else run_direct_overlap( + args, arguments, f"pair-{pair_index}-speculative" + ) + ) + pairs.append({"pair_index": pair_index, "arm_order": order, **rows}) + print( + json.dumps( + { + "phase": "qualify", + "pair": pair_index + 1, + "control_ms": round(rows["control"]["task_ms"], 3), + "overlap_ms": round(rows["speculative"]["task_ms"], 3), + "speedup": round( + rows["control"]["task_ms"] + / rows["speculative"]["task_ms"], + 3, + ), + }, + sort_keys=True, + ), + flush=True, + ) + + misses = [ + run_direct_miss(args, arguments, f"miss-{index}") + for index in range(args.miss_samples) + ] + controls = [pair["control"] for pair in pairs] + speculative = [pair["speculative"] for pair in pairs] + control_task = statistics.median(row["task_ms"] for row in controls) + speculative_task = statistics.median( + row["task_ms"] for row in speculative + ) + control_model = statistics.median( + row["model"]["model_compute_ms"] for row in controls + ) + speculative_model = statistics.median( + row["model"]["model_compute_ms"] for row in speculative + ) + miss_model = statistics.median( + row["model"]["model_compute_ms"] for row in misses + ) + slowdown_percent = 100.0 * ( + max(speculative_model, miss_model) / control_model - 1.0 + ) + expected_checksum = controls[0]["tool"]["result"]["checksum"] + canonical_model_identity = { + ( + row["model"]["tool_call_sha256"], + row["model"]["assistant_content_sha256"], + row["model"]["completion_tokens"], + ) + for row in controls + speculative + misses + } + all_tools_equal = all( + row["tool"]["result"]["checksum"] == expected_checksum + and row["tool"]["result"]["cpu_affinity"] == args.tool_cpus + for row in controls + speculative + ) and all( + row["authoritative_tool"]["result"]["checksum"] == expected_checksum + for row in misses + ) + ds4_active = all( + row["model"]["accept_rate"] > 0 + for row in controls + speculative + misses + ) + speedup = control_task / speculative_task + checks = { + "disjoint_cpu_affinity": not overlap, + "identical_model_outputs": len(canonical_model_identity) == 1, + "identical_tool_outputs": all_tools_equal, + "ds4_active": ds4_active, + "model_slowdown": slowdown_percent <= args.max_model_slowdown_percent, + "direct_speedup": speedup >= args.min_qualification_speedup, + "private_miss_result_hidden": all( + not row["private_result_exposed"] for row in misses + ), + } + passed = all(checks.values()) + profile = { + "profile_status": "qualified" if passed else "rejected", + "executor": "child_process_cpu_affinity", + "profile_kind": "disjoint_strix_cpu_sparse_compute", + "qualification": { + "host": "lucebox5", + "model_cpu_affinity": model_affinity, + "tool_cpu_affinity": args.tool_cpus, + "checks": checks, + }, + "path_summary": { + "100": { + "accelerator_relation": "non_accelerator", + "decode_interference_qualified": passed, + "hit": { + "control_task_mean_ms": statistics.fmean( + row["task_ms"] for row in controls + ), + "speculative_task_mean_ms": statistics.fmean( + row["task_ms"] for row in speculative + ), + "model_slowdown_percent": slowdown_percent, + }, + "miss": { + "control_task_mean_ms": statistics.fmean( + row["task_ms"] for row in controls + ), + "speculative_task_mean_ms": statistics.fmean( + row["task_ms"] for row in misses + ), + "model_slowdown_percent": slowdown_percent, + }, + } + }, + } + summary = { + "pairs": len(pairs), + "control_task_p50_ms": control_task, + "overlap_task_p50_ms": speculative_task, + "direct_exact_hit_speedup": speedup, + "control_model_compute_p50_ms": control_model, + "overlap_model_compute_p50_ms": speculative_model, + "model_compute_slowdown_percent": slowdown_percent, + "control_tool_wall_p50_ms": statistics.median( + row["tool"]["wall_ms"] for row in controls + ), + "overlap_tool_wall_p50_ms": statistics.median( + row["tool"]["wall_ms"] for row in speculative + ), + "miss_task_p50_ms": statistics.median( + row["task_ms"] for row in misses + ), + "median_accept_rate": statistics.median( + row["model"]["accept_rate"] + for row in controls + speculative + misses + ), + "checks": checks, + "passed": passed, + } + report = { + "phase": "qualification", + "host": "lucebox5", + "config": report_config(args, arguments), + "model_pid": args.model_pid, + "model_cpu_affinity": model_affinity, + "tool_cpu_affinity": args.tool_cpus, + "calibration": calibration, + "summary": summary, + "profile": profile, + "pairs": pairs, + "misses": misses, + } + write_report(args.output, report) + if passed: + write_report(args.profile_output, profile) + print(json.dumps(summary, indent=2, sort_keys=True), flush=True) + if not passed: + raise SystemExit("CPU-lane qualification failed") + + +def native_control( + args: argparse.Namespace, + arguments: dict[str, int], + label: str, +) -> dict[str, Any]: + direct = run_direct_control(args, arguments, label) + tool_result = direct["tool"]["result"] + return { + "mode": "control", + "task_ms": direct["task_ms"], + **direct["model"], + "tool_wall_ms": direct["tool"]["wall_ms"], + "tool_compute_ms": float(tool_result["compute_ms"]), + "tool_checksum": tool_result["checksum"], + "tool_cpu_affinity": tool_result["cpu_affinity"], + } + + +def native_speculative( + args: argparse.Namespace, + arguments: dict[str, int], + *, + prediction: dict[str, int] | None = None, +) -> dict[str, Any]: + row = post_model( + args.url, + arguments, + args.max_tokens, + args.timeout, + prediction=prediction or arguments, + ) + validate_model_call(row, arguments) + metadata = row["speculation"] if isinstance(row["speculation"], dict) else {} + tool_result = metadata.get("result", {}) + return { + "mode": "speculative", + "task_ms": row["request_wall_ms"], + **row, + "tool_wall_ms": float(metadata.get("executor_wall_ms", math.nan)), + "tool_compute_ms": float(tool_result.get("compute_ms", math.nan)), + "tool_checksum": tool_result.get("checksum"), + "tool_cpu_affinity": tool_result.get("cpu_affinity"), + } + + +def native_qwen_control( + args: argparse.Namespace, + arguments: dict[str, int], + label: str, +) -> dict[str, Any]: + started = time.perf_counter() + model = post_model( + args.url, + arguments, + args.max_tokens, + args.timeout, + automatic_prediction=False, + tool_choice="required", + ) + tool = run_executor( + args.binary, + arguments, + args.tool_cpus, + args.timeout, + f"{label}-tool", + ) + validate_model_call(model, arguments) + result = tool["result"] + return { + "mode": "control", + "task_ms": (time.perf_counter() - started) * 1000.0, + **model, + "tool_wall_ms": float(tool["wall_ms"]), + "tool_compute_ms": float(result["compute_ms"]), + "tool_checksum": result["checksum"], + "tool_cpu_affinity": result["cpu_affinity"], + "prediction_hit": False, + "predictor_wall_ms": 0.0, + } + + +def native_qwen_speculative( + args: argparse.Namespace, + arguments: dict[str, int], + label: str, +) -> dict[str, Any]: + started = time.perf_counter() + model = post_model( + args.url, + arguments, + args.max_tokens, + args.timeout, + automatic_prediction=True, + tool_choice="required", + ) + validate_model_call(model, arguments) + metadata = model["speculation"] if isinstance( + model["speculation"], dict + ) else {} + prediction_hit = metadata.get("status") == "hit" + if prediction_hit: + tool_result = metadata.get("result") + if not isinstance(tool_result, dict): + raise RuntimeError("automatic hit did not expose a tool result") + tool_wall_ms = float(metadata.get("executor_wall_ms", math.nan)) + else: + # This is the real miss path: discard the private speculative result, + # then execute the authoritative model call normally. + fallback = run_executor( + args.binary, + arguments, + args.tool_cpus, + args.timeout, + f"{label}-fallback", + ) + tool_result = fallback["result"] + tool_wall_ms = float(fallback["wall_ms"]) + return { + "mode": "qwen_speculative", + "task_ms": (time.perf_counter() - started) * 1000.0, + **model, + "tool_wall_ms": tool_wall_ms, + "tool_compute_ms": float(tool_result["compute_ms"]), + "tool_checksum": tool_result["checksum"], + "tool_cpu_affinity": tool_result["cpu_affinity"], + "prediction_hit": prediction_hit, + "predictor_wall_ms": float(metadata.get("predictor_wall_ms", 0.0)), + "prediction_source": metadata.get("prediction_source"), + "prediction_status": metadata.get("status"), + "prediction_reason": metadata.get("reason"), + } + + +def summarize_native( + pairs: list[dict[str, Any]], resamples: int, seed: int +) -> dict[str, Any]: + controls = [pair["control"] for pair in pairs] + speculative = [pair["speculative"] for pair in pairs] + control_task = statistics.median(row["task_ms"] for row in controls) + speculative_task = statistics.median( + row["task_ms"] for row in speculative + ) + control_model = statistics.median( + row["model_compute_ms"] for row in controls + ) + speculative_model = statistics.median( + row["model_compute_ms"] for row in speculative + ) + control_tool = statistics.median( + row["tool_compute_ms"] for row in controls + ) + speculative_tool = statistics.median( + row["tool_compute_ms"] for row in speculative + ) + paired_speedups = [ + float(pair["control"]["task_ms"]) + / float(pair["speculative"]["task_ms"]) + for pair in pairs + ] + return { + "pairs": len(pairs), + "control_task_p50_ms": control_task, + "control_task_p95_ms": percentile( + (row["task_ms"] for row in controls), 0.95 + ), + "control_task_max_ms": max(row["task_ms"] for row in controls), + "speculative_task_p50_ms": speculative_task, + "speculative_task_p95_ms": percentile( + (row["task_ms"] for row in speculative), 0.95 + ), + "speculative_task_max_ms": max( + row["task_ms"] for row in speculative + ), + "exact_hit_speedup": control_task / speculative_task, + "paired_speedup_p05": percentile(paired_speedups, 0.05), + "paired_speedup_min": min(paired_speedups), + "exact_hit_speedup_bootstrap_95ci": bootstrap_speedup_ci( + pairs, resamples, seed + ), + "task_latency_reduction_percent": 100.0 + * (control_task - speculative_task) + / control_task, + "control_model_compute_p50_ms": control_model, + "speculative_model_compute_p50_ms": speculative_model, + "model_compute_slowdown_percent": 100.0 + * (speculative_model / control_model - 1.0), + "control_tool_compute_p50_ms": control_tool, + "speculative_tool_compute_p50_ms": speculative_tool, + "tool_compute_slowdown_percent": 100.0 + * (speculative_tool / control_tool - 1.0), + "latency_match_ratio": min(control_model, control_tool) + / max(control_model, control_tool), + "ideal_zero_interference_speedup_ceiling": ( + control_model + control_tool + ) + / max(control_model, control_tool), + "median_decode_tokens_per_sec": statistics.median( + row["decode_tokens_per_sec"] for row in controls + speculative + ), + "median_accept_rate": statistics.median( + row["accept_rate"] for row in controls + speculative + ), + "native_hits": sum( + isinstance(row["speculation"], dict) + and row["speculation"].get("status") == "hit" + for row in speculative + ), + "all_calls_identical": all( + pair["control"]["tool_call_sha256"] + == pair["speculative"]["tool_call_sha256"] + for pair in pairs + ), + "all_model_outputs_identical": all( + pair["control"]["assistant_content_sha256"] + == pair["speculative"]["assistant_content_sha256"] + and pair["control"]["completion_tokens"] + == pair["speculative"]["completion_tokens"] + for pair in pairs + ), + "all_tool_outputs_equivalent": all( + pair["control"]["tool_checksum"] + == pair["speculative"]["tool_checksum"] + and pair["control"]["tool_cpu_affinity"] + == pair["speculative"]["tool_cpu_affinity"] + for pair in pairs + ), + } + + +def native(args: argparse.Namespace) -> None: + props = get_json(props_url(args.url), args.timeout) + tool_props = props.get("tool_speculation") + if not isinstance(tool_props, dict) or not tool_props.get("enabled"): + raise SystemExit("server tool speculation is not enabled") + expected_props = { + "execution_mode": "child_process_cpu_affinity", + "profile_status": "qualified", + "compute_isolation": "disjoint_cpu_affinity", + "cpu_affinity_isolated": True, + "preserves_token_speculation": True, + } + for key, expected in expected_props.items(): + if tool_props.get(key) != expected: + raise SystemExit( + f"server tool_speculation.{key}={tool_props.get(key)!r}, " + f"expected {expected!r}" + ) + if tool_props.get("tool_cpu_affinity") != args.tool_cpus: + raise SystemExit("server tool CPU affinity differs from benchmark") + model_affinity = tool_props.get("model_cpu_affinity") + if not isinstance(model_affinity, list) or set(model_affinity) & set( + args.tool_cpus + ): + raise SystemExit("server model/tool CPU affinity is not disjoint") + + arguments = expected_arguments( + args.rows, + args.nonzeros_per_row, + args.iterations, + args.threads, + args.tool_seed, + ) + for warmup in range(args.warmups): + native_control(args, arguments, f"native-warm-control-{warmup}") + row = native_speculative(args, arguments) + if (row["speculation"] or {}).get("status") != "hit": + raise RuntimeError("native speculative warmup did not commit") + + generator = random.Random(args.seed) + pairs = [] + for pair_index in range(args.pairs): + order = ["control", "speculative"] + generator.shuffle(order) + rows: dict[str, dict[str, Any]] = {} + for arm in order: + rows[arm] = ( + native_control( + args, arguments, f"native-pair-{pair_index}-control" + ) + if arm == "control" + else native_speculative(args, arguments) + ) + pairs.append({"pair_index": pair_index, "arm_order": order, **rows}) + print( + json.dumps( + { + "phase": "native", + "pair": pair_index + 1, + "control_ms": round(rows["control"]["task_ms"], 3), + "speculative_ms": round( + rows["speculative"]["task_ms"], 3 + ), + "speedup": round( + rows["control"]["task_ms"] + / rows["speculative"]["task_ms"], + 3, + ), + "status": (rows["speculative"]["speculation"] or {}).get( + "status" + ), + }, + sort_keys=True, + ), + flush=True, + ) + + wrong = dict(arguments) + wrong["iterations"] = max(1, arguments["iterations"] - 1) + if wrong["iterations"] == arguments["iterations"]: + wrong["iterations"] += 1 + miss = native_speculative(args, arguments, prediction=wrong) + miss_metadata = miss["speculation"] or {} + miss_check = { + "passed": miss_metadata.get("status") == "miss" + and miss_metadata.get("reason") == "invocation_mismatch" + and "result" not in miss_metadata + and all( + miss["assistant_content_sha256"] + == pair["control"]["assistant_content_sha256"] + and miss["completion_tokens"] + == pair["control"]["completion_tokens"] + for pair in pairs + ), + "status": miss_metadata.get("status"), + "reason": miss_metadata.get("reason"), + "private_result_exposed": "result" in miss_metadata, + } + summary = summarize_native(pairs, args.bootstrap_resamples, args.seed) + correctness_passed = ( + summary["native_hits"] == args.pairs + and summary["all_calls_identical"] + and summary["all_model_outputs_identical"] + and summary["all_tool_outputs_equivalent"] + and miss_check["passed"] + and all( + pair[arm]["accept_rate"] > 0 + for pair in pairs + for arm in ("control", "speculative") + ) + ) + ci_low = summary["exact_hit_speedup_bootstrap_95ci"][0] + checks = { + "correctness": correctness_passed, + "strong_sequential_baseline": True, + "exact_hit_speedup": summary["exact_hit_speedup"] >= args.min_speedup, + "speedup_ci_low": ci_low >= args.min_speedup_ci_low, + "model_slowdown": summary["model_compute_slowdown_percent"] + <= args.max_model_slowdown_percent, + } + production_gate = { + "passed": all(checks.values()), + "checks": checks, + "thresholds": { + "min_exact_hit_speedup": args.min_speedup, + "min_speedup_ci_low": args.min_speedup_ci_low, + "max_model_slowdown_percent": args.max_model_slowdown_percent, + }, + } + report = { + "phase": "native_engine", + "host": "lucebox5", + "config": report_config(args, arguments), + "server_snapshot": { + "runtime": props.get("runtime"), + "speculative": props.get("speculative"), + "tool_speculation": tool_props, + }, + "methodology": { + "control": "model request followed by the identical CPU-pinned sparse tool", + "speculative": "engine starts the identical CPU-pinned sparse tool before DS4 generation", + "commit": "result exposed only after exact canonical call match", + "pairing": "randomized arm order within every warm pair", + }, + "correctness_passed": correctness_passed, + "production_gate": production_gate, + "miss_check": miss_check, + "summary": summary, + "pairs": pairs, + } + write_report(args.output, report) + print( + json.dumps( + { + "correctness_passed": correctness_passed, + "production_gate": production_gate, + "miss_check": miss_check, + "summary": summary, + }, + indent=2, + sort_keys=True, + ), + flush=True, + ) + if not production_gate["passed"]: + raise SystemExit("native CPU tool-speculation production gate failed") + + +def native_qwen(args: argparse.Namespace) -> None: + props = get_json(props_url(args.url), args.timeout) + tool_props = props.get("tool_speculation") + if not isinstance(tool_props, dict) or not tool_props.get("enabled"): + raise SystemExit("server tool speculation is not enabled") + if not tool_props.get("automatic_prediction_enabled"): + raise SystemExit("server automatic Qwen prediction is not enabled") + if tool_props.get("execution_mode") != "child_process_cpu_affinity": + raise SystemExit("automatic benchmark requires the isolated CPU executor") + if tool_props.get("tool_cpu_affinity") != args.tool_cpus: + raise SystemExit("server tool CPU affinity differs from benchmark") + + arguments = expected_arguments( + args.rows, + args.nonzeros_per_row, + args.iterations, + args.threads, + args.tool_seed, + ) + for warmup in range(args.warmups): + native_qwen_control( + args, arguments, f"qwen-warm-{warmup}-control" + ) + native_qwen_speculative( + args, arguments, f"qwen-warm-{warmup}-speculative" + ) + + generator = random.Random(args.seed) + pairs: list[dict[str, Any]] = [] + for pair_index in range(args.pairs): + order = ["control", "speculative"] + generator.shuffle(order) + rows: dict[str, dict[str, Any]] = {} + for arm in order: + rows[arm] = ( + native_qwen_control( + args, arguments, f"qwen-{pair_index}-control" + ) + if arm == "control" + else native_qwen_speculative( + args, arguments, f"qwen-{pair_index}-speculative" + ) + ) + pairs.append({"pair_index": pair_index, "arm_order": order, **rows}) + print(json.dumps({ + "phase": "native-qwen", + "pair": pair_index + 1, + "control_ms": round(rows["control"]["task_ms"], 3), + "speculative_ms": round(rows["speculative"]["task_ms"], 3), + "speedup": round( + rows["control"]["task_ms"] / + rows["speculative"]["task_ms"], 3), + "prediction_status": rows["speculative"]["prediction_status"], + "predictor_ms": round( + rows["speculative"]["predictor_wall_ms"], 3), + }, sort_keys=True), flush=True) + + summary = summarize_native(pairs, args.bootstrap_resamples, args.seed) + speculative = [pair["speculative"] for pair in pairs] + hits = sum(row["prediction_hit"] for row in speculative) + summary.update({ + "qwen_prediction_hits": hits, + "qwen_prediction_hit_rate": hits / len(speculative), + "qwen_predictor_p50_ms": statistics.median( + row["predictor_wall_ms"] for row in speculative + ), + "qwen_predictor_p95_ms": percentile( + (row["predictor_wall_ms"] for row in speculative), 0.95 + ), + "qwen_prediction_source_valid": all( + row["prediction_source"] == NATIVE_PREDICTION_SOURCE + for row in speculative + ), + }) + checks = { + "all_model_outputs_identical": summary["all_model_outputs_identical"], + "all_calls_identical": summary["all_calls_identical"], + "all_tool_outputs_equivalent": summary["all_tool_outputs_equivalent"], + "ds4_active": summary["median_accept_rate"] > 0, + "prediction_source": summary["qwen_prediction_source_valid"], + "prediction_hit_rate": + summary["qwen_prediction_hit_rate"] >= args.min_prediction_hit_rate, + "speedup": summary["exact_hit_speedup"] >= args.min_speedup, + "speedup_ci_low": + summary["exact_hit_speedup_bootstrap_95ci"][0] >= + args.min_speedup_ci_low, + "speedup_p05": + summary["paired_speedup_p05"] >= args.min_speedup_p05, + "model_slowdown": summary["model_compute_slowdown_percent"] <= + args.max_model_slowdown_percent, + } + report = { + "phase": "native_qwen_engine", + "host": "lucebox5", + "config": report_config(args, arguments), + "methodology": { + "control": "DS4 generation, then the authoritative CPU-pinned tool", + "speculative": "Qwen3-0.6B predicts the call; the engine launches the private CPU-pinned tool before DS4 finishes", + "prediction_cost": "included in speculative request wall time", + "miss_cost": "included by running the authoritative tool after every miss", + "commit": "exact canonical function name and arguments", + "semantic_token_injection": False, + }, + "server_snapshot": {"tool_speculation": tool_props}, + "production_gate": { + "passed": all(checks.values()), + "checks": checks, + "thresholds": { + "min_speedup": args.min_speedup, + "min_speedup_ci_low": args.min_speedup_ci_low, + "min_speedup_p05": args.min_speedup_p05, + "min_prediction_hit_rate": args.min_prediction_hit_rate, + "max_model_slowdown_percent": + args.max_model_slowdown_percent, + }, + }, + "summary": summary, + "pairs": pairs, + } + write_report(args.output, report) + print(json.dumps({ + "production_gate": report["production_gate"], + "summary": summary, + }, indent=2, sort_keys=True), flush=True) + if not report["production_gate"]["passed"]: + raise SystemExit("automatic Qwen tool-speculation gate failed") + + +def report_config( + args: argparse.Namespace, arguments: dict[str, int] +) -> dict[str, Any]: + return { + "url": args.url, + "binary": str(args.binary.resolve()), + "tool_arguments": arguments, + "fixed_sparse_shape": { + "rows": args.rows, + "nonzeros_per_row": args.nonzeros_per_row, + "threads": args.threads, + "seed": args.tool_seed, + }, + "tool_cpus": args.tool_cpus, + "max_tokens": args.max_tokens, + "pairs": args.pairs, + "warmups": args.warmups, + "seed": args.seed, + } + + +def write_report(path: Path | None, value: dict[str, Any]) -> None: + if path is None: + return + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(value, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def add_common_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "--url", default="http://127.0.0.1:18145/v1/chat/completions" + ) + parser.add_argument("--binary", type=Path, required=True) + parser.add_argument("--tool-cpus", type=parse_cpu_list, required=True) + parser.add_argument("--rows", type=int, default=4096) + parser.add_argument("--nonzeros-per-row", type=int, default=16) + parser.add_argument("--threads", type=int, default=2) + parser.add_argument("--tool-seed", type=int, default=731) + parser.add_argument("--max-tokens", type=int, default=32) + parser.add_argument("--pairs", type=int, default=20) + parser.add_argument("--warmups", type=int, default=2) + parser.add_argument("--timeout", type=float, default=180.0) + parser.add_argument("--seed", type=int, default=814) + parser.add_argument("--max-model-slowdown-percent", type=float, default=5.0) + parser.add_argument("--output", type=Path) + + +def main() -> None: + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="phase", required=True) + + qualify_parser = subparsers.add_parser("qualify") + add_common_arguments(qualify_parser) + qualify_parser.add_argument("--model-pid", type=int, required=True) + qualify_parser.add_argument("--initial-iterations", type=int, default=20) + qualify_parser.add_argument("--calibration-steps", type=int, default=5) + qualify_parser.add_argument("--calibration-model-samples", type=int, default=3) + qualify_parser.add_argument("--calibration-tool-samples", type=int, default=2) + qualify_parser.add_argument("--miss-samples", type=int, default=5) + qualify_parser.add_argument("--min-qualification-speedup", type=float, default=1.70) + qualify_parser.add_argument("--profile-output", type=Path, required=True) + + native_parser = subparsers.add_parser("native") + add_common_arguments(native_parser) + native_parser.add_argument("--iterations", type=int, required=True) + native_parser.add_argument("--bootstrap-resamples", type=int, default=20_000) + native_parser.add_argument("--min-speedup", type=float, default=1.80) + native_parser.add_argument("--min-speedup-ci-low", type=float, default=1.70) + + qwen_parser = subparsers.add_parser("native-qwen") + add_common_arguments(qwen_parser) + qwen_parser.add_argument("--iterations", type=int, required=True) + qwen_parser.add_argument("--bootstrap-resamples", type=int, default=20_000) + qwen_parser.add_argument("--min-speedup", type=float, default=1.60) + qwen_parser.add_argument("--min-speedup-ci-low", type=float, default=1.50) + qwen_parser.add_argument("--min-speedup-p05", type=float, default=1.25) + qwen_parser.add_argument("--min-prediction-hit-rate", type=float, default=0.90) + + args = parser.parse_args() + if not args.binary.is_file(): + parser.error(f"executor binary does not exist: {args.binary}") + positive = [ + args.rows, + args.nonzeros_per_row, + args.threads, + args.max_tokens, + args.pairs, + args.warmups, + args.timeout, + ] + if any(value <= 0 for value in positive): + parser.error("counts and timeouts must be positive") + if args.tool_seed < 0 or args.max_model_slowdown_percent < 0: + parser.error("seed and slowdown threshold must be non-negative") + if args.phase == "qualify": + if ( + args.model_pid <= 0 + or args.initial_iterations <= 0 + or args.calibration_steps <= 0 + or args.calibration_model_samples <= 0 + or args.calibration_tool_samples <= 0 + or args.miss_samples <= 0 + or args.min_qualification_speedup <= 1.0 + ): + parser.error("qualification settings must be positive") + qualify(args) + else: + if ( + args.iterations <= 0 + or args.bootstrap_resamples <= 0 + or args.min_speedup <= 1.0 + or args.min_speedup_ci_low <= 1.0 + or args.min_speedup_ci_low > args.min_speedup + ): + parser.error("native benchmark settings are invalid") + if args.phase == "native-qwen": + if ( + not 0.0 <= args.min_prediction_hit_rate <= 1.0 + or args.min_speedup_p05 <= 1.0 + or args.min_speedup_p05 > args.min_speedup + ): + parser.error("native Qwen benchmark settings are invalid") + native_qwen(args) + else: + native(args) + + +if __name__ == "__main__": + main() diff --git a/optimizations/ooo_spec_lucebox5_cpu/benchmark_trace_compiled_workflows.py b/optimizations/ooo_spec_lucebox5_cpu/benchmark_trace_compiled_workflows.py new file mode 100644 index 000000000..208bb7861 --- /dev/null +++ b/optimizations/ooo_spec_lucebox5_cpu/benchmark_trace_compiled_workflows.py @@ -0,0 +1,1639 @@ +#!/usr/bin/env python3 +"""Benchmark no-training trace-compiled tool workflows on Lucebox5. + +The benchmark compares three end-to-end agent paths over 10--20 real model +tool calls: + +* ``stage_batched``: DS4 authorizes one batch per dependency stage and every + call in that stage executes concurrently. This gives the control parallel + tools without speculating or compiling the complete workflow. +* ``compiled``: a recurring, side-effect-free trace is exposed as one generated + macro tool; DS4 authorizes it once and independent branches run in parallel. +* ``speculative``: the PR's Qwen predictor proposes the macro call and the + engine starts its compiled graph before DS4 finishes. The private result is + committed only after an exact name-and-arguments match. + +The workflow compiler is learned from prior successful traces. It performs no +model training and refuses literals, side effects, ambiguous dataflow, or +inconsistent control flow. Every arm uses the production DS4+DSpark endpoint, +includes all model turns and tools in wall time, and must produce the same +underlying calls, tool results, and exact final answer. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import random +import re +import statistics +import subprocess +import time +from concurrent.futures import Future, ThreadPoolExecutor +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from benchmark_cpu_tool_speculation import ( + NATIVE_PREDICTION_SOURCE, + get_json, + normalize_tool_call, + parse_cpu_list, + percentile, + post_json, + props_url, +) +from bfcl_replay_tool_executor import ( + PROTOCOL, + call_ref, + call_sha256, + canonical_call, +) + + +@dataclass(frozen=True) +class ArgumentBinding: + source: str + key: str + + +@dataclass(frozen=True) +class PatternStep: + tool: str + arguments: tuple[tuple[str, ArgumentBinding], ...] + + +@dataclass(frozen=True) +class CompiledPattern: + steps: tuple[PatternStep, ...] + root_fields: tuple[str, ...] + training_traces: int + + @property + def macro_name(self) -> str: + # Semantic names are materially more reliable for tool-call generation + # than opaque hashes. The registry remains keyed by the full pattern + # fingerprint, so this user-facing alias does not define identity. + subject = self.steps[0].tool.rsplit("_", 1)[-1] + return f"execute_{subject}_workflows" + + @property + def fingerprint(self) -> str: + payload = json.dumps( + [ + [ + step.tool, + [[name, binding.source, binding.key] for name, binding in step.arguments], + ] + for step in self.steps + ], + separators=(",", ":"), + ) + return hashlib.sha256(payload.encode()).hexdigest() + + def instantiate( + self, root: dict[str, str], previous_result: dict[str, Any] | None, index: int + ) -> dict[str, Any]: + step = self.steps[index] + arguments: dict[str, Any] = {} + for name, binding in step.arguments: + source: dict[str, Any] + if binding.source == "root": + source = root + elif binding.source == "previous_result" and previous_result is not None: + source = previous_result + else: + raise RuntimeError( + f"cannot resolve {binding.source}.{binding.key} for {step.tool}" + ) + if binding.key not in source: + raise RuntimeError( + f"missing {binding.source}.{binding.key} for {step.tool}" + ) + arguments[name] = source[binding.key] + return {"name": step.tool, "arguments": arguments} + + def simulate(self, root: dict[str, str]) -> list[dict[str, Any]]: + calls = [] + previous: dict[str, Any] | None = None + for index in range(len(self.steps)): + call = self.instantiate(root, previous, index) + calls.append(call) + previous = simulated_tool_result(call) + return calls + + def macro_tool( + self, max_items: int, workflow_ref: str | None = None + ) -> dict[str, Any]: + if workflow_ref is not None: + parameters = { + "type": "object", + "properties": { + "workflow_ref": { + "type": "string", + "enum": [workflow_ref], + "description": "Bound workflow instance for this request.", + } + }, + "required": ["workflow_ref"], + "additionalProperties": False, + } + else: + properties = {field: {"type": "string"} for field in self.root_fields} + parameters = { + "type": "object", + "properties": { + "customers": { + "type": "array", + "items": { + "type": "object", + "properties": properties, + "required": list(self.root_fields), + "additionalProperties": False, + }, + "minItems": 1, + "maxItems": max_items, + } + }, + "required": ["customers"], + "additionalProperties": False, + } + return { + "type": "function", + "function": { + "name": self.macro_name, + "description": ( + "Execute the validated five-step customer workflow independently " + "for every requested customer." + ), + "parameters": parameters, + }, + } + + +def simulated_tool_result(call: dict[str, Any]) -> dict[str, Any]: + return { + "call_ref": call_ref(call), + "call_sha256": call_sha256(call), + "tool_name": call["name"], + "side_effects": False, + } + + +def _infer_binding( + value: Any, + root: dict[str, str], + previous_result: dict[str, Any] | None, +) -> ArgumentBinding: + previous_matches = [] + if previous_result is not None: + previous_matches = [key for key, candidate in previous_result.items() if candidate == value] + root_matches = [key for key, candidate in root.items() if candidate == value] + if len(previous_matches) == 1: + return ArgumentBinding("previous_result", previous_matches[0]) + if len(root_matches) == 1: + return ArgumentBinding("root", root_matches[0]) + raise ValueError("argument is literal or has ambiguous trace dataflow") + + +def mine_pattern(traces: list[dict[str, Any]]) -> CompiledPattern: + if len(traces) < 2: + raise ValueError("at least two successful traces are required") + signatures = [] + root_fields: set[str] = set() + for trace in traces: + root = trace.get("root") + calls = trace.get("calls") + results = trace.get("results") + if not isinstance(root, dict) or not isinstance(calls, list) or not isinstance(results, list): + raise ValueError("trace must contain root, calls, and results") + if not calls or len(calls) != len(results): + raise ValueError("trace calls and results must be non-empty and aligned") + signature = [] + previous: dict[str, Any] | None = None + for call, result in zip(calls, results, strict=True): + if not isinstance(call, dict) or not isinstance(call.get("arguments"), dict): + raise ValueError("trace call is malformed") + if not isinstance(result, dict) or result.get("side_effects") is not False: + raise ValueError("only explicitly side-effect-free traces can be compiled") + if result.get("call_sha256") != call_sha256(call): + raise ValueError("trace result does not match its call") + bindings = [] + for name, value in sorted(call["arguments"].items()): + binding = _infer_binding(value, root, previous) + if binding.source == "root": + root_fields.add(binding.key) + bindings.append((name, binding)) + signature.append(PatternStep(str(call.get("name", "")), tuple(bindings))) + previous = result + signatures.append(tuple(signature)) + if any(signature != signatures[0] for signature in signatures[1:]): + raise ValueError("training traces do not share one control/data-flow pattern") + if not root_fields: + raise ValueError("compiled workflow exposes no request-bound arguments") + return CompiledPattern( + steps=signatures[0], + root_fields=tuple(sorted(root_fields)), + training_traces=len(traces), + ) + + +def load_training_traces(path: Path, required_steps: int) -> list[dict[str, Any]]: + report = json.loads(path.read_text(encoding="utf-8")) + compact_traces = report.get("traces") if isinstance(report, dict) else None + if isinstance(compact_traces, list): + traces = [ + trace + for trace in compact_traces + if isinstance(trace, dict) + and isinstance(trace.get("calls"), list) + and len(trace["calls"]) == required_steps + ] + if len(traces) < 2: + raise ValueError( + "training trace file contains fewer than two complete traces" + ) + return traces + pairs = report.get("pairs") if isinstance(report, dict) else None + if not isinstance(pairs, list): + raise ValueError("training report has no pairs") + traces = [] + for pair in pairs: + task = pair.get("task") if isinstance(pair, dict) else None + control = pair.get("control") if isinstance(pair, dict) else None + steps = control.get("steps") if isinstance(control, dict) else None + if ( + not isinstance(task, dict) + or not isinstance(steps, list) + or len(steps) != required_steps + or not control.get("all_calls_correct") + ): + continue + traces.append( + { + "root": { + "customer_email": task["customer_email"], + "destination": task["destination"], + }, + "calls": [step["call"] for step in steps], + "results": [step["tool_result"] for step in steps], + } + ) + if len(traces) < 2: + raise ValueError("training report contains fewer than two complete correct traces") + return traces + + +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def alphabetic_identifier(value: int) -> str: + """Encode an integer with letters so copy accuracy is not biased by 0/o.""" + prefix = "warm" if value < 0 else "task" + number = abs(value) + encoded = "" + while True: + encoded = chr(ord("a") + number % 26) + encoded + number = number // 26 - 1 + if number < 0: + break + return prefix + encoded + + +def make_task( + index: int, branch_count: int, pattern: CompiledPattern +) -> dict[str, Any]: + destinations = ("Rome", "Milan", "Turin", "Bologna", "Florence", "Naples") + items = [] + used_refs = [set() for _ in pattern.steps] + candidate = max(index, 0) * 100 + while len(items) < branch_count and candidate < max(index, 0) * 100 + 20_000: + root = { + "customer_email": ( + f"agent-{alphabetic_identifier(index)}-" + f"{alphabetic_identifier(candidate)}@example.test" + ), + "destination": destinations[(index + candidate) % len(destinations)], + } + refs = [call_ref(call) for call in pattern.simulate(root)] + if all(reference not in used_refs[step] for step, reference in enumerate(refs)): + items.append(root) + for step, reference in enumerate(refs): + used_refs[step].add(reference) + candidate += 1 + if len(items) != branch_count: + raise RuntimeError("could not construct collision-free workflow branches") + return { + "id": f"trace_compiled_{index:03d}", + "workflow_ref": f"workflow_{alphabetic_identifier(index)}", + "items": items, + "branch_count": branch_count, + "call_count": branch_count * len(pattern.steps), + } + + +def request_content(task: dict[str, Any]) -> str: + rendered = "; ".join( + f"{item['customer_email']} to {item['destination']}" for item in task["items"] + ) + return f"Customers: {rendered}." + + +def workflow_reference(task: dict[str, Any], pattern: CompiledPattern) -> str: + del pattern + workflow_ref = task.get("workflow_ref") + if not isinstance(workflow_ref, str) or re.fullmatch( + r"workflow_[a-z]+", workflow_ref + ) is None: + raise ValueError("task has no valid request-scoped workflow_ref") + return workflow_ref + + +def write_workflow_registry( + path: Path, pattern: CompiledPattern, tasks: list[dict[str, Any]] +) -> None: + workflows = { + workflow_reference(task, pattern): { + "pattern_fingerprint": pattern.fingerprint, + "items": task["items"], + } + for task in tasks + } + if len(workflows) != len(tasks): + raise ValueError("workflow_ref collision in request-scoped registry") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps( + { + "schema_version": 1, + "pattern_fingerprint": pattern.fingerprint, + "workflows": workflows, + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + + +def parse_request_customers(content: str) -> list[dict[str, str]]: + prefix = "Customers: " + if not content.startswith(prefix) or not content.endswith("."): + raise ValueError("request does not use the validated customer-list format") + customers = [] + for entry in content[len(prefix) : -1].split("; "): + match = re.fullmatch(r"([^\s;]+) to ([A-Za-z][A-Za-z -]*)", entry) + if match is None: + raise ValueError("request customer entry is malformed") + customers.append( + {"customer_email": match.group(1), "destination": match.group(2)} + ) + if not customers: + raise ValueError("request contains no customers") + return customers + + +def stage_batch_tool( + pattern: CompiledPattern, + step_index: int, + max_items: int, + stage_ref: str | None = None, +) -> dict[str, Any]: + step = pattern.steps[step_index] + if stage_ref is not None: + parameters = { + "type": "object", + "properties": { + "stage_ref": { + "type": "string", + "enum": [stage_ref], + "description": "Bound batch for this workflow stage.", + } + }, + "required": ["stage_ref"], + "additionalProperties": False, + } + else: + properties = {name: {"type": "string"} for name, _ in step.arguments} + parameters = { + "type": "object", + "properties": { + "calls": { + "type": "array", + "items": { + "type": "object", + "properties": properties, + "required": list(properties), + "additionalProperties": False, + }, + "minItems": 1, + "maxItems": max_items, + } + }, + "required": ["calls"], + "additionalProperties": False, + } + return { + "type": "function", + "function": { + "name": f"batch_{step.tool}", + "description": ( + f"Run {step.tool} once for every item. Calls execute concurrently " + "and results preserve input order." + ), + "parameters": parameters, + }, + } + + +def stage_batched_messages( + task: dict[str, Any], pattern: CompiledPattern +) -> list[dict[str, Any]]: + del pattern + system = ( + "The scheduler exposes exactly one currently-ready batch tool at a time. " + "Call only that declared tool once and copy its bound stage_ref exactly. " + "The bound batch contains every customer in input order. Do not invent or " + "name future tools, and emit no prose." + ) + return [ + {"role": "system", "content": system}, + {"role": "user", "content": request_content(task)}, + ] + + +def macro_messages(task: dict[str, Any], pattern: CompiledPattern) -> list[dict[str, Any]]: + return [ + { + "role": "system", + "content": ( + "Use the workflow tool exactly once for every requested customer. " + "Copy its bound workflow_ref exactly. No prose." + ), + }, + {"role": "user", "content": request_content(task)}, + ] + + +def stage_reference(task: dict[str, Any], pattern: CompiledPattern, index: int) -> str: + stage_names = ("one", "two", "three", "four", "five") + if not 0 <= index < len(stage_names): + raise ValueError("stage index is outside the compiled workflow") + return f"{workflow_reference(task, pattern)}_stage_{stage_names[index]}" + + +def model_observation(result: dict[str, Any], wall_ms: float) -> dict[str, Any]: + choices = result.get("choices") + message = choices[0].get("message") if isinstance(choices, list) and choices else None + if not isinstance(message, dict): + raise RuntimeError("model response has no assistant message") + raw_calls = message.get("tool_calls") + if not isinstance(raw_calls, list): + raw_calls = [] + calls = [] + for raw in raw_calls: + function = raw.get("function") if isinstance(raw, dict) else None + if not isinstance(function, dict): + raise RuntimeError("model emitted a malformed tool call") + arguments = function.get("arguments") + if isinstance(arguments, str): + arguments = json.loads(arguments) + if not isinstance(function.get("name"), str) or not isinstance(arguments, dict): + raise RuntimeError("model emitted invalid tool name or arguments") + call_id = raw.get("id") + if not isinstance(call_id, str) or not call_id: + raise RuntimeError("model tool call has no id") + calls.append( + { + "id": call_id, + "call": {"name": function["name"], "arguments": arguments}, + } + ) + content = message.get("content") + if not isinstance(content, str): + content = "" + usage = result.get("usage") if isinstance(result.get("usage"), dict) else {} + timings = usage.get("timings") if isinstance(usage.get("timings"), dict) else {} + assistant_message = dict(message) + assistant_message["role"] = "assistant" + assistant_message["content"] = content + content_format_call = False + if not calls and content: + parsed_call = normalize_tool_call(result) + if parsed_call is not None: + content_format_call = True + call_id = "call_content_" + hashlib.sha256( + canonical_call(parsed_call).encode() + ).hexdigest()[:16] + calls.append({"id": call_id, "call": parsed_call}) + assistant_message = { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": call_id, + "type": "function", + "function": { + "name": parsed_call["name"], + "arguments": json.dumps( + parsed_call["arguments"], + sort_keys=True, + separators=(",", ":"), + ), + }, + } + ], + } + return { + "request_wall_ms": wall_ms, + "model_compute_ms": float(timings.get("prefill_ms", 0.0)) + + float(timings.get("decode_ms", 0.0)), + "prefill_ms": float(timings.get("prefill_ms", 0.0)), + "decode_ms": float(timings.get("decode_ms", 0.0)), + "decode_tokens_per_sec": float(timings.get("decode_tokens_per_sec", 0.0)), + "cache_hit": bool(timings.get("cache_hit", False)), + "cached_prefix_tokens": int(timings.get("cached_prefix_tokens", 0)), + "completion_tokens": int(usage.get("completion_tokens", 0)), + "accept_rate": float(usage.get("accept_rate", 0.0)), + "content": content, + "content_sha256": hashlib.sha256(content.encode()).hexdigest(), + "assistant_message": assistant_message, + "calls": calls, + "content_format_call": content_format_call, + } + + +def post_turn( + args: argparse.Namespace, + messages: list[dict[str, Any]], + tools: list[dict[str, Any]], + tool_choice: Any, + max_tokens: int, + *, + automatic_tool_speculation: bool = False, +) -> dict[str, Any]: + result, wall_ms = post_json( + args.url, + { + "model": "dflash", + "messages": messages, + "tools": tools, + "tool_choice": tool_choice, + "temperature": 0, + "seed": args.seed, + "max_tokens": max_tokens, + "stream": False, + "automatic_tool_speculation": automatic_tool_speculation, + }, + args.timeout, + ) + observation = model_observation(result, wall_ms) + observation["speculation"] = result.get("dflash_tool_speculation") + return observation + + +def execute_tool_safe( + binary: Path, + call: dict[str, Any], + cpus: list[int], + timeout: float, + request_id: str, +) -> dict[str, Any]: + request = { + "protocol": PROTOCOL, + "request_id": request_id, + "resource_percentage": 100, + "accelerator_relation": "non_accelerator", + "cpu_affinity": cpus, + "cpu_affinity_isolated": True, + "call": call, + } + command = [str(binary), "--dflash-tool-spec-v1"] + if os.name == "posix" and Path("/usr/bin/taskset").is_file(): + command = ["/usr/bin/taskset", "-c", ",".join(map(str, cpus)), *command] + started = time.perf_counter() + process = subprocess.run( + command, + input=json.dumps(request, separators=(",", ":")) + "\n", + text=True, + capture_output=True, + timeout=timeout, + check=False, + ) + wall_ms = (time.perf_counter() - started) * 1_000.0 + if process.returncode != 0: + raise RuntimeError( + f"tool executor exited {process.returncode}: {process.stderr.strip()}" + ) + envelope = json.loads(process.stdout) + result = envelope.get("result") if isinstance(envelope, dict) else None + if not envelope.get("ok") or not isinstance(result, dict): + raise RuntimeError(f"tool executor returned invalid data: {envelope!r}") + if ( + result.get("call_sha256") != call_sha256(call) + or result.get("call_ref") != call_ref(call) + or result.get("tool_name") != call.get("name") + or result.get("side_effects") is not False + ): + raise RuntimeError("tool result does not exactly match the read-only call") + return {"wall_ms": wall_ms, "result": result} + + +def run_branch( + args: argparse.Namespace, + pattern: CompiledPattern, + root: dict[str, str], + label: str, +) -> dict[str, Any]: + started = time.perf_counter() + steps = [] + previous: dict[str, Any] | None = None + for index in range(len(pattern.steps)): + call = pattern.instantiate(root, previous, index) + tool = execute_tool_safe( + args.binary, + call, + args.tool_cpus, + args.timeout, + f"{label}-step-{index + 1}", + ) + previous = tool["result"] + steps.append({"call": call, "tool_result": previous, "tool_wall_ms": tool["wall_ms"]}) + return { + "root": root, + "steps": steps, + "final_ref": steps[-1]["tool_result"]["call_ref"], + "wall_ms": (time.perf_counter() - started) * 1_000.0, + } + + +def start_graph( + args: argparse.Namespace, + pattern: CompiledPattern, + items: list[dict[str, str]], + label: str, +) -> dict[str, Any]: + pool = ThreadPoolExecutor(max_workers=len(items), thread_name_prefix="tool-graph") + started = time.perf_counter() + futures: list[Future[dict[str, Any]]] = [ + pool.submit(run_branch, args, pattern, root, f"{label}-branch-{index}") + for index, root in enumerate(items) + ] + return {"pool": pool, "futures": futures, "started": started} + + +def finish_graph(handle: dict[str, Any], timeout: float) -> dict[str, Any]: + pool: ThreadPoolExecutor = handle["pool"] + futures: list[Future[dict[str, Any]]] = handle["futures"] + try: + branches = [future.result(timeout=timeout) for future in futures] + finally: + pool.shutdown(wait=True, cancel_futures=True) + return { + "branches": branches, + "wall_ms": (time.perf_counter() - float(handle["started"])) * 1_000.0, + } + + +def expected_final(branches: list[dict[str, Any]]) -> str: + return "workflow_complete:" + ",".join(branch["final_ref"] for branch in branches) + + +def final_answer_correct(content: str, expected: str) -> bool: + """Accept the receipt literally or as the equivalent one-field JSON object.""" + content = content.strip() + if content == expected: + return True + prefix = "workflow_complete:" + if not expected.startswith(prefix): + return False + receipt = expected[len(prefix) :] + if content == receipt: + return True + try: + parsed = json.loads(content) + except json.JSONDecodeError: + return False + return parsed == {"workflow_complete": receipt} + + +def post_final( + args: argparse.Namespace, + expected: str, +) -> dict[str, Any]: + """Measure the same minimal final-response turn for every benchmark arm.""" + return post_turn( + args, + [ + { + "role": "system", + "content": ( + "Return the opaque workflow receipt from the user message exactly. " + "Do not explain, reformat, or add any characters." + ), + }, + {"role": "user", "content": expected}, + ], + [], + "none", + args.final_max_tokens, + ) + + +def flatten_graph_calls(graph: dict[str, Any]) -> list[str]: + return [ + canonical_call(step["call"]) + for branch in graph["branches"] + for step in branch["steps"] + ] + + +def flatten_graph_results(graph: dict[str, Any]) -> list[str]: + return [ + step["tool_result"]["call_sha256"] + for branch in graph["branches"] + for step in branch["steps"] + ] + + +def run_stage_batched( + args: argparse.Namespace, + task: dict[str, Any], + pattern: CompiledPattern, + label: str, +) -> dict[str, Any]: + """Run a strong non-speculative baseline with parallel calls per stage.""" + started = time.perf_counter() + stage_messages = stage_batched_messages(task, pattern) + current_tools: list[dict[str, Any]] = [] + branches = [ + {"root": root, "steps": [], "final_ref": ""} for root in task["items"] + ] + previous_results: list[dict[str, Any] | None] = [None] * len(branches) + turns = [] + exposed_tool_wait_ms = 0.0 + + for step_index, step in enumerate(pattern.steps): + expected_calls = [ + pattern.instantiate(branch["root"], previous_results[index], step_index) + for index, branch in enumerate(branches) + ] + batch_name = f"batch_{step.tool}" + stage_ref = stage_reference(task, pattern, step_index) + current_tools = [ + stage_batch_tool(pattern, step_index, args.max_branches, stage_ref) + ] + expected_batch = { + "name": batch_name, + "arguments": {"stage_ref": stage_ref}, + } + model = post_turn( + args, + stage_messages, + current_tools, + {"type": "function", "function": {"name": batch_name}}, + args.call_max_tokens, + ) + if len(model["calls"]) != 1 or model["calls"][0]["call"] != expected_batch: + raise RuntimeError( + f"{task['id']} stage {step_index + 1}: batch call " + f"{[item['call'] for item in model['calls']]!r} != " + f"{expected_batch!r}; content={model['content']!r}" + ) + stage_started = time.perf_counter() + with ThreadPoolExecutor( + max_workers=len(expected_calls), thread_name_prefix="batched-tool-stage" + ) as pool: + futures = [ + pool.submit( + execute_tool_safe, + args.binary, + call, + args.tool_cpus, + args.timeout, + f"{label}-stage-{step_index}-branch-{branch_index}", + ) + for branch_index, call in enumerate(expected_calls) + ] + stage_tools = [future.result(timeout=args.timeout) for future in futures] + exposed_tool_wait_ms += (time.perf_counter() - stage_started) * 1_000.0 + + for branch_index, (call, tool) in enumerate( + zip(expected_calls, stage_tools, strict=True) + ): + result = tool["result"] + previous_results[branch_index] = result + branches[branch_index]["steps"].append( + { + "call": call, + "tool_result": result, + "tool_wall_ms": tool["wall_ms"], + } + ) + turns.append(model) + + for branch in branches: + branch["final_ref"] = branch["steps"][-1]["tool_result"]["call_ref"] + expected = expected_final(branches) + final = post_final(args, expected) + all_turns = [*turns, final] + graph = {"branches": branches} + return { + "task_ms": (time.perf_counter() - started) * 1_000.0, + "model_turns": len(all_turns), + "call_turns": turns, + "final": final, + "expected_final": expected, + "final_correct": final_answer_correct(final["content"], expected), + "graph": graph, + "underlying_calls": flatten_graph_calls(graph), + "tool_results": flatten_graph_results(graph), + "model_compute_ms": sum(turn["model_compute_ms"] for turn in all_turns), + "decode_ms": sum(turn["decode_ms"] for turn in all_turns), + "completion_tokens": sum(turn["completion_tokens"] for turn in all_turns), + "exposed_tool_wait_ms": exposed_tool_wait_ms, + "all_ds4_active": all(turn["accept_rate"] > 0.0 for turn in turns), + } + + +def macro_result_message( + pattern: CompiledPattern, + graph: dict[str, Any], + tool_call_id: str, +) -> dict[str, Any]: + content = { + "workflow": pattern.macro_name, + "call_count": sum(len(branch["steps"]) for branch in graph["branches"]), + "items": [ + { + **branch["root"], + "final_ref": branch["final_ref"], + } + for branch in graph["branches"] + ], + "side_effects": False, + } + return { + "role": "tool", + "tool_call_id": tool_call_id, + "name": pattern.macro_name, + "content": json.dumps(content, sort_keys=True, separators=(",", ":")), + } + + +def graph_from_speculative_result( + metadata: dict[str, Any], pattern: CompiledPattern, call: dict[str, Any] +) -> dict[str, Any]: + result = metadata.get("result") + if not isinstance(result, dict): + raise RuntimeError("engine speculation hit has no compiled workflow result") + branches = result.get("branches") + if ( + result.get("call_sha256") != call_sha256(call) + or result.get("call_ref") != call_ref(call) + or result.get("tool_name") != pattern.macro_name + or result.get("workflow_fingerprint") != pattern.fingerprint + or result.get("side_effects") is not False + or not isinstance(branches, list) + ): + raise RuntimeError("engine returned an invalid compiled workflow result") + return {"branches": branches, "wall_ms": float(result.get("elapsed_ms", 0.0))} + + +def run_macro( + args: argparse.Namespace, + task: dict[str, Any], + pattern: CompiledPattern, + speculative: bool, + label: str, +) -> dict[str, Any]: + started = time.perf_counter() + messages = macro_messages(task, pattern) + workflow_ref = workflow_reference(task, pattern) + tools = [pattern.macro_tool(args.max_branches, workflow_ref)] + parsed_items = parse_request_customers(messages[-1]["content"]) + if parsed_items != task["items"]: + raise RuntimeError("event extractor did not reproduce structured request data") + expected_call = { + "name": pattern.macro_name, + "arguments": {"workflow_ref": workflow_ref}, + } + model = post_turn( + args, + messages, + tools, + "required", + args.macro_max_tokens, + automatic_tool_speculation=speculative, + ) + if len(model["calls"]) != 1: + raise RuntimeError(f"{task['id']}: macro turn emitted {len(model['calls'])} calls") + emitted = model["calls"][0] + macro_correct = emitted["call"] == expected_call + if not macro_correct: + raise RuntimeError( + f"{task['id']}: macro call {emitted['call']!r} != {expected_call!r}" + ) + + metadata = model.get("speculation") if speculative else None + if speculative and not isinstance(metadata, dict): + raise RuntimeError("automatic Qwen speculation returned no engine metadata") + prediction = metadata.get("prediction") if isinstance(metadata, dict) else None + prediction_hit = ( + speculative + and metadata.get("status") == "hit" + and prediction == expected_call + ) + predictor_ms = ( + float(metadata.get("predictor_wall_ms", 0.0)) + if isinstance(metadata, dict) + else 0.0 + ) + if prediction_hit: + graph = graph_from_speculative_result(metadata, pattern, emitted["call"]) + exposed_wait_ms = float(metadata.get("commit_wait_ms", 0.0)) + else: + graph_handle = start_graph(args, pattern, task["items"], f"{label}-authoritative") + wait_started = time.perf_counter() + graph = finish_graph(graph_handle, args.timeout) + exposed_wait_ms = (time.perf_counter() - wait_started) * 1_000.0 + expected_calls = [ + canonical_call(call) + for root in task["items"] + for call in pattern.simulate(root) + ] + actual_calls = flatten_graph_calls(graph) + if actual_calls != expected_calls: + raise RuntimeError(f"{task['id']}: compiled graph diverged from learned pattern") + messages.extend( + [ + model["assistant_message"], + macro_result_message(pattern, graph, emitted["id"]), + ] + ) + expected = expected_final(graph["branches"]) + final = post_final(args, expected) + all_turns = [model, final] + return { + "task_ms": (time.perf_counter() - started) * 1_000.0, + "model_turns": len(all_turns), + "call_turns": [model], + "macro_call": emitted["call"], + "macro_correct": macro_correct, + "prediction_hit": prediction_hit, + "prediction_source": ( + metadata.get("prediction_source") if isinstance(metadata, dict) else None + ), + "prediction_status": metadata.get("status") if isinstance(metadata, dict) else None, + "prediction_reason": metadata.get("reason") if isinstance(metadata, dict) else None, + "predictor_ms": predictor_ms, + "graph": graph, + "graph_wall_ms": graph["wall_ms"], + "exposed_tool_wait_ms": exposed_wait_ms, + "underlying_calls": actual_calls, + "tool_results": flatten_graph_results(graph), + "final": final, + "expected_final": expected, + "final_correct": final_answer_correct(final["content"], expected), + "model_compute_ms": sum(turn["model_compute_ms"] for turn in all_turns), + "decode_ms": sum(turn["decode_ms"] for turn in all_turns), + "completion_tokens": sum(turn["completion_tokens"] for turn in all_turns), + "all_ds4_active": model["accept_rate"] > 0.0, + } + + +def macro_signature(arm: dict[str, Any]) -> dict[str, Any]: + turn = arm["call_turns"][0] + return { + "macro_call": canonical_call(arm["macro_call"]), + "turn_content": turn["content_sha256"], + "turn_tokens": turn["completion_tokens"], + "final_content": arm["final"]["content_sha256"], + "final_tokens": arm["final"]["completion_tokens"], + } + + +def bootstrap_speedup_ci( + pairs: list[dict[str, Any]], numerator: str, denominator: str, resamples: int, seed: int +) -> list[float]: + generator = random.Random(seed) + values = [] + for _ in range(resamples): + sample = [pairs[generator.randrange(len(pairs))] for _ in pairs] + values.append( + statistics.median( + pair[numerator]["task_ms"] / pair[denominator]["task_ms"] + for pair in sample + ) + ) + return [percentile(values, 0.025), percentile(values, 0.975)] + + +def paired_slowdown( + pairs: list[dict[str, Any]], metric: str, quantile: float +) -> float: + ratios = [ + pair["speculative"][metric] / pair["compiled"][metric] + for pair in pairs + if pair["compiled"][metric] > 0.0 + ] + return 100.0 * (percentile(ratios, quantile) - 1.0) + + +def summarize( + pairs: list[dict[str, Any]], resamples: int, seed: int +) -> dict[str, Any]: + baseline_speedups = [ + pair["stage_batched"]["task_ms"] / pair["speculative"]["task_ms"] + for pair in pairs + ] + compiled_speedups = [ + pair["compiled"]["task_ms"] / pair["speculative"]["task_ms"] for pair in pairs + ] + continuation_turns = [ + turn + for pair in pairs + for arm_name in ("stage_batched", "compiled", "speculative") + for turn in [*pair[arm_name]["call_turns"][1:], pair[arm_name]["final"]] + ] + speedup_by_call_count = {} + for call_count in sorted({pair["task"]["call_count"] for pair in pairs}): + bucket = [pair for pair in pairs if pair["task"]["call_count"] == call_count] + speedup_by_call_count[str(call_count)] = { + "tasks": len(bucket), + "stage_batched_task_p50_ms": statistics.median( + pair["stage_batched"]["task_ms"] for pair in bucket + ), + "compiled_task_p50_ms": statistics.median( + pair["compiled"]["task_ms"] for pair in bucket + ), + "speculative_task_p50_ms": statistics.median( + pair["speculative"]["task_ms"] for pair in bucket + ), + "combined_speedup_p50": statistics.median( + pair["stage_batched"]["task_ms"] + / pair["speculative"]["task_ms"] + for pair in bucket + ), + "speculation_speedup_p50": statistics.median( + pair["compiled"]["task_ms"] / pair["speculative"]["task_ms"] + for pair in bucket + ), + } + return { + "tasks": len(pairs), + "calls_per_task": [pair["task"]["call_count"] for pair in pairs], + "stage_batched_task_p50_ms": statistics.median( + pair["stage_batched"]["task_ms"] for pair in pairs + ), + "compiled_task_p50_ms": statistics.median( + pair["compiled"]["task_ms"] for pair in pairs + ), + "speculative_task_p50_ms": statistics.median( + pair["speculative"]["task_ms"] for pair in pairs + ), + "stage_batched_to_speculative_speedup_p50": statistics.median( + baseline_speedups + ), + "stage_batched_to_speculative_speedup_p05": percentile( + baseline_speedups, 0.05 + ), + "stage_batched_to_speculative_speedup_min": min(baseline_speedups), + "stage_batched_to_speculative_bootstrap_95ci": bootstrap_speedup_ci( + pairs, "stage_batched", "speculative", resamples, seed + ), + "stage_batched_to_compiled_speedup_p50": statistics.median( + pair["stage_batched"]["task_ms"] / pair["compiled"]["task_ms"] + for pair in pairs + ), + "compiled_to_speculative_speedup_p50": statistics.median(compiled_speedups), + "compiled_to_speculative_speedup_p05": percentile(compiled_speedups, 0.05), + "compiled_to_speculative_bootstrap_95ci": bootstrap_speedup_ci( + pairs, "compiled", "speculative", resamples, seed + 1 + ), + "total_wall_speedup": sum( + pair["stage_batched"]["task_ms"] for pair in pairs + ) + / sum(pair["speculative"]["task_ms"] for pair in pairs), + "stage_batched_model_turns_p50": statistics.median( + pair["stage_batched"]["model_turns"] for pair in pairs + ), + "compiled_model_turns_p50": statistics.median( + pair["compiled"]["model_turns"] for pair in pairs + ), + "stage_batched_exposed_tool_wait_p50_ms": statistics.median( + pair["stage_batched"]["exposed_tool_wait_ms"] for pair in pairs + ), + "compiled_exposed_tool_wait_p50_ms": statistics.median( + pair["compiled"]["exposed_tool_wait_ms"] for pair in pairs + ), + "speculative_exposed_tool_wait_p50_ms": statistics.median( + pair["speculative"]["exposed_tool_wait_ms"] for pair in pairs + ), + "pattern_prediction_hit_rate": sum( + pair["speculative"]["prediction_hit"] for pair in pairs + ) + / len(pairs), + "all_predictions_from_qwen": all( + pair["speculative"]["prediction_source"] == NATIVE_PREDICTION_SOURCE + for pair in pairs + ), + "predictor_p50_ms": statistics.median( + pair["speculative"]["predictor_ms"] for pair in pairs + ), + "model_compute_slowdown_p50_percent": paired_slowdown( + pairs, "model_compute_ms", 0.50 + ), + "model_compute_slowdown_p95_percent": paired_slowdown( + pairs, "model_compute_ms", 0.95 + ), + "decode_slowdown_p50_percent": paired_slowdown(pairs, "decode_ms", 0.50), + "decode_slowdown_p95_percent": paired_slowdown(pairs, "decode_ms", 0.95), + "continuation_cache_hit_rate": sum( + turn["cache_hit"] and turn["cached_prefix_tokens"] > 0 + for turn in continuation_turns + ) + / len(continuation_turns), + "speedup_by_call_count": speedup_by_call_count, + "all_calls_stable": all( + pair["stage_batched"]["underlying_calls"] + == pair["compiled"]["underlying_calls"] + == pair["speculative"]["underlying_calls"] + for pair in pairs + ), + "all_tool_results_stable": all( + pair["stage_batched"]["tool_results"] + == pair["compiled"]["tool_results"] + == pair["speculative"]["tool_results"] + for pair in pairs + ), + "macro_output_stability_rate": sum( + macro_signature(pair["compiled"]) == macro_signature(pair["speculative"]) + for pair in pairs + ) + / len(pairs), + "all_final_answers_correct": all( + pair[arm]["final_correct"] + for pair in pairs + for arm in ("stage_batched", "compiled", "speculative") + ), + "all_final_outputs_stable": all( + pair["stage_batched"]["final"]["content"].strip() + == pair["compiled"]["final"]["content"].strip() + == pair["speculative"]["final"]["content"].strip() + for pair in pairs + ), + "all_macro_calls_correct": all( + pair[arm]["macro_correct"] + for pair in pairs + for arm in ("compiled", "speculative") + ), + "all_ds4_active": all( + pair[arm]["all_ds4_active"] + for pair in pairs + for arm in ("stage_batched", "compiled", "speculative") + ), + } + + +def production_checks(summary: dict[str, Any], args: argparse.Namespace) -> dict[str, bool]: + return { + "sample_size": summary["tasks"] >= args.min_production_pairs, + "end_to_end_speedup": summary["stage_batched_to_speculative_speedup_p50"] + >= args.min_e2e_speedup, + "end_to_end_ci": summary[ + "stage_batched_to_speculative_bootstrap_95ci" + ][0] + > 1.0, + "end_to_end_tail": summary["stage_batched_to_speculative_speedup_p05"] + >= args.min_e2e_speedup_p05, + "speculation_incremental_gain": summary["compiled_to_speculative_speedup_p50"] + >= args.min_incremental_speedup, + "speculation_incremental_ci": summary[ + "compiled_to_speculative_bootstrap_95ci" + ][0] + > 1.0, + "prediction_hit_rate": summary["pattern_prediction_hit_rate"] == 1.0, + "prediction_source": summary["all_predictions_from_qwen"], + "model_slowdown_p50": summary["model_compute_slowdown_p50_percent"] + <= args.max_model_slowdown_percent, + "model_slowdown_p95": summary["model_compute_slowdown_p95_percent"] + <= args.max_model_slowdown_p95_percent, + "decode_slowdown_p50": summary["decode_slowdown_p50_percent"] + <= args.max_decode_slowdown_percent, + "decode_slowdown_p95": summary["decode_slowdown_p95_percent"] + <= args.max_decode_slowdown_p95_percent, + "prefix_cache_configured": summary["prefix_cache_configured"], + "calls_stable": summary["all_calls_stable"], + "tool_results_stable": summary["all_tool_results_stable"], + "macro_outputs_stable": summary["macro_output_stability_rate"] == 1.0, + "final_answers_correct": summary["all_final_answers_correct"], + "final_outputs_stable": summary["all_final_outputs_stable"], + "macro_calls_correct": summary["all_macro_calls_correct"], + "ds4_active": summary["all_ds4_active"], + } + + +def compact_arm(arm: dict[str, Any]) -> dict[str, Any]: + """Keep auditable outputs and timings without embedding the full graph.""" + fields = ( + "task_ms", + "model_turns", + "model_compute_ms", + "decode_ms", + "completion_tokens", + "exposed_tool_wait_ms", + "all_ds4_active", + "final_correct", + "graph_wall_ms", + "macro_call", + "macro_correct", + "prediction_hit", + "prediction_reason", + "prediction_source", + "prediction_status", + "predictor_ms", + ) + compact = {field: arm[field] for field in fields if field in arm} + for field in ("underlying_calls", "tool_results"): + values = arm.get(field) + if isinstance(values, list): + compact[f"{field}_count"] = len(values) + compact[f"{field}_sha256"] = hashlib.sha256( + json.dumps(values, separators=(",", ":")).encode() + ).hexdigest() + final = arm.get("final") + if isinstance(final, dict): + compact["final"] = { + field: final[field] + for field in ( + "content_sha256", + "completion_tokens", + "accept_rate", + ) + if field in final + } + return compact + + +def compact_pair(pair: dict[str, Any]) -> dict[str, Any]: + task = pair["task"] + return { + "pair_index": pair["pair_index"], + "task": { + field: task[field] + for field in ("id", "branch_count", "call_count") + if field in task + }, + "arm_order": pair["arm_order"], + **{ + arm: compact_arm(pair[arm]) + for arm in ("stage_batched", "compiled", "speculative") + }, + } + + +def load_partial_pairs( + path: Path, + measured_tasks: list[dict[str, Any]], + arm_orders: list[list[str]], +) -> list[dict[str, Any]]: + """Load and strictly validate a checkpoint before resuming a long run.""" + checkpoint = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(checkpoint, dict): + raise ValueError("benchmark checkpoint is not an object") + pairs = checkpoint.get("pairs") + if ( + checkpoint.get("schema_version") != 1 + or checkpoint.get("complete") is not False + or not isinstance(pairs, list) + or len(pairs) > len(measured_tasks) + ): + raise ValueError("benchmark checkpoint is not resumable") + for index, pair in enumerate(pairs): + if ( + not isinstance(pair, dict) + or pair.get("pair_index") != index + or pair.get("task") != measured_tasks[index] + or pair.get("arm_order") != arm_orders[index] + or not all( + isinstance(pair.get(arm), dict) + for arm in ("stage_batched", "compiled", "speculative") + ) + ): + raise ValueError(f"benchmark checkpoint pair {index} does not match this run") + for arm in ("stage_batched", "compiled", "speculative"): + result = pair[arm] + final = result.get("final") + expected = result.get("expected_final") + if ( + not isinstance(final, dict) + or not isinstance(final.get("content"), str) + or not isinstance(expected, str) + ): + raise ValueError( + f"benchmark checkpoint pair {index} has an invalid {arm} final" + ) + result["final_correct"] = final_answer_correct( + final["content"], expected + ) + return pairs + + +def validate_args(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None: + if not args.binary.is_file() or not os.access(args.binary, os.X_OK): + parser.error("--binary must be an executable tool adapter") + if not args.training_report.is_file(): + parser.error("--training-report must be an existing trace file or report") + if ( + args.pairs <= 0 + or args.warmup_tasks < 0 + or not 2 <= args.min_branches <= args.max_branches <= 4 + or args.timeout <= 0 + or min(args.call_max_tokens, args.macro_max_tokens, args.final_max_tokens) <= 0 + or args.bootstrap_resamples <= 0 + or args.min_production_pairs < 2 + or args.min_e2e_speedup <= 1.0 + or args.min_e2e_speedup_p05 <= 1.0 + or args.min_incremental_speedup <= 1.0 + ): + parser.error("benchmark counts and thresholds are invalid") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--url", default="http://127.0.0.1:18145/v1/chat/completions") + parser.add_argument("--binary", type=Path, required=True) + parser.add_argument("--training-report", type=Path, required=True) + parser.add_argument( + "--workflow-registry", + type=Path, + default=Path(__file__).with_name("results") / "trace-workflow-registry.json", + ) + parser.add_argument("--tool-cpus", type=parse_cpu_list, default="14-15,30-31") + parser.add_argument("--pairs", type=int, default=6) + parser.add_argument("--warmup-tasks", type=int, default=1) + parser.add_argument("--min-branches", type=int, default=2) + parser.add_argument("--max-branches", type=int, default=4) + parser.add_argument("--call-max-tokens", type=int, default=160) + parser.add_argument("--macro-max-tokens", type=int, default=512) + parser.add_argument("--final-max-tokens", type=int, default=96) + parser.add_argument("--timeout", type=float, default=180.0) + parser.add_argument("--seed", type=int, default=814) + parser.add_argument("--bootstrap-resamples", type=int, default=20_000) + parser.add_argument("--min-production-pairs", type=int, default=6) + parser.add_argument("--min-e2e-speedup", type=float, default=2.0) + parser.add_argument("--min-e2e-speedup-p05", type=float, default=1.5) + parser.add_argument("--min-incremental-speedup", type=float, default=1.05) + parser.add_argument("--max-model-slowdown-percent", type=float, default=1.0) + parser.add_argument("--max-model-slowdown-p95-percent", type=float, default=5.0) + parser.add_argument("--max-decode-slowdown-percent", type=float, default=1.0) + parser.add_argument("--max-decode-slowdown-p95-percent", type=float, default=5.0) + parser.add_argument( + "--resume-partial", + action="store_true", + help="resume the strictly matching .partial checkpoint", + ) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + args.binary = args.binary.resolve() + args.training_report = args.training_report.resolve() + args.workflow_registry = args.workflow_registry.resolve() + validate_args(parser, args) + + traces = load_training_traces(args.training_report, required_steps=5) + pattern = mine_pattern(traces) + props = get_json(props_url(args.url), args.timeout) + prefix_cache = props.get("prefix_cache") + if not isinstance(prefix_cache, dict) or int(prefix_cache.get("capacity", 0)) <= 0: + raise SystemExit("prefix cache is not enabled") + tool_speculation = props.get("tool_speculation") + required_tool_props = { + "enabled": True, + "automatic_prediction_enabled": True, + "prediction_source": NATIVE_PREDICTION_SOURCE, + "predictor_schedule": "before-model", + "execution_mode": "child_process_cpu_affinity", + "cpu_affinity_isolated": True, + "preserves_token_speculation": True, + } + if not isinstance(tool_speculation, dict): + raise SystemExit("engine tool speculation is not enabled") + for key, expected in required_tool_props.items(): + if tool_speculation.get(key) != expected: + raise SystemExit( + f"engine tool_speculation.{key}={tool_speculation.get(key)!r}, " + f"expected {expected!r}" + ) + if pattern.macro_name not in tool_speculation.get("allowed_tools", []): + raise SystemExit(f"engine does not allow compiled macro {pattern.macro_name!r}") + + branch_span = args.max_branches - args.min_branches + 1 + warmup_tasks = [ + make_task(-1000 - index, args.min_branches, pattern) + for index in range(args.warmup_tasks) + ] + measured_tasks = [ + make_task( + pair_index, + args.min_branches + pair_index % branch_span, + pattern, + ) + for pair_index in range(args.pairs) + ] + write_workflow_registry( + args.workflow_registry, pattern, [*warmup_tasks, *measured_tasks] + ) + generator = random.Random(args.seed) + arm_orders = [] + for _ in measured_tasks: + order = ["stage_batched", "compiled", "speculative"] + generator.shuffle(order) + arm_orders.append(order) + partial_output = args.output.with_suffix(args.output.suffix + ".partial") + pairs = ( + load_partial_pairs(partial_output, measured_tasks, arm_orders) + if args.resume_partial + else [] + ) + if pairs: + print(json.dumps({"resumed_pairs": len(pairs)}), flush=True) + else: + for warmup, task in enumerate(warmup_tasks): + run_stage_batched(args, task, pattern, f"warm-{warmup}-stage-batched") + run_macro(args, task, pattern, False, f"warm-{warmup}-compiled") + run_macro(args, task, pattern, True, f"warm-{warmup}-speculative") + + for pair_index in range(len(pairs), len(measured_tasks)): + task = measured_tasks[pair_index] + order = arm_orders[pair_index] + arms = {} + for arm in order: + if arm == "stage_batched": + arms[arm] = run_stage_batched( + args, task, pattern, f"pair-{pair_index}-stage-batched" + ) + else: + arms[arm] = run_macro( + args, + task, + pattern, + arm == "speculative", + f"pair-{pair_index}-{arm}", + ) + pair = {"pair_index": pair_index, "task": task, "arm_order": order, **arms} + pairs.append(pair) + partial_output.parent.mkdir(parents=True, exist_ok=True) + partial_output.write_text( + json.dumps({"schema_version": 1, "complete": False, "pairs": pairs}, indent=2) + + "\n", + encoding="utf-8", + ) + print( + json.dumps( + { + "pair": pair_index + 1, + "calls": task["call_count"], + "order": order, + "stage_batched_ms": round( + arms["stage_batched"]["task_ms"], 1 + ), + "compiled_ms": round(arms["compiled"]["task_ms"], 1), + "speculative_ms": round(arms["speculative"]["task_ms"], 1), + "end_to_end_speedup": round( + arms["stage_batched"]["task_ms"] + / arms["speculative"]["task_ms"], + 3, + ), + "incremental_speedup": round( + arms["compiled"]["task_ms"] + / arms["speculative"]["task_ms"], + 3, + ), + "prediction_hit": arms["speculative"]["prediction_hit"], + "correct": all(arms[arm]["final_correct"] for arm in arms), + }, + sort_keys=True, + ), + flush=True, + ) + + summary = summarize(pairs, args.bootstrap_resamples, args.seed) + ending_props = get_json(props_url(args.url), args.timeout) + ending_prefix_cache = ending_props.get("prefix_cache") + if not isinstance(ending_prefix_cache, dict): + raise RuntimeError("prefix cache disappeared during the benchmark") + summary["prefix_cache_lifetime_hit_delta"] = int( + ending_prefix_cache.get("lifetime_hits", 0) + ) - int(prefix_cache.get("lifetime_hits", 0)) + summary["prefix_cache_configured"] = int(prefix_cache.get("capacity", 0)) > 0 + checks = production_checks(summary, args) + report = { + "schema_version": 1, + "host": "lucebox5", + "feature": "no-training trace-compiled speculative tool graphs", + "pattern": { + "macro_name": pattern.macro_name, + "fingerprint": pattern.fingerprint, + "training_traces": pattern.training_traces, + "training_report": str(args.training_report), + "training_report_sha256": file_sha256(args.training_report), + "workflow_registry": str(args.workflow_registry), + "workflow_registry_sha256": file_sha256(args.workflow_registry), + "root_fields": list(pattern.root_fields), + "steps": [ + { + "tool": step.tool, + "arguments": { + name: {"source": binding.source, "key": binding.key} + for name, binding in step.arguments + }, + } + for step in pattern.steps + ], + "model_training": False, + "side_effects_allowed": False, + }, + "workload": { + "tasks": args.pairs, + "branches_per_task": f"{args.min_branches}-{args.max_branches}", + "calls_per_task": f"{args.min_branches * len(pattern.steps)}-" + f"{args.max_branches * len(pattern.steps)}", + "dependency": "five serial calls per branch; branches are independent", + "tool_adapter": "deterministic read-only 2-second API replay", + }, + "methodology": { + "stage_batched": ( + "DS4+DSpark sees only the currently-ready typed batch, authorizes " + "one per dependency stage, runs its calls concurrently, and receives " + "a compact rolling state instead of replaying old tool history" + ), + "compiled": ( + "one DS4+DSpark macro authorization; independent branches execute " + "concurrently on the Strix CPU lane" + ), + "speculative": ( + "Qwen predicts the trace-derived macro through the engine; its CPU " + "graph overlaps DS4+DSpark and commits only on an exact call match" + ), + "arm_order": "randomized per task", + "measured_wall_time": "request through all tools and exact final answer", + "oracle_prediction": False, + "argument_binding": ( + "the harness binds validated structured inputs to a request-scoped " + "workflow_ref before either model runs" + ), + "model_seed": args.seed, + "warmup_tasks": args.warmup_tasks, + }, + "server_snapshot": { + "prefix_cache_before": prefix_cache, + "prefix_cache_after": ending_prefix_cache, + "tool_speculation": tool_speculation, + "model": props.get("model"), + }, + "production_gate": { + "passed": all(checks.values()), + "checks": checks, + "thresholds": { + "min_e2e_speedup": args.min_e2e_speedup, + "min_e2e_speedup_p05": args.min_e2e_speedup_p05, + "min_incremental_speedup": args.min_incremental_speedup, + "min_production_pairs": args.min_production_pairs, + "max_model_slowdown_percent": args.max_model_slowdown_percent, + "max_model_slowdown_p95_percent": args.max_model_slowdown_p95_percent, + "max_decode_slowdown_percent": args.max_decode_slowdown_percent, + "max_decode_slowdown_p95_percent": args.max_decode_slowdown_p95_percent, + }, + }, + "summary": summary, + "pairs": [compact_pair(pair) for pair in pairs], + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + if report["production_gate"]["passed"]: + partial_output.unlink(missing_ok=True) + print( + json.dumps( + {"production_gate": report["production_gate"], "summary": summary}, + indent=2, + sort_keys=True, + ), + flush=True, + ) + return 0 if report["production_gate"]["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/optimizations/ooo_spec_lucebox5_cpu/bfcl_replay_tool_executor.py b/optimizations/ooo_spec_lucebox5_cpu/bfcl_replay_tool_executor.py new file mode 100755 index 000000000..e1d79a101 --- /dev/null +++ b/optimizations/ooo_spec_lucebox5_cpu/bfcl_replay_tool_executor.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""Read-only executor for end-to-end BFCL tool-speculation replay. + +BFCL functions are specifications rather than deployable APIs. This adapter +therefore performs no external action: it waits for a fixed, documented API +latency and returns a deterministic digest of the canonical call. The engine's +own allowlist remains the authority for which predictions may reach it. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import sys +import time +from typing import Any + + +PROTOCOL = "dflash.tool-speculation.v1" +LATENCY_MS = 2_000 +REFERENCE_WORDS = ( + "amber", "azure", "cedar", "coral", "gold", "ivory", "maple", "olive", + "pearl", "plum", "sable", "silver", "teal", "violet", "willow", "jade", +) + + +def canonical_call(call: dict[str, Any]) -> str: + return json.dumps(call, sort_keys=True, separators=(",", ":")) + + +def call_sha256(call: dict[str, Any]) -> str: + return hashlib.sha256(canonical_call(call).encode()).hexdigest() + + +def call_ref(call: dict[str, Any]) -> str: + return REFERENCE_WORDS[int(call_sha256(call)[:8], 16) % len(REFERENCE_WORDS)] + + +def execute(request: dict[str, Any]) -> dict[str, Any]: + if request.get("protocol") != PROTOCOL: + raise ValueError("unsupported protocol") + call = request.get("call") + if not isinstance(call, dict): + raise ValueError("call must be an object") + name = call.get("name") + arguments = call.get("arguments") + if not isinstance(name, str) or not name: + raise ValueError("tool name must be a non-empty string") + if not isinstance(arguments, dict): + raise ValueError("tool arguments must be an object") + + expected = request.get("cpu_affinity") or [] + if not isinstance(expected, list) or not all( + isinstance(cpu, int) and cpu >= 0 for cpu in expected + ): + raise ValueError("cpu_affinity must contain non-negative integers") + observed = sorted(os.sched_getaffinity(0)) if hasattr( + os, "sched_getaffinity" + ) else [] + if expected and observed != sorted(set(expected)): + raise ValueError("observed CPU affinity does not match request") + + canonical = {"name": name, "arguments": arguments} + digest = call_sha256(canonical) + reference = call_ref(canonical) + started = time.perf_counter() + time.sleep(LATENCY_MS / 1_000.0) + elapsed = (time.perf_counter() - started) * 1_000.0 + return { + "ok": True, + "result": { + "call_sha256": digest, + "call_ref": reference, + "tool_name": name, + "latency_ms": LATENCY_MS, + "elapsed_ms": elapsed, + "cpu_affinity": observed, + "side_effects": False, + }, + } + + +def main() -> int: + if sys.argv[1:] != ["--dflash-tool-spec-v1"]: + print("expected --dflash-tool-spec-v1", file=sys.stderr) + return 2 + try: + line = sys.stdin.readline() + if not line: + raise ValueError("missing request") + request = json.loads(line) + if not isinstance(request, dict): + raise ValueError("request must be an object") + print(json.dumps(execute(request), separators=(",", ":")), flush=True) + return 0 + except (OSError, ValueError, json.JSONDecodeError) as error: + print(str(error), file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/optimizations/ooo_spec_lucebox5_cpu/build_cpu_sparse_executor.sh b/optimizations/ooo_spec_lucebox5_cpu/build_cpu_sparse_executor.sh new file mode 100755 index 000000000..4e48334e7 --- /dev/null +++ b/optimizations/ooo_spec_lucebox5_cpu/build_cpu_sparse_executor.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +source_file="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/cpu_sparse_tool_executor.cpp" +json_include="${JSON_INCLUDE:-}" +output="${1:-$(dirname "$source_file")/cpu_sparse_tool_executor}" + +if [[ -z "$json_include" ]]; then + printf 'JSON_INCLUDE must point to the directory containing nlohmann/json.hpp\n' >&2 + exit 2 +fi +if [[ ! -f "$json_include/nlohmann/json.hpp" ]]; then + printf 'missing JSON header: %s/nlohmann/json.hpp\n' "$json_include" >&2 + exit 2 +fi + +g++ -std=c++17 -O3 -DNDEBUG -pthread \ + -I"$json_include" "$source_file" -o "$output" diff --git a/optimizations/ooo_spec_lucebox5_cpu/cpu_sparse_tool_executor.cpp b/optimizations/ooo_spec_lucebox5_cpu/cpu_sparse_tool_executor.cpp new file mode 100644 index 000000000..95be7dd7b --- /dev/null +++ b/optimizations/ooo_spec_lucebox5_cpu/cpu_sparse_tool_executor.cpp @@ -0,0 +1,259 @@ +// Deterministic read-only sparse-compute adapter for +// dflash.tool-speculation.v1 qualification. +// +// This is a benchmark tool, not an application-specific tool. It provides a +// reproducible CPU-bound workload whose exact result can be compared between +// sequential and speculative execution. The engine pins this child before it +// releases the JSON request, and the adapter verifies the observed mask. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__linux__) +#include +#include +#endif + +using json = nlohmann::json; + +namespace { + +constexpr const char * kProtocol = "dflash.tool-speculation.v1"; +constexpr const char * kToolName = "benchmark_cpu_sparse"; +constexpr int kRows = 4096; +constexpr int kNonzerosPerRow = 16; +constexpr int kThreads = 2; +constexpr uint64_t kSeed = 731; + +uint64_t splitmix64(uint64_t & state) { + uint64_t value = (state += 0x9e3779b97f4a7c15ULL); + value = (value ^ (value >> 30U)) * 0xbf58476d1ce4e5b9ULL; + value = (value ^ (value >> 27U)) * 0x94d049bb133111ebULL; + return value ^ (value >> 31U); +} + +std::vector observed_affinity() { +#if defined(__linux__) + cpu_set_t mask; + CPU_ZERO(&mask); + if (::sched_getaffinity(0, sizeof(mask), &mask) != 0) { + throw std::runtime_error("sched_getaffinity failed"); + } + std::vector cpus; + for (int cpu = 0; cpu < CPU_SETSIZE; ++cpu) { + if (CPU_ISSET(cpu, &mask)) cpus.push_back(cpu); + } + return cpus; +#else + return {}; +#endif +} + +uint64_t sparse_worker(int worker, + int rows, + int nonzeros_per_row, + int iterations, + uint64_t seed) { + const size_t entries = + static_cast(rows) * static_cast(nonzeros_per_row); + std::vector columns(entries); + std::vector values(entries); + std::vector input(static_cast(rows)); + std::vector output(static_cast(rows)); + + uint64_t state = seed ^ + (0xd6e8feb86659fd93ULL * static_cast(worker + 1)); + for (size_t index = 0; index < entries; ++index) { + columns[index] = static_cast(splitmix64(state) % rows); + values[index] = static_cast(splitmix64(state) | 1ULL); + } + for (uint32_t & value : input) { + value = static_cast(splitmix64(state)); + } + + uint64_t rolling = 0xcbf29ce484222325ULL ^ + static_cast(worker); + for (int iteration = 0; iteration < iterations; ++iteration) { + for (int row = 0; row < rows; ++row) { + uint64_t accumulator = + static_cast(iteration + 1) * 0x9e3779b1U + + static_cast(row); + const size_t start = + static_cast(row) * nonzeros_per_row; + for (int offset = 0; offset < nonzeros_per_row; ++offset) { + const size_t index = start + static_cast(offset); + accumulator += static_cast(values[index]) * + input[columns[index]]; + } + const uint32_t folded = static_cast( + accumulator ^ (accumulator >> 32U)); + output[static_cast(row)] = + folded + static_cast(row * 2654435761U); + } + input.swap(output); + rolling ^= static_cast(input[ + static_cast(iteration) % input.size()]); + rolling *= 0x100000001b3ULL; + } + for (size_t index = 0; index < input.size(); index += 17) { + rolling ^= static_cast(input[index]) + index; + rolling *= 0x100000001b3ULL; + } + return rolling; +} + +int integer_argument(const json & arguments, + const char * name, + int minimum, + int maximum) { + if (!arguments.contains(name) || !arguments[name].is_number_integer()) { + throw std::runtime_error(std::string(name) + " must be an integer"); + } + const int value = arguments[name].get(); + if (value < minimum || value > maximum) { + throw std::runtime_error( + std::string(name) + " is outside the allowed range"); + } + return value; +} + +json execute(const json & request) { + if (!request.is_object() || request.value("protocol", "") != kProtocol) { + throw std::runtime_error("unsupported protocol"); + } + if (!request.contains("call") || !request["call"].is_object() || + request["call"].value("name", "") != kToolName) { + throw std::runtime_error("only benchmark_cpu_sparse is allowed"); + } + const json & arguments = request["call"].at("arguments"); + if (!arguments.is_object()) { + throw std::runtime_error("arguments must be an object"); + } + const int rows = arguments.contains("rows") + ? integer_argument(arguments, "rows", 64, 1 << 20) : kRows; + const int nonzeros = arguments.contains("nonzeros_per_row") + ? integer_argument(arguments, "nonzeros_per_row", 1, 256) + : kNonzerosPerRow; + const int iterations = integer_argument( + arguments, "iterations", 1, 1'000'000); + const int threads = arguments.contains("threads") + ? integer_argument(arguments, "threads", 1, 64) : kThreads; + if (static_cast(rows) * static_cast(nonzeros) > + 16ULL * 1024ULL * 1024ULL) { + throw std::runtime_error("sparse matrix exceeds the 16M-entry limit"); + } + uint64_t seed = kSeed; + if (arguments.contains("seed")) { + if (!arguments["seed"].is_number_integer()) { + throw std::runtime_error("seed must be an unsigned integer"); + } + if (arguments["seed"].is_number_unsigned()) { + seed = arguments["seed"].get(); + } else { + const int64_t signed_seed = arguments["seed"].get(); + if (signed_seed < 0) { + throw std::runtime_error("seed must be an unsigned integer"); + } + seed = static_cast(signed_seed); + } + } + + std::vector expected_affinity; + if (request.contains("cpu_affinity")) { + expected_affinity = request["cpu_affinity"].get>(); + std::sort(expected_affinity.begin(), expected_affinity.end()); + expected_affinity.erase( + std::unique(expected_affinity.begin(), expected_affinity.end()), + expected_affinity.end()); + } + const std::vector affinity = observed_affinity(); + if (!expected_affinity.empty() && affinity != expected_affinity) { + throw std::runtime_error("observed CPU affinity does not match request"); + } + if (!affinity.empty() && threads > static_cast(affinity.size())) { + throw std::runtime_error("threads exceed the pinned logical CPU count"); + } + + const auto started = std::chrono::steady_clock::now(); + std::vector partial(static_cast(threads)); + std::vector pin_errors(static_cast(threads), 0); + std::vector workers; + workers.reserve(static_cast(threads)); + for (int worker = 0; worker < threads; ++worker) { + workers.emplace_back([&, worker]() { +#if defined(__linux__) + if (!affinity.empty()) { + cpu_set_t worker_mask; + CPU_ZERO(&worker_mask); + CPU_SET(affinity[static_cast(worker)], &worker_mask); + pin_errors[static_cast(worker)] = + ::pthread_setaffinity_np( + ::pthread_self(), sizeof(worker_mask), &worker_mask); + if (pin_errors[static_cast(worker)] != 0) return; + } +#endif + partial[static_cast(worker)] = sparse_worker( + worker, rows, nonzeros, iterations, seed); + }); + } + for (std::thread & worker : workers) worker.join(); + if (std::any_of(pin_errors.begin(), pin_errors.end(), + [](int error) { return error != 0; })) { + throw std::runtime_error("worker CPU pinning failed"); + } + const double compute_ms = + std::chrono::duration( + std::chrono::steady_clock::now() - started).count(); + + uint64_t checksum = 0x6a09e667f3bcc909ULL; + for (const uint64_t value : partial) { + checksum ^= value + 0x9e3779b97f4a7c15ULL + + (checksum << 6U) + (checksum >> 2U); + } + return { + {"ok", true}, + {"result", { + {"checksum", std::to_string(checksum)}, + {"compute_ms", compute_ms}, + {"rows", rows}, + {"nonzeros_per_row", nonzeros}, + {"iterations", iterations}, + {"threads", threads}, + {"seed", seed}, + {"cpu_affinity", affinity}, + {"worker_cpus", std::vector( + affinity.begin(), affinity.begin() + + std::min(affinity.size(), static_cast(threads)))}, + }}, + }; +} + +} // namespace + +int main(int argc, char ** argv) { + if (argc != 2 || std::string(argv[1]) != "--dflash-tool-spec-v1") { + std::cerr << "expected --dflash-tool-spec-v1\n"; + return 2; + } + try { + std::string line; + if (!std::getline(std::cin, line) || line.empty()) { + throw std::runtime_error("missing request"); + } + std::cout << execute(json::parse(line)).dump() << '\n'; + std::cout.flush(); + return 0; + } catch (const std::exception & exception) { + std::cerr << exception.what() << '\n'; + return 2; + } +} diff --git a/optimizations/ooo_spec_lucebox5_cpu/dflash_server_native_tool_predictor_wrapper.sh b/optimizations/ooo_spec_lucebox5_cpu/dflash_server_native_tool_predictor_wrapper.sh new file mode 100755 index 000000000..d8e118d85 --- /dev/null +++ b/optimizations/ooo_spec_lucebox5_cpu/dflash_server_native_tool_predictor_wrapper.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Preserve every argument from the qualified 0731 launcher and add the native +# Qwen3 tool-prediction lane on the Strix GPU. The qualified launcher clears +# ambient variables, so an adjacent `candidate-build` symlink is the durable +# deployment override; direct launches may still use CANDIDATE_BUILD. +wrapper_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +default_candidate="/home/lucebox5/tool-spec-cpu-20260813/engine-ooo-spec/server/build-hip-dual" +if [[ -d "${wrapper_dir}/candidate-build" ]]; then + default_candidate="${wrapper_dir}/candidate-build" +fi +CANDIDATE_BUILD="${CANDIDATE_BUILD:-${default_candidate}}" +PREDICTOR_MODEL="${PREDICTOR_MODEL:-/home/lucebox5/tool-spec-cpu-20260813/models/Qwen3-0.6B-Q8_0.gguf}" +PREDICTOR_IPC_BIN="${PREDICTOR_IPC_BIN:-${CANDIDATE_BUILD}/backend_ipc_daemon}" +PREDICTOR_GPU="${PREDICTOR_GPU:-1}" +PREDICTOR_MAX_CTX="${PREDICTOR_MAX_CTX:-4096}" +PREDICTOR_MAX_TOKENS="${PREDICTOR_MAX_TOKENS:-256}" +PREDICTOR_CONFIDENCE="${PREDICTOR_CONFIDENCE:-0.75}" +PREDICTOR_SCHEDULE="${PREDICTOR_SCHEDULE:-before-model}" +# The qualified 0731 launcher disables caches for cold throughput benchmarks. +# Tool-using agent loops need turn-boundary reuse; this later CLI flag wins +# without modifying the qualified model/DSpark arguments. +PREFIX_CACHE_SLOTS_OVERRIDE="${PREFIX_CACHE_SLOTS_OVERRIDE:-32}" + +cache_args=() +if [[ -n "${PREFIX_CACHE_SLOTS_OVERRIDE}" ]]; then + [[ "${PREFIX_CACHE_SLOTS_OVERRIDE}" =~ ^[1-9][0-9]*$ ]] || { + printf 'invalid PREFIX_CACHE_SLOTS_OVERRIDE: %s\n' \ + "${PREFIX_CACHE_SLOTS_OVERRIDE}" >&2 + exit 2 + } + (( PREFIX_CACHE_SLOTS_OVERRIDE <= 64 )) || { + printf 'PREFIX_CACHE_SLOTS_OVERRIDE exceeds the 64-slot engine limit\n' >&2 + exit 2 + } + cache_args+=(--prefix-cache-slots "${PREFIX_CACHE_SLOTS_OVERRIDE}") +fi + +for required in \ + "${CANDIDATE_BUILD}/dflash_server" \ + "${PREDICTOR_IPC_BIN}" \ + "${PREDICTOR_MODEL}"; do + [[ -e "${required}" ]] || { + printf 'missing Qwen tool-predictor path: %s\n' "${required}" >&2 + exit 2 + } +done + +export LD_LIBRARY_PATH="${CANDIDATE_BUILD}/deps/llama.cpp/ggml/src:${CANDIDATE_BUILD}/deps/llama.cpp/ggml/src/ggml-hip:${LD_LIBRARY_PATH:-}" +export LUCE_MMVQ_MAX_NCOLS=5 + +exec "${CANDIDATE_BUILD}/dflash_server" "$@" \ + --tool-hint-native-model "${PREDICTOR_MODEL}" \ + --tool-hint-native-ipc-bin "${PREDICTOR_IPC_BIN}" \ + --tool-hint-native-gpu "${PREDICTOR_GPU}" \ + --tool-hint-native-max-ctx "${PREDICTOR_MAX_CTX}" \ + --tool-hint-sidecar-max-tokens "${PREDICTOR_MAX_TOKENS}" \ + --tool-hint-native-schedule "${PREDICTOR_SCHEDULE}" \ + --tool-hint-execution-confidence "${PREDICTOR_CONFIDENCE}" \ + "${cache_args[@]}" diff --git a/optimizations/ooo_spec_lucebox5_cpu/profiles/lucebox5-cpu-lane-qualified.json b/optimizations/ooo_spec_lucebox5_cpu/profiles/lucebox5-cpu-lane-qualified.json new file mode 100644 index 000000000..84b0ad107 --- /dev/null +++ b/optimizations/ooo_spec_lucebox5_cpu/profiles/lucebox5-cpu-lane-qualified.json @@ -0,0 +1,69 @@ +{ + "executor": "child_process_cpu_affinity", + "path_summary": { + "100": { + "accelerator_relation": "non_accelerator", + "decode_interference_qualified": true, + "hit": { + "control_task_mean_ms": 5581.2723303339835, + "model_slowdown_percent": 0.11341375399527287, + "speculative_task_mean_ms": 2973.236727998786 + }, + "miss": { + "control_task_mean_ms": 5581.2723303339835, + "model_slowdown_percent": 0.11341375399527287, + "speculative_task_mean_ms": 5550.103543003206 + } + } + }, + "profile_kind": "disjoint_strix_cpu_sparse_compute", + "profile_status": "qualified", + "qualification": { + "checks": { + "direct_speedup": true, + "disjoint_cpu_affinity": true, + "ds4_active": true, + "identical_model_outputs": true, + "identical_tool_outputs": true, + "model_slowdown": true, + "private_miss_result_hidden": true + }, + "host": "lucebox5", + "model_cpu_affinity": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29 + ], + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ] + } +} diff --git a/optimizations/ooo_spec_lucebox5_cpu/results/trace-compiled-engine-qwen-production-6pairs-compact.json b/optimizations/ooo_spec_lucebox5_cpu/results/trace-compiled-engine-qwen-production-6pairs-compact.json new file mode 100644 index 000000000..3031e16c2 --- /dev/null +++ b/optimizations/ooo_spec_lucebox5_cpu/results/trace-compiled-engine-qwen-production-6pairs-compact.json @@ -0,0 +1,872 @@ +{ + "feature": "no-training trace-compiled speculative tool graphs", + "host": "lucebox5", + "methodology": { + "argument_binding": "the harness binds validated structured inputs to a request-scoped workflow_ref before either model runs", + "arm_order": "randomized per task", + "compiled": "one DS4+DSpark macro authorization; independent branches execute concurrently on the Strix CPU lane", + "measured_wall_time": "request through all tools and exact final answer", + "model_seed": 814, + "oracle_prediction": false, + "speculative": "Qwen predicts the trace-derived macro through the engine; its CPU graph overlaps DS4+DSpark and commits only on an exact call match", + "stage_batched": "DS4+DSpark sees only the currently-ready typed batch, authorizes one per dependency stage, runs its calls concurrently, and receives a compact rolling state instead of replaying old tool history", + "warmup_tasks": 1 + }, + "pairs": [ + { + "arm_order": [ + "compiled", + "stage_batched", + "speculative" + ], + "compiled": { + "all_ds4_active": true, + "completion_tokens": 50, + "decode_ms": 2097.5, + "exposed_tool_wait_ms": 10124.96868299786, + "final": { + "accept_rate": 0.9166666865348816, + "completion_tokens": 14, + "content_sha256": "cfbe812f379c62a39fb1047ae84da4e1c121dc72dc0bcadda519322908c0d43e" + }, + "final_correct": true, + "graph_wall_ms": 10126.189057999, + "macro_call": { + "arguments": { + "workflow_ref": "workflow_taska" + }, + "name": "execute_customer_workflows" + }, + "macro_correct": true, + "model_compute_ms": 13641.900000000001, + "model_turns": 2, + "prediction_hit": false, + "prediction_reason": null, + "prediction_source": null, + "prediction_status": null, + "predictor_ms": 0.0, + "task_ms": 23787.2067290009, + "tool_results_count": 10, + "tool_results_sha256": "eb2ed6a561ebc1e15c6f1a5226fb751de351f983d52dfe47f94c23a97d8c9cb2", + "underlying_calls_count": 10, + "underlying_calls_sha256": "b17c72ad474ba5d0a431bd35335fcb839c0a329504f6eed9eda5d211249e6841" + }, + "pair_index": 0, + "speculative": { + "all_ds4_active": true, + "completion_tokens": 50, + "decode_ms": 2087.8, + "exposed_tool_wait_ms": 0.019826, + "final": { + "accept_rate": 0.9166666865348816, + "completion_tokens": 14, + "content_sha256": "cfbe812f379c62a39fb1047ae84da4e1c121dc72dc0bcadda519322908c0d43e" + }, + "final_correct": true, + "graph_wall_ms": 10002.203055999416, + "macro_call": { + "arguments": { + "workflow_ref": "workflow_taska" + }, + "name": "execute_customer_workflows" + }, + "macro_correct": true, + "model_compute_ms": 13568.7, + "model_turns": 2, + "prediction_hit": true, + "prediction_reason": null, + "prediction_source": "native-qwen3", + "prediction_status": "hit", + "predictor_ms": 198.696329, + "task_ms": 13771.538261993555, + "tool_results_count": 10, + "tool_results_sha256": "eb2ed6a561ebc1e15c6f1a5226fb751de351f983d52dfe47f94c23a97d8c9cb2", + "underlying_calls_count": 10, + "underlying_calls_sha256": "b17c72ad474ba5d0a431bd35335fcb839c0a329504f6eed9eda5d211249e6841" + }, + "stage_batched": { + "all_ds4_active": true, + "completion_tokens": 204, + "decode_ms": 8934.7, + "exposed_tool_wait_ms": 10135.228522005491, + "final": { + "accept_rate": 0.9166666865348816, + "completion_tokens": 14, + "content_sha256": "cfbe812f379c62a39fb1047ae84da4e1c121dc72dc0bcadda519322908c0d43e" + }, + "final_correct": true, + "model_compute_ms": 67834.7, + "model_turns": 6, + "task_ms": 77984.87404600019, + "tool_results_count": 10, + "tool_results_sha256": "eb2ed6a561ebc1e15c6f1a5226fb751de351f983d52dfe47f94c23a97d8c9cb2", + "underlying_calls_count": 10, + "underlying_calls_sha256": "b17c72ad474ba5d0a431bd35335fcb839c0a329504f6eed9eda5d211249e6841" + }, + "task": { + "branch_count": 2, + "call_count": 10, + "id": "trace_compiled_000" + } + }, + { + "arm_order": [ + "compiled", + "speculative", + "stage_batched" + ], + "compiled": { + "all_ds4_active": true, + "completion_tokens": 51, + "decode_ms": 2239.5, + "exposed_tool_wait_ms": 10143.1347070029, + "final": { + "accept_rate": 0.6875, + "completion_tokens": 15, + "content_sha256": "e9618263bdfa8c2639cf7326931749b0625a0f5ca309f638c95ad621c5809303" + }, + "final_correct": true, + "graph_wall_ms": 10144.398914002522, + "macro_call": { + "arguments": { + "workflow_ref": "workflow_taskb" + }, + "name": "execute_customer_workflows" + }, + "macro_correct": true, + "model_compute_ms": 14251.7, + "model_turns": 2, + "prediction_hit": false, + "prediction_reason": null, + "prediction_source": null, + "prediction_status": null, + "predictor_ms": 0.0, + "task_ms": 24427.28014900058, + "tool_results_count": 15, + "tool_results_sha256": "2037fc8f56106ba9bc50763cc4b82c0f3262f0ffb8682ca5ca727eaf9d46b07b", + "underlying_calls_count": 15, + "underlying_calls_sha256": "2263e8d373d489cb82f03725feec381fb4ec218f6a706ebff2de102ed1e7c40b" + }, + "pair_index": 1, + "speculative": { + "all_ds4_active": true, + "completion_tokens": 51, + "decode_ms": 2241.1, + "exposed_tool_wait_ms": 0.035275, + "final": { + "accept_rate": 0.6875, + "completion_tokens": 15, + "content_sha256": "e9618263bdfa8c2639cf7326931749b0625a0f5ca309f638c95ad621c5809303" + }, + "final_correct": true, + "graph_wall_ms": 10001.984403999813, + "macro_call": { + "arguments": { + "workflow_ref": "workflow_taskb" + }, + "name": "execute_customer_workflows" + }, + "macro_correct": true, + "model_compute_ms": 14184.8, + "model_turns": 2, + "prediction_hit": true, + "prediction_reason": null, + "prediction_source": "native-qwen3", + "prediction_status": "hit", + "predictor_ms": 201.174369, + "task_ms": 14390.547144001175, + "tool_results_count": 15, + "tool_results_sha256": "2037fc8f56106ba9bc50763cc4b82c0f3262f0ffb8682ca5ca727eaf9d46b07b", + "underlying_calls_count": 15, + "underlying_calls_sha256": "2263e8d373d489cb82f03725feec381fb4ec218f6a706ebff2de102ed1e7c40b" + }, + "stage_batched": { + "all_ds4_active": true, + "completion_tokens": 210, + "decode_ms": 9601.1, + "exposed_tool_wait_ms": 10159.580159001052, + "final": { + "accept_rate": 0.6875, + "completion_tokens": 15, + "content_sha256": "e9618263bdfa8c2639cf7326931749b0625a0f5ca309f638c95ad621c5809303" + }, + "final_correct": true, + "model_compute_ms": 70723.3, + "model_turns": 6, + "task_ms": 80902.57541000028, + "tool_results_count": 15, + "tool_results_sha256": "2037fc8f56106ba9bc50763cc4b82c0f3262f0ffb8682ca5ca727eaf9d46b07b", + "underlying_calls_count": 15, + "underlying_calls_sha256": "2263e8d373d489cb82f03725feec381fb4ec218f6a706ebff2de102ed1e7c40b" + }, + "task": { + "branch_count": 3, + "call_count": 15, + "id": "trace_compiled_001" + } + }, + { + "arm_order": [ + "compiled", + "stage_batched", + "speculative" + ], + "compiled": { + "all_ds4_active": true, + "completion_tokens": 55, + "decode_ms": 2136.2, + "exposed_tool_wait_ms": 10164.059007001924, + "final": { + "accept_rate": 0.9375, + "completion_tokens": 19, + "content_sha256": "77d293cd086d290a81c7ba651034f8d95d944b3305f9956ae1dd9dae33671953" + }, + "final_correct": true, + "graph_wall_ms": 10165.094661002513, + "macro_call": { + "arguments": { + "workflow_ref": "workflow_taskc" + }, + "name": "execute_customer_workflows" + }, + "macro_correct": true, + "model_compute_ms": 14691.8, + "model_turns": 2, + "prediction_hit": false, + "prediction_reason": null, + "prediction_source": null, + "prediction_status": null, + "predictor_ms": 0.0, + "task_ms": 24958.613470000273, + "tool_results_count": 20, + "tool_results_sha256": "8eb88b665ebcafe693c031c44e5c724a2f3aebe00f5e726008dc316050fa211c", + "underlying_calls_count": 20, + "underlying_calls_sha256": "467ec65a34e98b78d3d4a6990fef3a6292140cb019e9ec1635fd7b5c7aa9b3ef" + }, + "pair_index": 2, + "speculative": { + "all_ds4_active": true, + "completion_tokens": 55, + "decode_ms": 2135.2, + "exposed_tool_wait_ms": 0.03236, + "final": { + "accept_rate": 0.9375, + "completion_tokens": 19, + "content_sha256": "77d293cd086d290a81c7ba651034f8d95d944b3305f9956ae1dd9dae33671953" + }, + "final_correct": true, + "graph_wall_ms": 10002.267649004352, + "macro_call": { + "arguments": { + "workflow_ref": "workflow_taskc" + }, + "name": "execute_customer_workflows" + }, + "macro_correct": true, + "model_compute_ms": 14628.900000000001, + "model_turns": 2, + "prediction_hit": true, + "prediction_reason": null, + "prediction_source": "native-qwen3", + "prediction_status": "hit", + "predictor_ms": 205.821132, + "task_ms": 14839.366328000324, + "tool_results_count": 20, + "tool_results_sha256": "8eb88b665ebcafe693c031c44e5c724a2f3aebe00f5e726008dc316050fa211c", + "underlying_calls_count": 20, + "underlying_calls_sha256": "467ec65a34e98b78d3d4a6990fef3a6292140cb019e9ec1635fd7b5c7aa9b3ef" + }, + "stage_batched": { + "all_ds4_active": true, + "completion_tokens": 211, + "decode_ms": 9963.0, + "exposed_tool_wait_ms": 10172.528195005725, + "final": { + "accept_rate": 0.9375, + "completion_tokens": 19, + "content_sha256": "77d293cd086d290a81c7ba651034f8d95d944b3305f9956ae1dd9dae33671953" + }, + "final_correct": true, + "model_compute_ms": 73757.7, + "model_turns": 6, + "task_ms": 83947.50960399688, + "tool_results_count": 20, + "tool_results_sha256": "8eb88b665ebcafe693c031c44e5c724a2f3aebe00f5e726008dc316050fa211c", + "underlying_calls_count": 20, + "underlying_calls_sha256": "467ec65a34e98b78d3d4a6990fef3a6292140cb019e9ec1635fd7b5c7aa9b3ef" + }, + "task": { + "branch_count": 4, + "call_count": 20, + "id": "trace_compiled_002" + } + }, + { + "arm_order": [ + "speculative", + "stage_batched", + "compiled" + ], + "compiled": { + "all_ds4_active": true, + "completion_tokens": 50, + "decode_ms": 2561.7, + "exposed_tool_wait_ms": 10127.542959999118, + "final": { + "accept_rate": 0.9166666865348816, + "completion_tokens": 14, + "content_sha256": "b8f612f1f27ff9b32b2fb5d747b5b09e9ff0de8e4241ea1810893b522b673906" + }, + "final_correct": true, + "graph_wall_ms": 10128.030801002751, + "macro_call": { + "arguments": { + "workflow_ref": "workflow_taskd" + }, + "name": "execute_customer_workflows" + }, + "macro_correct": true, + "model_compute_ms": 14129.5, + "model_turns": 2, + "prediction_hit": false, + "prediction_reason": null, + "prediction_source": null, + "prediction_status": null, + "predictor_ms": 0.0, + "task_ms": 24262.652850004088, + "tool_results_count": 10, + "tool_results_sha256": "c2041b2f6e3aeddaafa6ffb35bcd8affbc01e75cc62de4258206a53a6559a771", + "underlying_calls_count": 10, + "underlying_calls_sha256": "ff04b3b23c0147caffa852e474ed9f9fec517c227e12674dfacdba771bf34e33" + }, + "pair_index": 3, + "speculative": { + "all_ds4_active": true, + "completion_tokens": 50, + "decode_ms": 2557.7, + "exposed_tool_wait_ms": 0.020318, + "final": { + "accept_rate": 0.9166666865348816, + "completion_tokens": 14, + "content_sha256": "b8f612f1f27ff9b32b2fb5d747b5b09e9ff0de8e4241ea1810893b522b673906" + }, + "final_correct": true, + "graph_wall_ms": 10002.225474003353, + "macro_call": { + "arguments": { + "workflow_ref": "workflow_taskd" + }, + "name": "execute_customer_workflows" + }, + "macro_correct": true, + "model_compute_ms": 14066.4, + "model_turns": 2, + "prediction_hit": true, + "prediction_reason": null, + "prediction_source": "native-qwen3", + "prediction_status": "hit", + "predictor_ms": 198.264642, + "task_ms": 14328.593565005576, + "tool_results_count": 10, + "tool_results_sha256": "c2041b2f6e3aeddaafa6ffb35bcd8affbc01e75cc62de4258206a53a6559a771", + "underlying_calls_count": 10, + "underlying_calls_sha256": "ff04b3b23c0147caffa852e474ed9f9fec517c227e12674dfacdba771bf34e33" + }, + "stage_batched": { + "all_ds4_active": true, + "completion_tokens": 204, + "decode_ms": 8685.0, + "exposed_tool_wait_ms": 10141.129875002662, + "final": { + "accept_rate": 0.9166666865348816, + "completion_tokens": 14, + "content_sha256": "b8f612f1f27ff9b32b2fb5d747b5b09e9ff0de8e4241ea1810893b522b673906" + }, + "final_correct": true, + "model_compute_ms": 67694.4, + "model_turns": 6, + "task_ms": 77853.58790800092, + "tool_results_count": 10, + "tool_results_sha256": "c2041b2f6e3aeddaafa6ffb35bcd8affbc01e75cc62de4258206a53a6559a771", + "underlying_calls_count": 10, + "underlying_calls_sha256": "ff04b3b23c0147caffa852e474ed9f9fec517c227e12674dfacdba771bf34e33" + }, + "task": { + "branch_count": 2, + "call_count": 10, + "id": "trace_compiled_003" + } + }, + { + "arm_order": [ + "speculative", + "stage_batched", + "compiled" + ], + "compiled": { + "all_ds4_active": true, + "completion_tokens": 54, + "decode_ms": 2277.8, + "exposed_tool_wait_ms": 10143.549353000708, + "final": { + "accept_rate": 0.8125, + "completion_tokens": 17, + "content_sha256": "781a9894a33fb9de5e33c892e322d4181cd9a5838ea34bfacb5bcd6dda59e9e4" + }, + "final_correct": true, + "graph_wall_ms": 10144.438636001723, + "macro_call": { + "arguments": { + "workflow_ref": "workflow_taske" + }, + "name": "execute_customer_workflows" + }, + "macro_correct": true, + "model_compute_ms": 14573.5, + "model_turns": 2, + "prediction_hit": false, + "prediction_reason": null, + "prediction_source": null, + "prediction_status": null, + "predictor_ms": 0.0, + "task_ms": 24722.972717005177, + "tool_results_count": 15, + "tool_results_sha256": "968113db6b1f63c186b8124472fb64ac38b8d81957a7d9a05898a41eb23af577", + "underlying_calls_count": 15, + "underlying_calls_sha256": "0be2134f957d10c601ed26e39380751ede7d2b05ec5419d36680e9f981c2ba57" + }, + "pair_index": 4, + "speculative": { + "all_ds4_active": true, + "completion_tokens": 54, + "decode_ms": 2283.9, + "exposed_tool_wait_ms": 0.021269, + "final": { + "accept_rate": 0.8125, + "completion_tokens": 17, + "content_sha256": "781a9894a33fb9de5e33c892e322d4181cd9a5838ea34bfacb5bcd6dda59e9e4" + }, + "final_correct": true, + "graph_wall_ms": 10001.9813550025, + "macro_call": { + "arguments": { + "workflow_ref": "workflow_taske" + }, + "name": "execute_customer_workflows" + }, + "macro_correct": true, + "model_compute_ms": 14529.800000000001, + "model_turns": 2, + "prediction_hit": true, + "prediction_reason": null, + "prediction_source": "native-qwen3", + "prediction_status": "hit", + "predictor_ms": 244.327376, + "task_ms": 14804.260248994979, + "tool_results_count": 15, + "tool_results_sha256": "968113db6b1f63c186b8124472fb64ac38b8d81957a7d9a05898a41eb23af577", + "underlying_calls_count": 15, + "underlying_calls_sha256": "0be2134f957d10c601ed26e39380751ede7d2b05ec5419d36680e9f981c2ba57" + }, + "stage_batched": { + "all_ds4_active": true, + "completion_tokens": 213, + "decode_ms": 8830.1, + "exposed_tool_wait_ms": 10159.158977992774, + "final": { + "accept_rate": 0.8125, + "completion_tokens": 17, + "content_sha256": "781a9894a33fb9de5e33c892e322d4181cd9a5838ea34bfacb5bcd6dda59e9e4" + }, + "final_correct": true, + "model_compute_ms": 70980.1, + "model_turns": 6, + "task_ms": 81157.67812900594, + "tool_results_count": 15, + "tool_results_sha256": "968113db6b1f63c186b8124472fb64ac38b8d81957a7d9a05898a41eb23af577", + "underlying_calls_count": 15, + "underlying_calls_sha256": "0be2134f957d10c601ed26e39380751ede7d2b05ec5419d36680e9f981c2ba57" + }, + "task": { + "branch_count": 3, + "call_count": 15, + "id": "trace_compiled_004" + } + }, + { + "arm_order": [ + "stage_batched", + "compiled", + "speculative" + ], + "compiled": { + "all_ds4_active": true, + "completion_tokens": 46, + "decode_ms": 3468.2, + "exposed_tool_wait_ms": 10165.250458005175, + "final": { + "accept_rate": 0.5833333134651184, + "completion_tokens": 10, + "content_sha256": "4323774d53ee50de45337a0ce501766a1b32b5398ac8d64044006ed6f5143ef7" + }, + "final_correct": true, + "graph_wall_ms": 10166.563869002857, + "macro_call": { + "arguments": { + "workflow_ref": "workflow_taskf" + }, + "name": "execute_customer_workflows" + }, + "macro_correct": true, + "model_compute_ms": 16199.5, + "model_turns": 2, + "prediction_hit": false, + "prediction_reason": null, + "prediction_source": null, + "prediction_status": null, + "predictor_ms": 0.0, + "task_ms": 26370.94549000176, + "tool_results_count": 20, + "tool_results_sha256": "de71e45a390fca728e07a8bf4cdc467e91604fc52f68abe6ed0bd75ca77e87d2", + "underlying_calls_count": 20, + "underlying_calls_sha256": "bd91a2316a76490265d64557860d9a2acdc97a8f1a18014dacaa0a21cd241e2f" + }, + "pair_index": 5, + "speculative": { + "all_ds4_active": true, + "completion_tokens": 46, + "decode_ms": 2451.7, + "exposed_tool_wait_ms": 0.032681, + "final": { + "accept_rate": 0.5833333134651184, + "completion_tokens": 10, + "content_sha256": "4323774d53ee50de45337a0ce501766a1b32b5398ac8d64044006ed6f5143ef7" + }, + "final_correct": true, + "graph_wall_ms": 10002.300546002516, + "macro_call": { + "arguments": { + "workflow_ref": "workflow_taskf" + }, + "name": "execute_customer_workflows" + }, + "macro_correct": true, + "model_compute_ms": 15157.8, + "model_turns": 2, + "prediction_hit": true, + "prediction_reason": null, + "prediction_source": "native-qwen3", + "prediction_status": "hit", + "predictor_ms": 214.897278, + "task_ms": 15378.521895996528, + "tool_results_count": 20, + "tool_results_sha256": "de71e45a390fca728e07a8bf4cdc467e91604fc52f68abe6ed0bd75ca77e87d2", + "underlying_calls_count": 20, + "underlying_calls_sha256": "bd91a2316a76490265d64557860d9a2acdc97a8f1a18014dacaa0a21cd241e2f" + }, + "stage_batched": { + "all_ds4_active": true, + "completion_tokens": 191, + "decode_ms": 11077.9, + "exposed_tool_wait_ms": 10172.786328992515, + "final": { + "accept_rate": 0.5833333134651184, + "completion_tokens": 10, + "content_sha256": "4323774d53ee50de45337a0ce501766a1b32b5398ac8d64044006ed6f5143ef7" + }, + "final_correct": true, + "model_compute_ms": 75437.0, + "model_turns": 6, + "task_ms": 85663.01394799666, + "tool_results_count": 20, + "tool_results_sha256": "de71e45a390fca728e07a8bf4cdc467e91604fc52f68abe6ed0bd75ca77e87d2", + "underlying_calls_count": 20, + "underlying_calls_sha256": "bd91a2316a76490265d64557860d9a2acdc97a8f1a18014dacaa0a21cd241e2f" + }, + "task": { + "branch_count": 4, + "call_count": 20, + "id": "trace_compiled_005" + } + } + ], + "pattern": { + "fingerprint": "06d95882b485daf3f30f3fc12ea62ccfd18656de9deafa2f4ed77c49bb8c0645", + "macro_name": "execute_customer_workflows", + "model_training": false, + "root_fields": [ + "customer_email", + "destination" + ], + "side_effects_allowed": false, + "steps": [ + { + "arguments": { + "customer_email": { + "key": "customer_email", + "source": "root" + } + }, + "tool": "resolve_customer" + }, + { + "arguments": { + "customer_ref": { + "key": "call_ref", + "source": "previous_result" + } + }, + "tool": "list_open_orders" + }, + { + "arguments": { + "orders_ref": { + "key": "call_ref", + "source": "previous_result" + } + }, + "tool": "get_order_details" + }, + { + "arguments": { + "destination": { + "key": "destination", + "source": "root" + }, + "order_ref": { + "key": "call_ref", + "source": "previous_result" + } + }, + "tool": "calculate_shipping" + }, + { + "arguments": { + "shipping_ref": { + "key": "call_ref", + "source": "previous_result" + } + }, + "tool": "prepare_customer_summary" + } + ], + "training_report": "/home/lucebox5/tool-spec-cpu-20260813/results/multiturn-cached-wordref-production-6tasks.json", + "training_report_sha256": "2475697d418bffed0e9668da26ce6c88a85a952ce97d99749f440f97f9ac5bf9", + "training_traces": 2, + "workflow_registry": "/home/lucebox5/tool-spec-cpu-20260813/results/trace-workflow-registry.json", + "workflow_registry_sha256": "0b73cb4f45284f07a4d6890906d1eaf5a25469c2b80eb542b1d74f1e3a9a0e1b" + }, + "production_gate": { + "checks": { + "calls_stable": true, + "decode_slowdown_p50": true, + "decode_slowdown_p95": true, + "ds4_active": true, + "end_to_end_ci": true, + "end_to_end_speedup": true, + "end_to_end_tail": true, + "final_answers_correct": true, + "final_outputs_stable": true, + "macro_calls_correct": true, + "macro_outputs_stable": true, + "model_slowdown_p50": true, + "model_slowdown_p95": true, + "prediction_hit_rate": true, + "prediction_source": true, + "prefix_cache_configured": true, + "sample_size": true, + "speculation_incremental_ci": true, + "speculation_incremental_gain": true, + "tool_results_stable": true + }, + "passed": true, + "thresholds": { + "max_decode_slowdown_p95_percent": 5.0, + "max_decode_slowdown_percent": 1.0, + "max_model_slowdown_p95_percent": 5.0, + "max_model_slowdown_percent": 1.0, + "min_e2e_speedup": 2.0, + "min_e2e_speedup_p05": 1.5, + "min_incremental_speedup": 1.05, + "min_production_pairs": 6 + } + }, + "schema_version": 1, + "server_snapshot": { + "model": { + "arch": "deepseek4", + "draft_path": null, + "tokenizer_id": null + }, + "prefix_cache_after": { + "capacity": 32, + "in_use": 0, + "lifetime_hits": 0 + }, + "prefix_cache_before": { + "capacity": 32, + "in_use": 0, + "lifetime_hits": 0 + }, + "tool_speculation": { + "allowed_tools": [ + "calculate_shipping", + "execute_customer_workflows", + "get_order_details", + "list_open_orders", + "prepare_customer_summary", + "resolve_customer" + ], + "automatic_prediction_enabled": true, + "compute_isolation": "disjoint_cpu_affinity", + "cpu_affinity_isolated": true, + "enabled": true, + "execution_mode": "child_process_cpu_affinity", + "executor_contract": "child_process_cpu_affinity", + "hip_reserved_tool_compute_units": 0, + "hip_tool_device": null, + "max_model_slowdown_ratio": 1.05, + "model_cpu_affinity": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29 + ], + "model_expert_ownership_unique": false, + "model_routing_static": false, + "prediction_confidence": 0.75, + "prediction_source": "native-qwen3", + "predictor_decode_isolated": true, + "predictor_schedule": "before-model", + "preserves_token_speculation": true, + "profile_lanes": [ + { + "accelerator_relation": "non_accelerator", + "decode_interference_qualified": true, + "model_slowdown_ratio": 1.0011341375399527, + "requires_static_model_routing": false, + "requires_unique_expert_ownership": false, + "resource_percentage": 100 + } + ], + "profile_status": "qualified", + "protocol": "dflash.tool-speculation.v1", + "requires_client_support": false, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "unqualified_lane_policy": "defer" + } + }, + "summary": { + "all_calls_stable": true, + "all_ds4_active": true, + "all_final_answers_correct": true, + "all_final_outputs_stable": true, + "all_macro_calls_correct": true, + "all_predictions_from_qwen": true, + "all_tool_results_stable": true, + "calls_per_task": [ + 10, + 15, + 20, + 10, + 15, + 20 + ], + "compiled_exposed_tool_wait_p50_ms": 10143.342030001804, + "compiled_model_turns_p50": 2.0, + "compiled_task_p50_ms": 24575.12643300288, + "compiled_to_speculative_bootstrap_95ci": [ + 1.6759547511337496, + 1.7210318416877688 + ], + "compiled_to_speculative_speedup_p05": 1.6729725830674844, + "compiled_to_speculative_speedup_p50": 1.6953781778482404, + "continuation_cache_hit_rate": 0.0, + "decode_slowdown_p50_percent": -0.10147920266865285, + "decode_slowdown_p95_percent": 0.21871282872425457, + "macro_output_stability_rate": 1.0, + "model_compute_slowdown_p50_percent": -0.4580005364335671, + "model_compute_slowdown_p95_percent": -0.3319269946081671, + "pattern_prediction_hit_rate": 1.0, + "predictor_p50_ms": 203.4977505, + "prefix_cache_configured": true, + "prefix_cache_lifetime_hit_delta": 0, + "speculative_exposed_tool_wait_p50_ms": 0.026814499999999998, + "speculative_task_p50_ms": 14597.403696498077, + "speedup_by_call_count": { + "10": { + "combined_speedup_p50": 5.548099679951088, + "compiled_task_p50_ms": 24024.929789502494, + "speculation_speedup_p50": 1.7102881019726668, + "speculative_task_p50_ms": 14050.065913499566, + "stage_batched_task_p50_ms": 77919.23097700055, + "tasks": 2 + }, + "15": { + "combined_speedup_p50": 5.551986886996318, + "compiled_task_p50_ms": 24575.12643300288, + "speculation_speedup_p50": 1.683721802277213, + "speculative_task_p50_ms": 14597.403696498077, + "stage_batched_task_p50_ms": 81030.12676950311, + "tasks": 2 + }, + "20": { + "combined_speedup_p50": 5.613692001294089, + "compiled_task_p50_ms": 25664.779480001016, + "speculation_speedup_p50": 1.698354866419879, + "speculative_task_p50_ms": 15108.944111998426, + "stage_batched_task_p50_ms": 84805.26177599677, + "tasks": 2 + } + }, + "stage_batched_exposed_tool_wait_p50_ms": 10159.369568496913, + "stage_batched_model_turns_p50": 6.0, + "stage_batched_task_p50_ms": 81030.12676950311, + "stage_batched_to_compiled_speedup_p50": 3.2805602399674787, + "stage_batched_to_speculative_bootstrap_95ci": [ + 5.457745637116071, + 5.659919390842823 + ], + "stage_batched_to_speculative_speedup_min": 5.433442406946423, + "stage_batched_to_speculative_speedup_p05": 5.445594022031247, + "stage_batched_to_speculative_speedup_p50": 5.5961135402826, + "tasks": 6, + "total_wall_speedup": 5.570717496895011 + }, + "workload": { + "branches_per_task": "2-4", + "calls_per_task": "10-20", + "dependency": "five serial calls per branch; branches are independent", + "tasks": 6, + "tool_adapter": "deterministic read-only 2-second API replay" + } +} diff --git a/optimizations/ooo_spec_lucebox5_cpu/results/trace-compiled-training-traces.json b/optimizations/ooo_spec_lucebox5_cpu/results/trace-compiled-training-traces.json new file mode 100644 index 000000000..089220994 --- /dev/null +++ b/optimizations/ooo_spec_lucebox5_cpu/results/trace-compiled-training-traces.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "source": "successful read-only workflow traces", + "traces": [ + { + "root": { + "customer_email": "agent-benchmark-2@example.test", + "destination": "Turin" + }, + "calls": [ + {"name": "resolve_customer", "arguments": {"customer_email": "agent-benchmark-2@example.test"}}, + {"name": "list_open_orders", "arguments": {"customer_ref": "plum"}}, + {"name": "get_order_details", "arguments": {"orders_ref": "jade"}}, + {"name": "calculate_shipping", "arguments": {"destination": "Turin", "order_ref": "amber"}}, + {"name": "prepare_customer_summary", "arguments": {"shipping_ref": "amber"}} + ], + "results": [ + {"call_ref": "plum", "call_sha256": "93824d1968f2c3ab058a3ef69d2625c0fea88fad3b27b8c30f95418fb83ae6cf", "tool_name": "resolve_customer", "side_effects": false}, + {"call_ref": "jade", "call_sha256": "772a4b9fde9e69fb1da323ba4c22e7a7d01061c0600c133e74de7bd73bba931f", "tool_name": "list_open_orders", "side_effects": false}, + {"call_ref": "amber", "call_sha256": "428dbfc05f17e7a9295c2b4741dbae2bb91d3f107150b77bb34c6ca25bd8af23", "tool_name": "get_order_details", "side_effects": false}, + {"call_ref": "amber", "call_sha256": "21ec3c60e83bfbc0ab4e6e8356a5c87e7210ffd16b53ca8299017261d97f70a1", "tool_name": "calculate_shipping", "side_effects": false}, + {"call_ref": "ivory", "call_sha256": "adaa761561a84b7e457acb7881e856285fcdccf21a6d82751dd2bea984b1d6d0", "tool_name": "prepare_customer_summary", "side_effects": false} + ] + }, + { + "root": { + "customer_email": "agent-benchmark-5@example.test", + "destination": "Naples" + }, + "calls": [ + {"name": "resolve_customer", "arguments": {"customer_email": "agent-benchmark-5@example.test"}}, + {"name": "list_open_orders", "arguments": {"customer_ref": "maple"}}, + {"name": "get_order_details", "arguments": {"orders_ref": "willow"}}, + {"name": "calculate_shipping", "arguments": {"destination": "Naples", "order_ref": "olive"}}, + {"name": "prepare_customer_summary", "arguments": {"shipping_ref": "cedar"}} + ], + "results": [ + {"call_ref": "maple", "call_sha256": "71063336b1901dd2a0050d4a7a0ebe52ad0b0755f5af079efe22db9c1df00b29", "tool_name": "resolve_customer", "side_effects": false}, + {"call_ref": "willow", "call_sha256": "8da0147e626bec41f9bafd2ec61d07d01a33e176990483fd62a2b93eeea82e2f", "tool_name": "list_open_orders", "side_effects": false}, + {"call_ref": "olive", "call_sha256": "66ea7a270ffff62c364786d69dc72144b0b0eb928c262f682dff72635964c6a9", "tool_name": "get_order_details", "side_effects": false}, + {"call_ref": "cedar", "call_sha256": "db784ed2c9e33569d9c6bdd36e6963076032cd8d0bf48cf954420a843cb8d593", "tool_name": "calculate_shipping", "side_effects": false}, + {"call_ref": "amber", "call_sha256": "6ba7d6c0326fbe23206f51a9a4268beb41a3050097565421946c4ac32900f9bc", "tool_name": "prepare_customer_summary", "side_effects": false} + ] + } + ] +} diff --git a/optimizations/ooo_spec_lucebox5_cpu/run_native_cpu_server_lucebox5.sh b/optimizations/ooo_spec_lucebox5_cpu/run_native_cpu_server_lucebox5.sh new file mode 100755 index 000000000..2e067aab9 --- /dev/null +++ b/optimizations/ooo_spec_lucebox5_cpu/run_native_cpu_server_lucebox5.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +set -euo pipefail + +root="/home/lucebox5" +experiment="$root/tool-spec-cpu-20260813" +launcher="${LUCEBOX_LAUNCHER:-$experiment/run-deepseek-0731-cpu-tool.sh}" +executor="${TOOL_SPEC_EXECUTOR:-$experiment/cpu_sparse_tool_executor}" +profile="${TOOL_SPEC_PROFILE:-$experiment/profiles/lucebox5-cpu-lane-qualified.json}" +allowed="${TOOL_SPEC_ALLOW:-benchmark_cpu_sparse}" +native_wrapper_dir="${NATIVE_WRAPPER_DIR:-$experiment/native-wrapper}" +candidate_build="${CANDIDATE_BUILD:-$experiment/engine-ooo-spec/server/build-hip-dual}" +predictor_model="${PREDICTOR_MODEL:-$experiment/models/Qwen3-0.6B-Q8_0.gguf}" + +for required in \ + "$launcher" \ + "$executor" \ + "$profile" \ + "$native_wrapper_dir/dflash_server" \ + "$candidate_build/dflash_server" \ + "$candidate_build/backend_ipc_daemon" \ + "$predictor_model"; do + [[ -e "$required" ]] || { + printf 'missing required path: %s\n' "$required" >&2 + exit 2 + } +done +if pgrep -x dflash_server >/dev/null; then + printf 'a dflash_server is already running; refusing to overlap it\n' >&2 + exit 75 +fi +if fuser -s /dev/kfd 2>/dev/null; then + printf '/dev/kfd already has an owner; refusing to overlap it\n' >&2 + fuser -v /dev/kfd >&2 || true + exit 75 +fi + +exec env \ + HOME="$root" \ + USER="lucebox5" \ + PATH="$root/.local/bin:/opt/rocm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \ + ENGINE_DIR="$root/lucebox-engine-0731" \ + BUILD_DIR="$native_wrapper_dir" \ + CANDIDATE_BUILD="$candidate_build" \ + PREDICTOR_MODEL="$predictor_model" \ + PREDICTOR_GPU="${PREDICTOR_GPU:-1}" \ + PREDICTOR_MAX_CTX="${PREDICTOR_MAX_CTX:-4096}" \ + PREDICTOR_MAX_TOKENS="${PREDICTOR_MAX_TOKENS:-256}" \ + PREDICTOR_CONFIDENCE="${PREDICTOR_CONFIDENCE:-0.75}" \ + PREDICTOR_SCHEDULE="${PREDICTOR_SCHEDULE:-before-model}" \ + PREFIX_CACHE_SLOTS_OVERRIDE="${PREFIX_CACHE_SLOTS_OVERRIDE:-}" \ + QUALIFIED_CONFIG_DIR="/opt/lucebox-manage/qualified/r9700_deepseek/runtime-config" \ + TARGET_MODEL="$root/lucebox-models/DeepSeek-V4-Flash-0731-ROCMFPX-MIX-STRIX.gguf" \ + DRAFT_MODEL="$root/lucebox-models/DeepSeek-V4-Flash-0731-DSpark-draft-Q4RMFP4-denseF16.gguf" \ + SERVER_PORT="18145" \ + MODEL_CPU_AFFINITY="0-13,16-29" \ + TOOL_SPEC_EXECUTOR="$executor" \ + TOOL_SPEC_PROFILE="$profile" \ + TOOL_SPEC_ALLOW="$allowed" \ + TOOL_SPEC_CPU_AFFINITY="14-15,30-31" \ + TOOL_SPEC_MAX_MODEL_SLOWDOWN="1.05" \ + LUCEBOX_INFERENCE_PROFILE="quality" \ + "$launcher" diff --git a/optimizations/ooo_spec_lucebox5_cpu/test_benchmark_cpu_tool_speculation.py b/optimizations/ooo_spec_lucebox5_cpu/test_benchmark_cpu_tool_speculation.py new file mode 100644 index 000000000..8e03e399b --- /dev/null +++ b/optimizations/ooo_spec_lucebox5_cpu/test_benchmark_cpu_tool_speculation.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import argparse +import unittest + +from benchmark_cpu_tool_speculation import ( + TOOL_NAME, + expected_arguments, + normalize_tool_call, + parse_cpu_list, + percentile, + props_url, + request_body, +) + + +class CpuToolSpeculationBenchmarkTest(unittest.TestCase): + def test_cpu_list_parser_canonicalizes_ranges(self) -> None: + self.assertEqual(parse_cpu_list("30-31,15,14-15"), [14, 15, 30, 31]) + with self.assertRaises(argparse.ArgumentTypeError): + parse_cpu_list("14,,15") + + def test_request_has_exact_concrete_prediction(self) -> None: + arguments = expected_arguments(4096, 16, 77, 2, 731) + self.assertEqual(arguments, {"iterations": 77}) + body = request_body(arguments, 32, prediction=arguments) + self.assertEqual( + body["tool_speculation"]["call"], + {"name": TOOL_NAME, "arguments": arguments}, + ) + self.assertEqual(body["tool_speculation"]["confidence"], 1.0) + self.assertEqual(body["tools"][0]["name"], TOOL_NAME) + + def test_props_url_uses_server_origin(self) -> None: + self.assertEqual( + props_url("http://127.0.0.1:18145/v1/chat/completions?x=1"), + "http://127.0.0.1:18145/props", + ) + + def test_percentile_interpolates_sorted_values(self) -> None: + self.assertEqual(percentile([4, 1, 3, 2], 0.0), 1.0) + self.assertEqual(percentile([4, 1, 3, 2], 0.5), 2.5) + self.assertEqual(percentile([4, 1, 3, 2], 1.0), 4.0) + + def test_automatic_qwen_arm_has_no_oracle_prediction(self) -> None: + arguments = {"iterations": 77} + body = request_body( + arguments, + 32, + prediction=None, + automatic_prediction=True, + tool_choice="required", + ) + self.assertNotIn("tool_speculation", body) + self.assertTrue(body["automatic_tool_speculation"]) + self.assertEqual(body["tool_choice"], "required") + + def test_normalizes_deepseek_single_parameter_envelope(self) -> None: + result = { + "choices": [ + { + "message": { + "content": ( + '{"function":"batch_resolve_customer",' + '"parameter":"stage_ref",' + '"parameter_value":"workflow_taskf_stage_one"}' + ) + } + } + ] + } + self.assertEqual( + normalize_tool_call(result), + { + "name": "batch_resolve_customer", + "arguments": {"stage_ref": "workflow_taskf_stage_one"}, + }, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/optimizations/ooo_spec_lucebox5_cpu/test_benchmark_trace_compiled_workflows.py b/optimizations/ooo_spec_lucebox5_cpu/test_benchmark_trace_compiled_workflows.py new file mode 100644 index 000000000..7f4ff69a2 --- /dev/null +++ b/optimizations/ooo_spec_lucebox5_cpu/test_benchmark_trace_compiled_workflows.py @@ -0,0 +1,400 @@ +from __future__ import annotations + +import argparse +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from benchmark_trace_compiled_workflows import ( + alphabetic_identifier, + compact_arm, + final_answer_correct, + load_training_traces, + load_partial_pairs, + make_task, + mine_pattern, + model_observation, + parse_request_customers, + post_final, + production_checks, + simulated_tool_result, + stage_batch_tool, + stage_batched_messages, + stage_reference, + workflow_reference, +) + + +def workflow_trace(email: str, destination: str) -> dict: + root = {"customer_email": email, "destination": destination} + calls = [] + results = [] + + def add(name: str, arguments: dict) -> None: + call = {"name": name, "arguments": arguments} + calls.append(call) + results.append(simulated_tool_result(call)) + + add("resolve_customer", {"customer_email": email}) + add("list_open_orders", {"customer_ref": results[-1]["call_ref"]}) + add("get_order_details", {"orders_ref": results[-1]["call_ref"]}) + add( + "calculate_shipping", + {"order_ref": results[-1]["call_ref"], "destination": destination}, + ) + add("prepare_customer_summary", {"shipping_ref": results[-1]["call_ref"]}) + return {"root": root, "calls": calls, "results": results} + + +class TraceCompiledWorkflowBenchmarkTest(unittest.TestCase): + def setUp(self) -> None: + self.pattern = mine_pattern( + [ + workflow_trace("first@example.test", "Rome"), + workflow_trace("second@example.test", "Milan"), + ] + ) + + def test_mines_control_flow_and_late_bound_arguments(self) -> None: + self.assertEqual(self.pattern.training_traces, 2) + self.assertEqual( + [step.tool for step in self.pattern.steps], + [ + "resolve_customer", + "list_open_orders", + "get_order_details", + "calculate_shipping", + "prepare_customer_summary", + ], + ) + shipping_bindings = dict(self.pattern.steps[3].arguments) + self.assertEqual(shipping_bindings["order_ref"].source, "previous_result") + self.assertEqual(shipping_bindings["order_ref"].key, "call_ref") + self.assertEqual(shipping_bindings["destination"].source, "root") + + def test_loads_compact_training_trace_file(self) -> None: + traces = [ + workflow_trace("first@example.test", "Rome"), + workflow_trace("second@example.test", "Milan"), + ] + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "traces.json" + path.write_text(json.dumps({"traces": traces}), encoding="utf-8") + loaded = load_training_traces(path, required_steps=5) + self.assertEqual(loaded, traces) + + def test_compact_arm_keeps_evidence_but_drops_full_graph(self) -> None: + arm = { + "task_ms": 12.0, + "underlying_calls": ["call"], + "tool_results": ["digest"], + "graph": {"large": "payload"}, + "call_turns": [{"large": "payload"}], + "final": { + "content": "done", + "content_sha256": "hash", + "completion_tokens": 1, + "unused": "payload", + }, + } + compact = compact_arm(arm) + self.assertEqual(compact["underlying_calls_count"], 1) + self.assertEqual( + compact["underlying_calls_sha256"], + "4f2a91df1674ac67599f9835f2d43b0ca94e1e769f6a666ce448ae07ac1d94f7", + ) + self.assertEqual(compact["final"]["content_sha256"], "hash") + self.assertNotIn("underlying_calls", compact) + self.assertNotIn("content", compact["final"]) + self.assertNotIn("graph", compact) + self.assertNotIn("call_turns", compact) + self.assertNotIn("unused", compact["final"]) + + def test_pattern_expands_unseen_request_without_literals(self) -> None: + root = {"customer_email": "new@example.test", "destination": "Turin"} + calls = self.pattern.simulate(root) + self.assertEqual(calls[0]["arguments"], {"customer_email": root["customer_email"]}) + self.assertEqual( + calls[1]["arguments"], {"customer_ref": simulated_tool_result(calls[0])["call_ref"]} + ) + self.assertEqual(calls[3]["arguments"]["destination"], "Turin") + + def test_compiler_rejects_side_effecting_trace(self) -> None: + traces = [ + workflow_trace("first@example.test", "Rome"), + workflow_trace("second@example.test", "Milan"), + ] + traces[1]["results"][2]["side_effects"] = True + with self.assertRaisesRegex(ValueError, "side-effect-free"): + mine_pattern(traces) + + def test_compiler_rejects_literal_argument(self) -> None: + traces = [ + workflow_trace("first@example.test", "Rome"), + workflow_trace("second@example.test", "Milan"), + ] + for trace in traces: + trace["calls"][0]["arguments"]["constant"] = "not-in-request" + trace["results"][0] = simulated_tool_result(trace["calls"][0]) + with self.assertRaisesRegex(ValueError, "literal"): + mine_pattern(traces) + + def test_macro_schema_is_closed_and_typed(self) -> None: + tool = self.pattern.macro_tool(4)["function"] + parameters = tool["parameters"] + item = parameters["properties"]["customers"]["items"] + self.assertFalse(parameters["additionalProperties"]) + self.assertFalse(item["additionalProperties"]) + self.assertEqual(set(item["required"]), set(self.pattern.root_fields)) + self.assertEqual(parameters["properties"]["customers"]["maxItems"], 4) + + def test_bound_macro_uses_a_short_single_value_reference(self) -> None: + task = make_task(4, 4, self.pattern) + workflow_ref = workflow_reference(task, self.pattern) + parameters = self.pattern.macro_tool(4, workflow_ref)["function"]["parameters"] + self.assertEqual(set(parameters["properties"]), {"workflow_ref"}) + self.assertEqual( + parameters["properties"]["workflow_ref"]["enum"], [workflow_ref] + ) + self.assertEqual(workflow_ref, "workflow_taske") + + def test_generated_branches_do_not_collapse_on_short_refs(self) -> None: + task = make_task(3, 4, self.pattern) + refs_by_stage = list( + zip( + *[ + [simulated_tool_result(call)["call_ref"] for call in self.pattern.simulate(root)] + for root in task["items"] + ], + strict=True, + ) + ) + self.assertTrue(all(len(set(refs)) == 4 for refs in refs_by_stage)) + + def test_generated_identifiers_avoid_ambiguous_digits(self) -> None: + self.assertEqual(alphabetic_identifier(0), "taska") + self.assertEqual(alphabetic_identifier(26), "taskaa") + task = make_task(50, 2, self.pattern) + self.assertTrue( + all(not any(character.isdigit() for character in item["customer_email"]) + for item in task["items"]) + ) + + def test_event_extractor_recovers_macro_arguments_without_a_model(self) -> None: + content = "Customers: a@example.test to Rome; b@example.test to Milan." + self.assertEqual( + parse_request_customers(content), + [ + {"customer_email": "a@example.test", "destination": "Rome"}, + {"customer_email": "b@example.test", "destination": "Milan"}, + ], + ) + + def test_stage_batch_schema_preserves_every_call(self) -> None: + task = make_task(2, 3, self.pattern) + tool = stage_batch_tool(self.pattern, 3, 4)["function"] + calls = tool["parameters"]["properties"]["calls"] + self.assertEqual(tool["name"], "batch_calculate_shipping") + self.assertEqual(calls["maxItems"], 4) + self.assertEqual( + set(calls["items"]["required"]), {"order_ref", "destination"} + ) + self.assertIn( + "exactly one currently-ready batch tool", + stage_batched_messages(task, self.pattern)[0]["content"], + ) + + def test_bound_stage_uses_the_request_scoped_reference(self) -> None: + task = make_task(2, 4, self.pattern) + stage_ref = stage_reference(task, self.pattern, 2) + parameters = stage_batch_tool( + self.pattern, 2, 4, stage_ref + )["function"]["parameters"] + self.assertEqual(set(parameters["properties"]), {"stage_ref"}) + self.assertEqual(parameters["properties"]["stage_ref"]["enum"], [stage_ref]) + self.assertEqual(stage_ref, "workflow_taskc_stage_three") + + def test_parses_multiple_native_tool_calls(self) -> None: + response = { + "choices": [ + { + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "resolve_customer", + "arguments": '{"customer_email":"a@example.test"}', + }, + }, + { + "id": "call_2", + "type": "function", + "function": { + "name": "resolve_customer", + "arguments": {"customer_email": "b@example.test"}, + }, + }, + ], + } + } + ], + "usage": {"timings": {}}, + } + observed = model_observation(response, 12.0) + self.assertEqual(len(observed["calls"]), 2) + self.assertEqual(observed["calls"][0]["id"], "call_1") + self.assertEqual( + observed["calls"][1]["call"]["arguments"]["customer_email"], + "b@example.test", + ) + + def test_normalizes_content_format_call_for_conversation_history(self) -> None: + response = { + "choices": [ + { + "message": { + "role": "assistant", + "content": ( + '{"function":"resolve_customer","parameters":' + '{"customer_email":"a@example.test"},"type":"function"}' + ), + } + } + ], + "usage": {"timings": {}}, + } + observed = model_observation(response, 12.0) + self.assertTrue(observed["content_format_call"]) + self.assertEqual( + observed["calls"][0]["call"], + { + "name": "resolve_customer", + "arguments": {"customer_email": "a@example.test"}, + }, + ) + self.assertEqual( + observed["assistant_message"]["tool_calls"][0]["id"], + observed["calls"][0]["id"], + ) + + def test_production_gate_requires_two_x_and_exact_outputs(self) -> None: + summary = { + "tasks": 6, + "stage_batched_to_speculative_speedup_p50": 1.99, + "stage_batched_to_speculative_bootstrap_95ci": [1.5, 2.4], + "stage_batched_to_speculative_speedup_p05": 1.6, + "compiled_to_speculative_speedup_p50": 1.1, + "compiled_to_speculative_bootstrap_95ci": [1.01, 1.2], + "pattern_prediction_hit_rate": 1.0, + "all_predictions_from_qwen": True, + "model_compute_slowdown_p50_percent": 0.0, + "model_compute_slowdown_p95_percent": 0.0, + "decode_slowdown_p50_percent": 0.0, + "decode_slowdown_p95_percent": 0.0, + "continuation_cache_hit_rate": 1.0, + "prefix_cache_configured": True, + "all_calls_stable": True, + "all_tool_results_stable": True, + "macro_output_stability_rate": 1.0, + "all_final_answers_correct": True, + "all_final_outputs_stable": True, + "all_macro_calls_correct": True, + "all_ds4_active": True, + } + args = argparse.Namespace( + min_e2e_speedup=2.0, + min_e2e_speedup_p05=1.5, + min_incremental_speedup=1.05, + min_production_pairs=6, + max_model_slowdown_percent=1.0, + max_model_slowdown_p95_percent=5.0, + max_decode_slowdown_percent=1.0, + max_decode_slowdown_p95_percent=5.0, + ) + checks = production_checks(summary, args) + self.assertFalse(checks["end_to_end_speedup"]) + self.assertTrue( + all(value for key, value in checks.items() if key != "end_to_end_speedup") + ) + + def test_final_turn_is_identical_and_context_free_for_every_arm(self) -> None: + args = argparse.Namespace(final_max_tokens=32) + with patch( + "benchmark_trace_compiled_workflows.post_turn", + return_value={"content": "workflow_complete:plum"}, + ) as mocked: + post_final(args, "workflow_complete:plum") + + call_args = mocked.call_args.args + self.assertEqual(call_args[2], []) + self.assertEqual(call_args[3], "none") + self.assertEqual(call_args[4], 32) + self.assertEqual( + call_args[1][-1], + {"role": "user", "content": "workflow_complete:plum"}, + ) + + def test_final_receipt_accepts_literal_or_equivalent_json(self) -> None: + expected = "workflow_complete:plum,ivory" + self.assertTrue(final_answer_correct(expected, expected)) + self.assertTrue( + final_answer_correct('{"workflow_complete": "plum,ivory"}', expected) + ) + self.assertTrue(final_answer_correct("plum,ivory", expected)) + self.assertFalse(final_answer_correct('{"workflow_complete": "plum"}', expected)) + self.assertFalse(final_answer_correct("workflow_complete:ivory,plum", expected)) + + def test_resume_checkpoint_requires_matching_task_and_arm_order(self) -> None: + tasks = [make_task(0, 2, self.pattern), make_task(1, 3, self.pattern)] + orders = [ + ["compiled", "stage_batched", "speculative"], + ["speculative", "stage_batched", "compiled"], + ] + checkpoint = { + "schema_version": 1, + "complete": False, + "pairs": [ + { + "pair_index": 0, + "task": tasks[0], + "arm_order": orders[0], + "stage_batched": { + "final": {"content": "plum,ivory"}, + "expected_final": "workflow_complete:plum,ivory", + }, + "compiled": { + "final": {"content": "plum,ivory"}, + "expected_final": "workflow_complete:plum,ivory", + }, + "speculative": { + "final": {"content": "plum,ivory"}, + "expected_final": "workflow_complete:plum,ivory", + }, + } + ], + } + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "run.json.partial" + path.write_text(json.dumps(checkpoint), encoding="utf-8") + resumed = load_partial_pairs(path, tasks, orders) + self.assertEqual(len(resumed), 1) + self.assertTrue( + all( + resumed[0][arm]["final_correct"] + for arm in ("stage_batched", "compiled", "speculative") + ) + ) + checkpoint["pairs"][0]["arm_order"] = orders[1] + path.write_text(json.dumps(checkpoint), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "does not match"): + load_partial_pairs(path, tasks, orders) + + +if __name__ == "__main__": + unittest.main() diff --git a/optimizations/ooo_spec_lucebox5_cpu/test_trace_compiled_tool_executor.py b/optimizations/ooo_spec_lucebox5_cpu/test_trace_compiled_tool_executor.py new file mode 100644 index 000000000..9c95884bf --- /dev/null +++ b/optimizations/ooo_spec_lucebox5_cpu/test_trace_compiled_tool_executor.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import unittest + +from benchmark_trace_compiled_workflows import mine_pattern, simulated_tool_result +from test_benchmark_trace_compiled_workflows import workflow_trace +from trace_compiled_tool_executor import execute_macro, resolve_items + + +class TraceCompiledToolExecutorTest(unittest.TestCase): + def setUp(self) -> None: + self.pattern = mine_pattern( + [ + workflow_trace("first@example.test", "Rome"), + workflow_trace("second@example.test", "Milan"), + ] + ) + self.items = [ + {"customer_email": "alice@example.test", "destination": "Turin"}, + {"customer_email": "bob@example.test", "destination": "Naples"}, + ] + self.workflow_ref = "workflow_alpha" + self.registry = { + "schema_version": 1, + "pattern_fingerprint": self.pattern.fingerprint, + "workflows": { + self.workflow_ref: { + "pattern_fingerprint": self.pattern.fingerprint, + "items": self.items, + } + }, + } + + @staticmethod + def fake_leaf(request: dict) -> dict: + return {"ok": True, "result": simulated_tool_result(request["call"])} + + def test_executes_every_branch_and_preserves_order(self) -> None: + call = { + "name": self.pattern.macro_name, + "arguments": {"workflow_ref": self.workflow_ref}, + } + request = {"call": call} + envelope = execute_macro( + request, self.pattern, self.fake_leaf, self.registry + ) + result = envelope["result"] + self.assertEqual(result["call_count"], 10) + self.assertEqual([branch["root"] for branch in result["branches"]], self.items) + self.assertEqual( + [len(branch["steps"]) for branch in result["branches"]], [5, 5] + ) + self.assertFalse(result["side_effects"]) + + def test_rejects_unknown_or_missing_inputs(self) -> None: + with self.assertRaisesRegex(ValueError, "fields"): + resolve_items( + {"workflow_ref": self.workflow_ref}, + self.pattern, + { + **self.registry, + "workflows": { + self.workflow_ref: { + "pattern_fingerprint": self.pattern.fingerprint, + "items": [{**self.items[0], "undeclared": "value"}], + } + }, + }, + ) + with self.assertRaisesRegex(ValueError, "workflow_ref"): + resolve_items({"items": self.items}, self.pattern, self.registry) + + +if __name__ == "__main__": + unittest.main() diff --git a/optimizations/ooo_spec_lucebox5_cpu/trace_compiled_tool_executor.py b/optimizations/ooo_spec_lucebox5_cpu/trace_compiled_tool_executor.py new file mode 100755 index 000000000..20cd771fc --- /dev/null +++ b/optimizations/ooo_spec_lucebox5_cpu/trace_compiled_tool_executor.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +"""Execute a trace-compiled read-only workflow through the tool-spec protocol. + +The engine treats the compiled workflow like any other predicted tool: Qwen +proposes its typed arguments, DS4 remains authoritative, and the private result +is released only after an exact call match. Independent workflow branches run +concurrently inside the CPU-pinned executor process. +""" + +from __future__ import annotations + +import json +import os +import re +import sys +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Any, Callable + +from benchmark_trace_compiled_workflows import CompiledPattern, load_training_traces, mine_pattern +from bfcl_replay_tool_executor import ( + PROTOCOL, + call_ref, + call_sha256, + execute as execute_leaf, +) + + +MAX_BRANCHES = 4 +TRAINING_REPORT_ENV = "DFLASH_TRACE_TRAINING_REPORT" +WORKFLOW_REGISTRY_ENV = "DFLASH_TRACE_WORKFLOW_REGISTRY" +LeafExecutor = Callable[[dict[str, Any]], dict[str, Any]] + + +def load_pattern() -> CompiledPattern: + default = Path(__file__).with_name("results") / "trace-compiled-training-traces.json" + report = Path(os.environ.get(TRAINING_REPORT_ENV, str(default))) + return mine_pattern(load_training_traces(report, required_steps=5)) + + +def load_registry() -> dict[str, Any]: + default = Path(__file__).with_name("results") / "trace-workflow-registry.json" + path = Path(os.environ.get(WORKFLOW_REGISTRY_ENV, str(default))) + registry = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(registry, dict) or registry.get("schema_version") != 1: + raise ValueError("compiled workflow registry is invalid") + return registry + + +def validate_items(items: Any, pattern: CompiledPattern) -> list[dict[str, str]]: + if not isinstance(items, list) or not 1 <= len(items) <= MAX_BRANCHES: + raise ValueError(f"customers must contain between 1 and {MAX_BRANCHES} items") + expected_fields = set(pattern.root_fields) + validated = [] + for item in items: + if not isinstance(item, dict) or set(item) != expected_fields: + raise ValueError("customer fields do not match the compiled workflow") + if not all(isinstance(value, str) and value for value in item.values()): + raise ValueError("compiled workflow inputs must be non-empty strings") + validated.append(dict(item)) + return validated + + +def resolve_items( + arguments: Any, pattern: CompiledPattern, registry: dict[str, Any] +) -> list[dict[str, str]]: + if not isinstance(arguments, dict) or set(arguments) != {"workflow_ref"}: + raise ValueError("compiled workflow requires only a workflow_ref") + workflow_ref = arguments["workflow_ref"] + if not isinstance(workflow_ref, str) or re.fullmatch( + r"workflow_[a-z]+", workflow_ref + ) is None: + raise ValueError("workflow_ref is malformed") + if registry.get("pattern_fingerprint") != pattern.fingerprint: + raise ValueError("workflow registry pattern does not match the executor") + workflows = registry.get("workflows") + entry = workflows.get(workflow_ref) if isinstance(workflows, dict) else None + if ( + not isinstance(entry, dict) + or entry.get("pattern_fingerprint") != pattern.fingerprint + ): + raise ValueError("workflow_ref is unknown or bound to another pattern") + return validate_items(entry.get("items"), pattern) + + +def execute_branch( + request: dict[str, Any], + pattern: CompiledPattern, + root: dict[str, str], + leaf_executor: LeafExecutor, +) -> dict[str, Any]: + previous: dict[str, Any] | None = None + steps = [] + for index in range(len(pattern.steps)): + call = pattern.instantiate(root, previous, index) + leaf_request = {**request, "call": call} + envelope = leaf_executor(leaf_request) + result = envelope.get("result") if isinstance(envelope, dict) else None + if not envelope.get("ok") or not isinstance(result, dict): + raise RuntimeError("leaf tool returned an invalid result") + if ( + result.get("call_sha256") != call_sha256(call) + or result.get("call_ref") != call_ref(call) + or result.get("tool_name") != call["name"] + or result.get("side_effects") is not False + ): + raise RuntimeError("leaf tool result did not match its compiled call") + previous = result + steps.append({"call": call, "tool_result": result}) + return {"root": root, "steps": steps, "final_ref": steps[-1]["tool_result"]["call_ref"]} + + +def execute_macro( + request: dict[str, Any], + pattern: CompiledPattern, + leaf_executor: LeafExecutor = execute_leaf, + registry: dict[str, Any] | None = None, +) -> dict[str, Any]: + call = request.get("call") + if not isinstance(call, dict) or call.get("name") != pattern.macro_name: + raise ValueError("request is not for the compiled workflow") + items = resolve_items( + call.get("arguments"), pattern, registry if registry is not None else load_registry() + ) + started = time.perf_counter() + with ThreadPoolExecutor(max_workers=len(items), thread_name_prefix="compiled-workflow") as pool: + futures = [ + pool.submit(execute_branch, request, pattern, root, leaf_executor) + for root in items + ] + branches = [future.result() for future in futures] + elapsed_ms = (time.perf_counter() - started) * 1_000.0 + affinity = sorted(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else [] + return { + "ok": True, + "result": { + "call_sha256": call_sha256(call), + "call_ref": call_ref(call), + "tool_name": pattern.macro_name, + "workflow_fingerprint": pattern.fingerprint, + "branches": branches, + "call_count": len(items) * len(pattern.steps), + "elapsed_ms": elapsed_ms, + "cpu_affinity": affinity, + "side_effects": False, + }, + } + + +def execute(request: dict[str, Any], pattern: CompiledPattern) -> dict[str, Any]: + call = request.get("call") + if isinstance(call, dict) and call.get("name") == pattern.macro_name: + return execute_macro(request, pattern) + return execute_leaf(request) + + +def main() -> int: + if sys.argv[1:] != ["--dflash-tool-spec-v1"]: + print("expected --dflash-tool-spec-v1", file=sys.stderr) + return 2 + try: + line = sys.stdin.readline() + if not line: + raise ValueError("missing request") + request = json.loads(line) + if not isinstance(request, dict) or request.get("protocol") != PROTOCOL: + raise ValueError("unsupported tool-speculation request") + print(json.dumps(execute(request, load_pattern()), separators=(",", ":")), flush=True) + return 0 + except (OSError, RuntimeError, ValueError, json.JSONDecodeError) as error: + print(str(error), file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index 46ecfda81..ef4206105 100644 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -446,6 +446,7 @@ add_library(dflash_common STATIC src/common/dflash_draft_ipc.cpp src/common/dflash_draft_ipc_daemon.cpp src/common/pflash_drafter_ipc.cpp + src/common/qwen3_tool_predictor_ipc.cpp src/common/dflash_draft_graph.cpp src/common/dflash_draft_kv.cpp src/common/dflash_spec_decode.cpp @@ -497,6 +498,8 @@ add_library(dflash_common STATIC src/server/chat_template.cpp src/server/tool_parser.cpp src/server/tool_hint.cpp + src/server/semantic_tool_hint.cpp + src/server/native_semantic_tool_predictor.cpp src/server/reasoning.cpp src/server/tool_memory.cpp src/server/sse_emitter.cpp @@ -1308,6 +1311,14 @@ if(DFLASH27B_TESTS) target_include_directories(smoke_qwen3_forward PRIVATE ${DFLASH27B_SRC_INCLUDE_DIRS}) target_link_libraries(smoke_qwen3_forward PRIVATE dflash_common ggml ${DFLASH27B_GGML_BACKEND_TARGET}) endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/smoke_qwen3_tool_predictor_ipc.cpp") + add_executable(smoke_qwen3_tool_predictor_ipc + test/smoke_qwen3_tool_predictor_ipc.cpp) + target_include_directories(smoke_qwen3_tool_predictor_ipc PRIVATE + ${DFLASH27B_SRC_INCLUDE_DIRS}) + target_link_libraries(smoke_qwen3_tool_predictor_ipc PRIVATE + dflash_common ggml ${DFLASH27B_GGML_BACKEND_TARGET}) + endif() if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_vs_oracle.cpp") add_executable(test_vs_oracle test/test_vs_oracle.cpp) target_include_directories(test_vs_oracle PRIVATE ${DFLASH27B_SRC_INCLUDE_DIRS}) @@ -1522,6 +1533,8 @@ if(DFLASH27B_TESTS) set(_server_unit_sources test/test_unit_main.cpp test/test_server_unit.cpp + test/test_tool_speculation.cpp + test/test_semantic_tool_hint.cpp test/test_anchor_params.cpp test/test_derived_scalars.cpp test/test_adaptive_keep_ratio.cpp @@ -1554,6 +1567,7 @@ if(DFLASH27B_TESTS) add_executable(test_server_unit ${_server_unit_sources}) target_sources(test_server_unit PRIVATE src/server/http_server.cpp + src/server/tool_speculation.cpp src/server/scheduler.cpp src/server/model_card.cpp src/server/prompt_normalize.cpp @@ -1925,6 +1939,7 @@ if(DFLASH27B_SERVER) add_executable(dflash_server src/server/server_main.cpp src/server/http_server.cpp + src/server/tool_speculation.cpp src/server/scheduler.cpp src/server/model_card.cpp src/server/prompt_normalize.cpp @@ -1932,6 +1947,8 @@ if(DFLASH27B_SERVER) target_include_directories(dflash_server PRIVATE ${DFLASH27B_SRC_INCLUDE_DIRS}) if(DFLASH27B_GPU_BACKEND STREQUAL "hip") target_compile_definitions(dflash_server PRIVATE DFLASH27B_BACKEND_HIP=1 GGML_USE_HIP) + target_sources(dflash_server PRIVATE + src/server/tool_speculation_hip_probe.cpp) if(DFLASH27B_ENABLE_MIXED_CUDA_HIP) target_compile_definitions(dflash_server PRIVATE DFLASH27B_BACKEND_MIXED=1) @@ -1958,7 +1975,11 @@ if(DFLASH27B_SERVER) find_package(CUDAToolkit REQUIRED) target_link_libraries(dflash_server PRIVATE CUDA::cudart) else() - target_link_libraries(dflash_server PRIVATE hip::host) + # ggml-hip finds hipBLAS in a child-directory scope. The trusted + # in-process tool adapter also calls hipBLAS directly, so import + # the target in this scope before linking the server executable. + find_package(hipblas REQUIRED) + target_link_libraries(dflash_server PRIVATE hip::host roc::hipblas) endif() # Copy share/status.html next to the binary so it can be found at runtime. @@ -1980,6 +2001,7 @@ if(DFLASH27B_SERVER) add_executable(backend_ipc_daemon src/ipc/backend_ipc_main.cpp src/common/pflash_drafter_ipc_daemon.cpp + src/common/qwen3_tool_predictor_ipc_daemon.cpp ) target_include_directories(backend_ipc_daemon PRIVATE ${DFLASH27B_SRC_INCLUDE_DIRS}) if(DFLASH27B_GPU_BACKEND STREQUAL "hip") diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/common.cuh b/server/deps/llama.cpp/ggml/src/ggml-cuda/common.cuh index 31b04cd14..0680df55e 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cuda/common.cuh +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/common.cuh @@ -25,8 +25,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -161,6 +163,60 @@ static int ggml_cuda_highest_compiled_arch(const int arch) { #define GGML_CUDA_MAX_STREAMS 8 +#if defined(GGML_USE_HIP) +// Optional Lucebox experiment: reserve the lowest CUs on one HIP device for a +// trusted in-process tool stream. Every lazily-created ggml stream on that +// device receives the complementary CU mask, creating a real disjoint lane +// inside one HIP context. Default behavior is unchanged when the variable is +// absent. Format: DFLASH_HIP_RESERVED_TOOL_LANE=DEVICE:CUS (for example 0:1). +static inline bool dflash_hip_model_stream_mask( + int device, std::vector & mask, int & reserved_cus) { + const char * raw = std::getenv("DFLASH_HIP_RESERVED_TOOL_LANE"); + if (!raw || !*raw) return false; + + errno = 0; + char * separator = nullptr; + const long configured_device = std::strtol(raw, &separator, 10); + if (errno != 0 || separator == raw || !separator || *separator != ':') { + GGML_ABORT( + "DFLASH_HIP_RESERVED_TOOL_LANE must be DEVICE:CUS, got '%s'\n", + raw); + } + char * end = nullptr; + errno = 0; + const long configured_cus = std::strtol(separator + 1, &end, 10); + if (errno != 0 || !end || *end != '\0' || configured_device < 0 || + configured_cus <= 0) { + GGML_ABORT( + "DFLASH_HIP_RESERVED_TOOL_LANE must be DEVICE:CUS with positive " + "integers, got '%s'\n", raw); + } + if (configured_device != device) return false; + + hipDeviceProp_t properties{}; + const hipError_t status = hipGetDeviceProperties(&properties, device); + if (status != hipSuccess) { + GGML_ABORT( + "DFLASH_HIP_RESERVED_TOOL_LANE could not inspect HIP device %d: " + "%s\n", + device, hipGetErrorString(status)); + } + if (configured_cus >= properties.multiProcessorCount) { + GGML_ABORT( + "DFLASH_HIP_RESERVED_TOOL_LANE reserves %ld of %d CUs on device " + "%d; at least one model CU is required\n", + configured_cus, properties.multiProcessorCount, device); + } + reserved_cus = static_cast(configured_cus); + mask.assign( + static_cast((properties.multiProcessorCount + 31) / 32), 0); + for (int cu = reserved_cus; cu < properties.multiProcessorCount; ++cu) { + mask[static_cast(cu / 32)] |= uint32_t{1} << (cu % 32); + } + return true; +} +#endif + [[noreturn]] void ggml_cuda_error(const char * stmt, const char * func, const char * file, int line, const char * msg); @@ -1486,6 +1542,37 @@ struct ggml_backend_cuda_context { cudaStream_t stream(int device, int stream) { if (streams[device][stream] == nullptr) { ggml_cuda_set_device(device); +#if defined(GGML_USE_HIP) + std::vector model_cu_mask; + int reserved_cus = 0; + const bool disjoint_tool_lane = dflash_hip_model_stream_mask( + device, model_cu_mask, reserved_cus); + if (disjoint_tool_lane) { + CUDA_CHECK(hipExtStreamCreateWithCUMask( + &streams[device][stream], + static_cast(model_cu_mask.size()), + model_cu_mask.data())); + if (low_priority_streams) { +#if HIP_VERSION_MAJOR >= 7 + hipStreamAttrValue priority{}; + priority.priority = stream_priority; + CUDA_CHECK(hipStreamSetAttribute( + streams[device][stream], + hipStreamAttributePriority, &priority)); +#else + GGML_ABORT( + "DFLASH_HIP_RESERVED_TOOL_LANE requires ROCm 7+ " + "to preserve low-priority DSpark streams\n"); +#endif + } + if (stream == 0) { + std::fprintf(stderr, + "ggml_hip: device %d model streams exclude %d " + "low CU(s) reserved for in-process tools\n", + device, reserved_cus); + } + } else +#endif if (low_priority_streams) { CUDA_CHECK(cudaStreamCreateWithPriority( &streams[device][stream], cudaStreamNonBlocking, diff --git a/server/src/common/backend_ipc.cpp b/server/src/common/backend_ipc.cpp index 2a98b8e9c..4e72cd86a 100644 --- a/server/src/common/backend_ipc.cpp +++ b/server/src/common/backend_ipc.cpp @@ -29,6 +29,7 @@ const char * backend_ipc_mode_name(BackendIpcMode mode) { case BackendIpcMode::Invalid: return "invalid"; case BackendIpcMode::DFlashDraft: return "dflash-draft"; case BackendIpcMode::PFlashCompress: return "pflash-compress"; + case BackendIpcMode::Qwen3ToolPredict: return "qwen3-tool-predict"; case BackendIpcMode::Qwen35TargetShard: return "qwen35-target-shard"; case BackendIpcMode::Gemma4TargetShard: return "gemma4-target-shard"; case BackendIpcMode::LagunaTargetShard: return "laguna-target-shard"; @@ -47,6 +48,10 @@ bool parse_backend_ipc_mode(const std::string & value, BackendIpcMode & out) { out = BackendIpcMode::PFlashCompress; return true; } + if (value == "qwen3-tool-predict") { + out = BackendIpcMode::Qwen3ToolPredict; + return true; + } if (value == "qwen35-target-shard") { out = BackendIpcMode::Qwen35TargetShard; return true; diff --git a/server/src/common/backend_ipc.h b/server/src/common/backend_ipc.h index d995759ac..8731f3714 100644 --- a/server/src/common/backend_ipc.h +++ b/server/src/common/backend_ipc.h @@ -25,6 +25,7 @@ enum class BackendIpcMode { Invalid, DFlashDraft, PFlashCompress, + Qwen3ToolPredict, Qwen35TargetShard, Gemma4TargetShard, LagunaTargetShard, diff --git a/server/src/common/model_backend.h b/server/src/common/model_backend.h index 12445e6b2..36970babf 100644 --- a/server/src/common/model_backend.h +++ b/server/src/common/model_backend.h @@ -197,6 +197,10 @@ struct GenerateRequest { // path returns success but emits no tokens, so each backend can route the // retry through its existing AR path without copying retry policy. bool force_ar_decode = false; + // Opt out of the common speculative-to-AR empty-output retry. Tool + // speculation sets this false so the external optimization can never + // change the request's model decode strategy. + bool allow_decode_mode_retry = true; }; // Stable, backend-independent generation failure categories. Backends should @@ -369,6 +373,7 @@ struct ModelBackend { static bool should_retry_empty_spec_decode(const GenerateRequest & req, const GenerateResult & result) { return req.n_gen > 0 + && req.allow_decode_mode_retry && !req.force_ar_decode && result.ok() && result.spec_decode_ran diff --git a/server/src/common/qwen3_tool_predictor_ipc.cpp b/server/src/common/qwen3_tool_predictor_ipc.cpp new file mode 100644 index 000000000..a7bdf038c --- /dev/null +++ b/server/src/common/qwen3_tool_predictor_ipc.cpp @@ -0,0 +1,119 @@ +#include "qwen3_tool_predictor_ipc.h" + +#include "io_utils.h" + +#include +#include + +namespace dflash::common { + +bool Qwen3ToolPredictorIpcClient::start( + const std::string & bin, + const std::string & model_path, + int gpu, + int max_ctx, + const std::string & work_dir) { +#if defined(_WIN32) + (void)bin; (void)model_path; (void)gpu; (void)max_ctx; (void)work_dir; + std::fprintf(stderr, + "Qwen3 tool-predictor IPC is only implemented on POSIX hosts\n"); + return false; +#else + std::lock_guard lock(mutex_); + close_locked(); + if (bin.empty() || model_path.empty() || max_ctx <= 0) return false; + + BackendIpcLaunchConfig launch; + launch.bin = bin; + launch.mode = BackendIpcMode::Qwen3ToolPredict; + launch.payload_path = model_path; + launch.work_dir = work_dir; + launch.args.push_back("--target-gpu=" + std::to_string(std::max(0, gpu))); + launch.args.push_back("--max-ctx=" + std::to_string(max_ctx)); + if (!process_.start(launch)) { + std::fprintf(stderr, "[tool-predictor-ipc] backend process start failed\n"); + return false; + } + active_ = true; + std::fprintf(stderr, + "[tool-predictor-ipc] ready model=%s gpu=%d max_ctx=%d work_dir=%s\n", + model_path.c_str(), std::max(0, gpu), max_ctx, + process_.work_dir().c_str()); + return true; +#endif +} + +bool Qwen3ToolPredictorIpcClient::predict( + const std::vector & prompt_ids, + int max_tokens, + std::vector & output_ids, + std::string & error) { + output_ids.clear(); + error.clear(); +#if defined(_WIN32) + (void)prompt_ids; (void)max_tokens; + error = "native_predictor_ipc_unsupported"; + return false; +#else + std::lock_guard lock(mutex_); + FILE * command = process_.command_stream(); + const int stream_fd = process_.stream_fd(); + if (!active_ || !command || stream_fd < 0) { + error = "native_predictor_not_active"; + return false; + } + if (prompt_ids.empty() || max_tokens <= 0) { + error = "native_predictor_invalid_request"; + return false; + } + + const std::string path = process_.next_path("tool_predictor_prompt"); + if (!write_int32_file(path, prompt_ids)) { + error = "native_predictor_prompt_write_failed"; + return false; + } + + std::fprintf(command, "predict %d %s\n", max_tokens, path.c_str()); + std::fflush(command); + + int32_t status = -1; + bool ok = read_exact_fd(stream_fd, &status, sizeof(status)) && status == 0; + if (ok) { + int32_t count = -1; + ok = read_exact_fd(stream_fd, &count, sizeof(count)) && + count > 0 && count <= max_tokens; + if (ok) { + output_ids.assign(static_cast(count), 0); + ok = read_exact_fd(stream_fd, output_ids.data(), + output_ids.size() * sizeof(int32_t)); + } + } + std::remove(path.c_str()); + if (!ok) { + error = status == 0 + ? "native_predictor_invalid_response" + : "native_predictor_generation_failed"; + output_ids.clear(); + close_locked(); + return false; + } + return true; +#endif +} + +bool Qwen3ToolPredictorIpcClient::active() const { + std::lock_guard lock(mutex_); + return active_; +} + +void Qwen3ToolPredictorIpcClient::close_locked() { + process_.close(); + active_ = false; +} + +void Qwen3ToolPredictorIpcClient::close() { + std::lock_guard lock(mutex_); + close_locked(); +} + +} // namespace dflash::common diff --git a/server/src/common/qwen3_tool_predictor_ipc.h b/server/src/common/qwen3_tool_predictor_ipc.h new file mode 100644 index 000000000..5cab9513c --- /dev/null +++ b/server/src/common/qwen3_tool_predictor_ipc.h @@ -0,0 +1,57 @@ +// Persistent Qwen3 tool-predictor IPC lane. +// +// The HTTP server tokenizes the predictor prompt with the predictor's own +// vocabulary, then sends token IDs to a small out-of-process Qwen3 backend. +// Keeping this lane behind BackendIpcProcess isolates the target decoder from +// predictor crashes and lets heterogeneous deployments choose a different GPU. + +#pragma once + +#include "backend_ipc.h" + +#include +#include +#include +#include +#include + +namespace dflash::common { + +class Qwen3ToolPredictorIpcClient { +public: + Qwen3ToolPredictorIpcClient() = default; + Qwen3ToolPredictorIpcClient(const Qwen3ToolPredictorIpcClient &) = delete; + Qwen3ToolPredictorIpcClient & operator=( + const Qwen3ToolPredictorIpcClient &) = delete; + ~Qwen3ToolPredictorIpcClient() { close(); } + + bool start(const std::string & bin, + const std::string & model_path, + int gpu, + int max_ctx, + const std::string & work_dir); + + // Requests are serialized: one compact predictor model owns one KV cache. + // On transport or generation failure the lane closes and fails shut. + bool predict(const std::vector & prompt_ids, + int max_tokens, + std::vector & output_ids, + std::string & error); + + bool active() const; + void close(); + +private: + void close_locked(); + + mutable std::mutex mutex_; + BackendIpcProcess process_; + bool active_ = false; +}; + +int run_qwen3_tool_predictor_ipc_daemon(const char * model_path, + int gpu, + int max_ctx, + int stream_fd); + +} // namespace dflash::common diff --git a/server/src/common/qwen3_tool_predictor_ipc_daemon.cpp b/server/src/common/qwen3_tool_predictor_ipc_daemon.cpp new file mode 100644 index 000000000..9ebe96fd3 --- /dev/null +++ b/server/src/common/qwen3_tool_predictor_ipc_daemon.cpp @@ -0,0 +1,120 @@ +#include "qwen3_tool_predictor_ipc.h" + +#include "io_utils.h" +#include "model_backend.h" +#include "qwen3/qwen3_backend.h" + +#include +#include +#include +#include +#include + +namespace dflash::common { +namespace { + +bool send_status(int stream_fd, int32_t status) { + return write_exact_fd(stream_fd, &status, sizeof(status)); +} + +} // namespace + +int run_qwen3_tool_predictor_ipc_daemon( + const char * model_path, + int gpu, + int max_ctx, + int stream_fd) { +#if defined(_WIN32) + (void)model_path; (void)gpu; (void)max_ctx; (void)stream_fd; + return 2; +#else + if (!model_path || !*model_path || stream_fd < 0 || max_ctx <= 0) { + std::fprintf(stderr, + "usage: backend_ipc_daemon --backend-ipc-mode=qwen3-tool-predict " + " --stream-fd=FD --target-gpu=N --max-ctx=N\n"); + return 2; + } + + Qwen3BackendConfig config; + config.model_path = model_path; + config.device.backend = PlacementBackend::Auto; + config.device.gpu = std::max(0, gpu); + config.device.max_ctx = max_ctx; + config.chunk = 512; + + Qwen3Backend backend(config); + if (!backend.init()) { + std::fprintf(stderr, "[tool-predictor-daemon] Qwen3 init failed\n"); + send_status(stream_fd, -1); + return 1; + } + std::fprintf(stderr, + "[tool-predictor-daemon] ready gpu=%d max_ctx=%d\n", + std::max(0, gpu), max_ctx); + send_status(stream_fd, 0); + + std::string line; + while (std::getline(std::cin, line)) { + std::istringstream input(line); + std::string command; + input >> command; + if (command == "quit" || command == "exit") break; + if (command != "predict") { + std::fprintf(stderr, + "[tool-predictor-daemon] unknown command: %s\n", + line.c_str()); + send_status(stream_fd, -1); + continue; + } + + int max_tokens = 0; + input >> max_tokens; + const std::string path = read_line_tail(input); + if (max_tokens <= 0 || path.empty()) { + send_status(stream_fd, -1); + continue; + } + const auto prompt = read_int32_file(path); + if (prompt.empty() || + prompt.size() + static_cast(max_tokens) > + static_cast(max_ctx)) { + std::fprintf(stderr, + "[tool-predictor-daemon] invalid context prompt=%zu max_tokens=%d max_ctx=%d\n", + prompt.size(), max_tokens, max_ctx); + send_status(stream_fd, -1); + continue; + } + + GenerateRequest request; + request.prompt = prompt; + request.n_gen = max_tokens; + request.do_sample = false; + request.stream = false; + DaemonIO io; + const GenerateResult result = backend.generate(request, io); + if (!result.ok() || result.tokens.empty()) { + std::fprintf(stderr, + "[tool-predictor-daemon] generation failed code=%s\n", + result.error_code().data()); + send_status(stream_fd, -1); + continue; + } + + const int32_t count = static_cast(result.tokens.size()); + if (!send_status(stream_fd, 0) || + !write_exact_fd(stream_fd, &count, sizeof(count)) || + !write_exact_fd(stream_fd, result.tokens.data(), + result.tokens.size() * sizeof(int32_t))) { + std::fprintf(stderr, + "[tool-predictor-daemon] response write failed\n"); + break; + } + } + + backend.shutdown(); + std::fprintf(stderr, "[tool-predictor-daemon] stopped\n"); + return 0; +#endif +} + +} // namespace dflash::common diff --git a/server/src/ipc/backend_ipc_main.cpp b/server/src/ipc/backend_ipc_main.cpp index b2fae7791..e2764ef9d 100644 --- a/server/src/ipc/backend_ipc_main.cpp +++ b/server/src/ipc/backend_ipc_main.cpp @@ -8,6 +8,7 @@ #include "gemma4/gemma4_layer_split_adapter.h" #include "laguna/laguna_layer_split_adapter.h" #include "pflash_drafter_ipc.h" +#include "qwen3_tool_predictor_ipc.h" #include "common/platform_env.h" #include "qwen35_target_shard_ipc.h" @@ -117,6 +118,8 @@ int main(int argc, char ** argv) { "[--shared-payload-fd=FD --shared-payload-bytes=N] [--draft-gpu=N]\n" " or: %s --backend-ipc-mode=pflash-compress " "--stream-fd=FD [--draft-gpu=N]\n" + " or: %s --backend-ipc-mode=qwen3-tool-predict " + "--stream-fd=FD --target-gpu=N --max-ctx=N\n" " or: %s --backend-ipc-mode=qwen35-target-shard " "--stream-fd=FD --target-gpu=N --layer-begin=N --layer-end=N " "--max-ctx=N [--hidden=N --vocab=N --max-tokens=N]\n" @@ -138,6 +141,8 @@ int main(int argc, char ** argv) { argv[0], argv[0], argv[0], + argv[0], + argv[0], argv[0]); return 2; } @@ -315,6 +320,9 @@ int main(int argc, char ** argv) { shared_payload_bytes); case BackendIpcMode::PFlashCompress: return run_pflash_drafter_ipc_daemon(payload_path, draft_gpu, stream_fd); + case BackendIpcMode::Qwen3ToolPredict: + return run_qwen3_tool_predictor_ipc_daemon( + payload_path, target_gpu, max_ctx, stream_fd); case BackendIpcMode::Qwen35TargetShard: if (target_gpus.empty()) target_gpus.push_back(target_gpu); if (layer_begins.empty()) layer_begins.push_back(layer_begin); diff --git a/server/src/qwen3/qwen3_loader.cpp b/server/src/qwen3/qwen3_loader.cpp index 583261992..52bc3291e 100644 --- a/server/src/qwen3/qwen3_loader.cpp +++ b/server/src/qwen3/qwen3_loader.cpp @@ -134,7 +134,11 @@ bool load_qwen3_drafter_model(const std::string & path, out.head_dim = (int)get_u32(gctx, "qwen3.attention.key_length", 128); out.rope_theta = get_f32(gctx, "qwen3.rope.freq_base", 1000000.0f); - // Detect weight quant type from blk.0.attn_q.weight; support BF16 and Q8_0. + // Preserve Q8_0 storage when the predictor reuses the production compact + // GGUF. Activations/KV still use the backend precision policy; ggml's + // mul_mat and get_rows kernels dequantize Q8_0 weights as they are read. + // This avoids expanding a 0.6B sidecar to BF16 merely to use the native + // IPC lane. ggml_type wtype = GGML_TYPE_BF16; { int64_t tidx = gguf_find_tensor(gctx, "blk.0.attn_q.weight"); @@ -142,8 +146,16 @@ bool load_qwen3_drafter_model(const std::string & path, wtype = gguf_get_tensor_type(gctx, tidx); } } + if (wtype == GGML_TYPE_Q8_0) { + out.weight_type = GGML_TYPE_Q8_0; + } else if (wtype != GGML_TYPE_BF16 && wtype != GGML_TYPE_F16) { + set_last_error(std::string("unsupported Qwen3-0.6B weight type: ") + + ggml_type_name(wtype)); + gguf_free(gctx); + return false; + } std::fprintf(stderr, "[qwen3-0.6b] detected weight type: %s\n", - wtype == GGML_TYPE_Q8_0 ? "Q8_0" : "BF16"); + ggml_type_name(wtype)); std::fflush(stderr); // Compute total tensor metadata size for context allocation. diff --git a/server/src/server/chat_template.cpp b/server/src/server/chat_template.cpp index 2d5970efa..b402752dc 100644 --- a/server/src/server/chat_template.cpp +++ b/server/src/server/chat_template.cpp @@ -76,7 +76,8 @@ std::string render_chat_template( ChatFormat format, bool add_generation_prompt, bool enable_thinking, - const std::string & tools_json) + const std::string & tools_json, + bool tool_call_required) { std::string result; bool has_tools = !tools_json.empty() && tools_json != "[]" && tools_json != "null"; @@ -383,9 +384,13 @@ std::string render_chat_template( result += "For each function call, you MUST return a single JSON object " "within '' and '' tags, " "containing the function name and arguments, like this:\n" - "\n" - "{\"name\": \"function_name\", \"arguments\": {\"param_name\": \"value\"}}\n" - "\n\n"; + "\n" + "{\"name\": \"function_name\", \"arguments\": {\"param_name\": \"value\"}}\n" + "\n\n"; + if (tool_call_required) { + result += "You MUST call exactly one available function and emit no " + "text outside its tags.\n\n"; + } } result += system_content; diff --git a/server/src/server/chat_template.h b/server/src/server/chat_template.h index ecade9217..b7825153c 100644 --- a/server/src/server/chat_template.h +++ b/server/src/server/chat_template.h @@ -39,14 +39,17 @@ enum class ChatFormat { // false → assistant starts with \n\n\n\n (skip thinking) // // `tools_json` is an optional JSON string containing the tool definitions -// array. When non-empty, the Qwen3/3.5 template injects a tool preamble -// into the system message instructing the model how to emit tags. +// array. When non-empty, tool-capable templates inject a tool preamble into +// the system message instructing the model how to emit tags. +// `tool_call_required` strengthens that instruction for OpenAI +// `tool_choice="required"` and forced-function requests. std::string render_chat_template( const std::vector & messages, ChatFormat format, bool add_generation_prompt = true, bool enable_thinking = false, - const std::string & tools_json = ""); + const std::string & tools_json = "", + bool tool_call_required = false); // Detect the appropriate chat format for an architecture. ChatFormat chat_format_for_arch(const std::string & arch); diff --git a/server/src/server/http_server.cpp b/server/src/server/http_server.cpp index e92f495bf..d079a1934 100644 --- a/server/src/server/http_server.cpp +++ b/server/src/server/http_server.cpp @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -72,6 +73,7 @@ static inline bool sock_is_eintr (int e) { return e == WSAEINTR; } static inline bool sock_is_eagain(int e) { return e == WSAEWOULDBLOCK; } #else #include +#include #include static inline int sock_get_flags(SocketHandle fd) { return fcntl(fd, F_GETFL, 0); } static inline void sock_set_nonblock(SocketHandle fd) { int f = fcntl(fd, F_GETFL, 0); if (f >= 0) fcntl(fd, F_SETFL, f | O_NONBLOCK); } @@ -465,6 +467,262 @@ static bool curl_forward(const std::string & url, } #endif // DFLASH_HAS_CURL +namespace { + +struct SemanticSidecarUrl { + std::string host; + std::string port; + std::string path; +}; + +bool parse_semantic_sidecar_url( + const std::string & value, SemanticSidecarUrl & out) { + constexpr char kHttpPrefix[] = "http://"; + if (value.rfind(kHttpPrefix, 0) != 0) return false; + const size_t authority_begin = sizeof(kHttpPrefix) - 1; + const size_t path_begin = value.find('/', authority_begin); + const std::string authority = value.substr( + authority_begin, + path_begin == std::string::npos + ? std::string::npos + : path_begin - authority_begin); + if (authority.empty() || authority.find('@') != std::string::npos) { + return false; + } + + const size_t colon = authority.rfind(':'); + if (colon == std::string::npos) { + out.host = authority; + out.port = "80"; + } else { + out.host = authority.substr(0, colon); + out.port = authority.substr(colon + 1); + } + out.path = path_begin == std::string::npos + ? "/" : value.substr(path_begin); + if (out.host.empty() || out.port.empty() || out.path.empty()) return false; + if (out.host.front() == '[' || out.host.find(':') != std::string::npos) { + // The production bridge is loopback IPv4. Reject ambiguous IPv6 + // authority parsing instead of silently connecting to the wrong host. + return false; + } + return std::all_of(out.port.begin(), out.port.end(), [](unsigned char ch) { + return std::isdigit(ch) != 0; + }); +} + +bool semantic_sidecar_send_all( + SocketHandle fd, const char * data, size_t size) { + size_t sent = 0; + while (sent < size) { +#if defined(_WIN32) + const int n = ::send( + fd, data + sent, + static_cast(std::min(size - sent, INT_MAX)), 0); +#else + const ssize_t n = ::send( + fd, data + sent, size - sent, MSG_NOSIGNAL); +#endif + if (n <= 0) return false; + sent += static_cast(n); + } + return true; +} + +void set_semantic_sidecar_socket_timeout(SocketHandle fd, int timeout_ms) { +#if defined(_WIN32) + const DWORD timeout = static_cast(timeout_ms); + setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, + reinterpret_cast(&timeout), sizeof(timeout)); + setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, + reinterpret_cast(&timeout), sizeof(timeout)); +#else + struct timeval timeout{}; + timeout.tv_sec = timeout_ms / 1000; + timeout.tv_usec = (timeout_ms % 1000) * 1000; + setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout)); + setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(timeout)); +#endif +} + +std::string lowercase_ascii(std::string value) { + std::transform(value.begin(), value.end(), value.begin(), + [](unsigned char ch) { return static_cast(std::tolower(ch)); }); + return value; +} + +bool decode_chunked_http_body( + const std::string & encoded, std::string & decoded) { + size_t offset = 0; + while (true) { + const size_t line_end = encoded.find("\r\n", offset); + if (line_end == std::string::npos) return false; + const std::string size_text = encoded.substr(offset, line_end - offset); + const size_t extension = size_text.find(';'); + const std::string hex = size_text.substr(0, extension); + size_t parsed = 0; + unsigned long chunk_size = 0; + try { + chunk_size = std::stoul(hex, &parsed, 16); + } catch (...) { + return false; + } + if (parsed != hex.size()) return false; + offset = line_end + 2; + if (chunk_size == 0) return true; + if (chunk_size > encoded.size() - std::min(offset, encoded.size())) { + return false; + } + decoded.append(encoded, offset, static_cast(chunk_size)); + offset += static_cast(chunk_size); + if (offset + 2 > encoded.size() || + encoded.compare(offset, 2, "\r\n") != 0) { + return false; + } + offset += 2; + } +} + +SemanticToolPrediction request_semantic_tool_prediction( + const SemanticToolPredictorConfig & config, + const json & payload, + const json & request_tools) { + const auto started = std::chrono::steady_clock::now(); + SemanticToolPrediction prediction; + prediction.source = config.model; + auto finish = [&]() { + prediction.wall_ms = std::chrono::duration( + std::chrono::steady_clock::now() - started).count(); + return prediction; + }; + + SemanticSidecarUrl url; + if (!parse_semantic_sidecar_url(config.url, url)) { + prediction.error = "predictor_url_must_be_http_host_port_path"; + return finish(); + } + + struct addrinfo hints{}; + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + struct addrinfo * addresses = nullptr; + if (getaddrinfo(url.host.c_str(), url.port.c_str(), &hints, &addresses) != 0) { + prediction.error = "predictor_host_resolution_failed"; + return finish(); + } + + SocketHandle fd = kInvalidSocket; + for (auto * address = addresses; address; address = address->ai_next) { + fd = socket(address->ai_family, address->ai_socktype, + address->ai_protocol); + if (!socket_is_valid(fd)) continue; + set_semantic_sidecar_socket_timeout(fd, config.timeout_ms); + if (connect(fd, address->ai_addr, + static_cast(address->ai_addrlen)) == 0) { + break; + } + socket_close(fd); + fd = kInvalidSocket; + } + freeaddrinfo(addresses); + if (!socket_is_valid(fd)) { + prediction.error = "predictor_connect_failed"; + return finish(); + } + + const std::string body = payload.dump(); + const std::string request = + "POST " + url.path + " HTTP/1.1\r\n" + + "Host: " + url.host + ":" + url.port + "\r\n" + + "Content-Type: application/json\r\n" + + "Accept: application/json\r\n" + + "Connection: close\r\n" + + "Content-Length: " + std::to_string(body.size()) + "\r\n\r\n" + + body; + if (!semantic_sidecar_send_all(fd, request.data(), request.size())) { + socket_close(fd); + prediction.error = "predictor_send_failed"; + return finish(); + } + + constexpr size_t kMaxResponseBytes = 1024 * 1024; + std::string response; + std::array buffer{}; + while (response.size() < kMaxResponseBytes) { +#if defined(_WIN32) + const int received = recv( + fd, buffer.data(), static_cast(buffer.size()), 0); +#else + const ssize_t received = recv(fd, buffer.data(), buffer.size(), 0); +#endif + if (received == 0) break; + if (received < 0) { + socket_close(fd); + prediction.error = "predictor_receive_failed_or_timed_out"; + return finish(); + } + response.append(buffer.data(), static_cast(received)); + } + socket_close(fd); + if (response.size() >= kMaxResponseBytes) { + prediction.error = "predictor_response_too_large"; + return finish(); + } + + const size_t header_end = response.find("\r\n\r\n"); + const size_t status_end = response.find("\r\n"); + if (header_end == std::string::npos || status_end == std::string::npos) { + prediction.error = "predictor_malformed_http_response"; + return finish(); + } + const std::string status_line = response.substr(0, status_end); + const size_t status_space = status_line.find(' '); + if (status_space == std::string::npos || + status_line.size() < status_space + 4) { + prediction.error = "predictor_malformed_http_status"; + return finish(); + } + const int status = std::atoi(status_line.c_str() + status_space + 1); + if (status < 200 || status >= 300) { + prediction.error = "predictor_http_status_" + std::to_string(status); + return finish(); + } + + const std::string headers = lowercase_ascii( + response.substr(status_end + 2, header_end - status_end - 2)); + const std::string encoded_body = response.substr(header_end + 4); + std::string response_body; + if (headers.find("transfer-encoding: chunked") != std::string::npos) { + if (!decode_chunked_http_body(encoded_body, response_body)) { + prediction.error = "predictor_invalid_chunked_response"; + return finish(); + } + } else { + response_body = encoded_body; + } + + json response_json; + try { + response_json = json::parse(response_body); + } catch (...) { + prediction.error = "predictor_response_invalid_json"; + return finish(); + } + if (!parse_semantic_tool_prediction( + response_json, request_tools, prediction.call, + prediction.error)) { + return finish(); + } + if (!materialize_declared_tool_defaults( + request_tools, prediction.call, prediction.error)) { + return finish(); + } + prediction.ok = true; + return finish(); +} + +} // namespace + // ─── /props constants ─────────────────────────────────────────────────── // // SERVER_NAME / SERVER_VERSION mirror the Python server's identity strings @@ -593,6 +851,15 @@ static const json * find_tool_function(const json & tools, return nullptr; } +static bool tool_choice_requires_call(const json & tool_choice) { + if (tool_choice.is_string()) { + return tool_choice.get() == "required"; + } + return tool_choice.is_object() && tool_choice.contains("function") && + tool_choice["function"].is_object() && + !tool_choice["function"].value("name", "").empty(); +} + static std::string first_tool_parameter_name(const json & function_def) { const auto & params = function_def.value("parameters", json::object()); if (params.contains("required") && params["required"].is_array()) { @@ -750,6 +1017,20 @@ json build_props_body(const ServerConfig & config, // benchmarks to silently run at temp=0 (degenerate-decode collapse) // when the model card specifies temp=1.0/top_p=0.95/top_k=64. const auto & smp = config.sampler_defaults; + json tool_spec_lanes = json::array(); + for (const auto & lane : config.tool_speculation.policy.lanes()) { + tool_spec_lanes.push_back({ + {"resource_percentage", lane.resource_percentage}, + {"model_slowdown_ratio", lane.model_slowdown_ratio}, + {"decode_interference_qualified", + lane.decode_interference_qualified}, + {"accelerator_relation", lane.accelerator_relation}, + {"requires_static_model_routing", + lane.requires_static_model_routing}, + {"requires_unique_expert_ownership", + lane.requires_unique_expert_ownership}, + }); + } json body = { {"default_generation_settings", { {"n_ctx", config.max_ctx}, @@ -827,6 +1108,70 @@ json build_props_body(const ServerConfig & config, {"ddtree_budget", config.speculative_enabled ? json(config.ddtree_budget) : json(nullptr)}, }}, + {"tool_speculation", { + {"enabled", config.tool_speculation.enabled()}, + {"automatic_prediction_enabled", + config.tool_speculation.enabled() && + config.semantic_tool_predictor.enabled()}, + {"prediction_source", + config.semantic_tool_predictor.native_enabled() + ? json("native-qwen3") + : config.semantic_tool_predictor.http_enabled() + ? json(config.semantic_tool_predictor.model) + : json(nullptr)}, + {"prediction_confidence", + config.semantic_tool_predictor.enabled() + ? json(config.semantic_tool_predictor.execution_confidence) + : json(nullptr)}, + {"predictor_schedule", + config.semantic_tool_predictor.native_enabled() + ? json(native_tool_predictor_schedule_name( + config.semantic_tool_predictor.native_schedule)) + : config.semantic_tool_predictor.http_enabled() + ? json("overlap") : json(nullptr)}, + {"predictor_decode_isolated", + config.semantic_tool_predictor.native_runs_before_model()}, + {"execution_mode", config.tool_speculation.execution_mode()}, + {"profile_status", + config.tool_speculation.policy.empty() + ? json(nullptr) + : json(config.tool_speculation.policy.profile_status())}, + {"executor_contract", + config.tool_speculation.policy.executor_contract().empty() + ? json(nullptr) + : json(config.tool_speculation.policy.executor_contract())}, + {"protocol", "dflash.tool-speculation.v1"}, + {"requires_client_support", + !(config.tool_speculation.enabled() && + config.semantic_tool_predictor.enabled())}, + {"preserves_token_speculation", true}, + {"unqualified_lane_policy", "defer"}, + {"allowed_tools", config.tool_speculation.allowed_tools}, + {"max_model_slowdown_ratio", + config.tool_speculation.max_model_slowdown_ratio}, + {"model_routing_static", + config.tool_speculation.model_routing_static}, + {"model_expert_ownership_unique", + config.tool_speculation.model_expert_ownership_unique}, + {"compute_isolation", + config.tool_speculation.cpu_affinity_isolated + ? "disjoint_cpu_affinity" + : config.tool_speculation.hip_reserved_tool_compute_units > 0 + ? "disjoint_hip_cu_masks" : "none"}, + {"cpu_affinity_isolated", + config.tool_speculation.cpu_affinity_isolated}, + {"tool_cpu_affinity", + config.tool_speculation.cpu_affinity}, + {"model_cpu_affinity", + config.tool_speculation.model_cpu_affinity}, + {"hip_tool_device", + config.tool_speculation.hip_tool_device >= 0 + ? json(config.tool_speculation.hip_tool_device) + : json(nullptr)}, + {"hip_reserved_tool_compute_units", + config.tool_speculation.hip_reserved_tool_compute_units}, + {"profile_lanes", tool_spec_lanes}, + }}, {"sampling", { {"capabilities", { {"supports_temperature", true}, @@ -1264,6 +1609,24 @@ HttpServer::~HttpServer() { #endif } +bool HttpServer::init_semantic_tool_predictor(std::string & error) { + error.clear(); + if (!config_.semantic_tool_predictor.native_enabled()) return true; + native_semantic_predictor_ = NativeSemanticToolPredictor::create( + config_.semantic_tool_predictor, error); + if (native_semantic_predictor_) return true; + // An explicitly configured HTTP lane is a valid fail-open fallback. The + // target remains authoritative and still verifies every prediction. + if (config_.semantic_tool_predictor.http_enabled()) { + std::fprintf(stderr, + "[tool-hint] native predictor unavailable (%s); using HTTP fallback\n", + error.c_str()); + error.clear(); + return true; + } + return false; +} + void HttpServer::shutdown() { // Signal worker and accept loop to stop. stopping_.store(true); @@ -1664,6 +2027,27 @@ bool HttpServer::parse_common_request_fields( if (body.contains("tools")) req.tools = body["tools"]; // Tool choice constraint for hint generation. if (body.contains("tool_choice")) req.tool_choice = body["tool_choice"]; + if (body.contains("automatic_tool_speculation")) { + if (!body["automatic_tool_speculation"].is_boolean()) { + send_error(fd, 400, + "automatic_tool_speculation must be a boolean"); + return false; + } + req.automatic_tool_speculation_enabled = + body["automatic_tool_speculation"].get(); + } + + if (body.contains("tool_speculation")) { + ToolSpeculationPrediction prediction; + std::string prediction_error; + if (!parse_tool_speculation_prediction( + body["tool_speculation"], req.tools, prediction, + prediction_error)) { + send_error(fd, 400, prediction_error); + return false; + } + req.tool_speculation = std::move(prediction); + } if (body.contains("prefix_cache") && body["prefix_cache"].is_object()) { const auto & prefix_cache = body["prefix_cache"]; @@ -1928,7 +2312,8 @@ bool HttpServer::render_and_tokenize_request( } else { rendered = render_chat_template( chat_messages, chat_format_, /*add_generation_prompt=*/true, - req.thinking_enabled, tools_json); + req.thinking_enabled, tools_json, + tool_choice_requires_call(req.tool_choice)); } req.started_in_thinking = prompt_ends_in_open_think(rendered); @@ -1969,6 +2354,112 @@ void HttpServer::log_parsed_request(const ParsedRequest & req) const { req.stop_sequences.size(), req.model.c_str()); } +void HttpServer::launch_semantic_tool_prediction(ParsedRequest & req) const { + if (req.semantic_tool_prediction.valid() || + req.automatic_tool_speculation.valid()) { + return; + } + const bool automatic_execution = + req.automatic_tool_speculation_enabled && + !req.tool_speculation.has_value() && + config_.tool_speculation.enabled(); + if (!automatic_execution || + !config_.semantic_tool_predictor.enabled() || req.tools.empty() || + !req.raw_body.is_object()) { + return; + } + const SemanticToolPredictorConfig predictor = + config_.semantic_tool_predictor; + json semantic_request = req.raw_body; + // Endpoint parsers normalize Anthropic/Responses dialogue into req.messages. + // Supplying it here gives both transports one OpenAI-shaped semantic view. + semantic_request["messages"] = req.messages; + semantic_request["tools"] = req.tools; + if (!req.tool_choice.is_null()) { + semantic_request["tool_choice"] = req.tool_choice; + } + const json payload = build_semantic_tool_predictor_request( + semantic_request, + predictor.model.empty() ? "native-qwen3" : predictor.model, + predictor.max_tokens); + const json tools = req.tools; + const auto native = native_semantic_predictor_; + req.semantic_tool_prediction = std::async( + std::launch::async, + [predictor, payload, tools, native]() { + SemanticToolPrediction native_result; + if (native && native->active()) { + native_result = native->predict(payload, tools); + if (native_result.ok || !predictor.http_enabled()) { + return native_result; + } + } + if (predictor.http_enabled()) { + SemanticToolPrediction fallback = + request_semantic_tool_prediction(predictor, payload, tools); + if (!fallback.ok && !native_result.error.empty()) { + fallback.error = "native=" + native_result.error + + ";http=" + fallback.error; + } + return fallback; + } + if (native_result.error.empty()) { + native_result.error = "native_predictor_not_initialized"; + } + return native_result; + }).share(); + if (automatic_execution) { + const auto semantic_prediction = req.semantic_tool_prediction; + const ToolSpeculationConfig tool_config = config_.tool_speculation; + const double confidence = predictor.execution_confidence; + const std::string request_id = req.response_id; + req.automatic_tool_speculation = std::async( + std::launch::async, + [semantic_prediction, tool_config, confidence, request_id]() { + ParsedRequest::AutomaticToolSpeculationLaunch launch; + try { + const SemanticToolPrediction & semantic = + semantic_prediction.get(); + launch.predictor_wall_ms = semantic.wall_ms; + launch.prediction_source = semantic.source; + if (!semantic.ok) { + launch.predictor_error = semantic.error.empty() + ? "predictor_unavailable" : semantic.error; + return launch; + } + ToolSpeculationPrediction prediction; + std::string error; + const json arguments = json::parse( + semantic.call.arguments.dump()); + if (!build_tool_speculation_prediction( + semantic.call.name, arguments, confidence, + prediction, error)) { + launch.predictor_error = std::move(error); + return launch; + } + auto attempt = ToolSpeculationAttempt::create( + tool_config, prediction, request_id); + launch.attempt = std::shared_ptr( + attempt.release()); + launch.attempt->start(); + } catch (const std::exception & error) { + launch.predictor_error = + std::string("automatic_prediction_failed: ") + + error.what(); + } catch (...) { + launch.predictor_error = + "automatic_prediction_failed: unknown error"; + } + return launch; + }).share(); + } + std::fprintf(stderr, + "[tool-hint] launched predictor transport=%s%s tools=%zu execute=%s\n", + native ? "native-qwen3" : "http", + native && predictor.http_enabled() ? "+http-fallback" : "", + json_array_size(req.tools), automatic_execution ? "true" : "false"); +} + void HttpServer::enqueue_request_and_wait(SocketHandle fd, ParsedRequest req) { // Set socket non-blocking for send() stall detection during streaming. sock_set_nonblock(fd); @@ -2065,6 +2556,9 @@ bool HttpServer::route_request(SocketHandle fd, const HttpRequest & hr) { } if (!validate_request_context(fd, req)) return true; + if (!config_.semantic_tool_predictor.native_runs_before_model()) { + launch_semantic_tool_prediction(req); + } log_parsed_request(req); enqueue_request_and_wait(fd, std::move(req)); return true; @@ -2673,7 +3167,8 @@ void HttpServer::apply_flowkv_compression( } else { rendered = render_chat_template( chat_messages, chat_format_, /*add_generation_prompt=*/true, - req.thinking_enabled, tools_json); + req.thinking_enabled, tools_json, + tool_choice_requires_call(req.tool_choice)); } const int tokens_before = (int) prepared.tokens.size(); @@ -3460,6 +3955,16 @@ void HttpServer::prepare_generation_inputs( inputs.request.n_gen = inputs.generation_cap; inputs.request.sampler = req.sampler; inputs.request.do_sample = req.sampler.needs_logit_processing(); + // Tool prediction must never change the target decoder or trigger an + // autoregressive retry. DS4/DSpark remains authoritative on every arm. + const bool semantic_tool_request = + req.automatic_tool_speculation_enabled && + config_.tool_speculation.enabled() && + config_.semantic_tool_predictor.enabled() && + !req.tools.empty(); + inputs.request.allow_decode_mode_retry = + !req.tool_speculation.has_value() && + !semantic_tool_request; // Tokens are delivered through DaemonIO so all API formats share the // same disconnect and streaming state machine. inputs.request.stream = false; @@ -3647,7 +4152,7 @@ void HttpServer::worker_loop() { void HttpServer::process_job(ServerJob * job) { SocketHandle fd = job->fd; - const auto & req = job->req; + auto & req = job->req; auto started_at = std::chrono::steady_clock::now(); // Track live status for /status page. RAII guard ensures idle on all paths. @@ -3743,6 +4248,26 @@ void HttpServer::process_job(ServerJob * job) { } if (req.stream) start_job_stream(job); + if (config_.semantic_tool_predictor.native_runs_before_model()) { + const auto predictor_wait_started = std::chrono::steady_clock::now(); + launch_semantic_tool_prediction(req); + if (req.automatic_tool_speculation.valid()) { + req.automatic_tool_speculation.wait(); + } else if (req.semantic_tool_prediction.valid()) { + req.semantic_tool_prediction.wait(); + } + const double predictor_wait_ms = + std::chrono::duration( + std::chrono::steady_clock::now() - predictor_wait_started) + .count(); + if (req.automatic_tool_speculation.valid() || + req.semantic_tool_prediction.valid()) { + std::fprintf(stderr, + "[tool-hint] before-model barrier complete %.1f ms\n", + predictor_wait_ms); + } + } + PreparedPrompt prepared = prepare_prompt(req); if (prepared.error_status != 0) { fail_request(prepared.error_status, prepared.error); @@ -3753,6 +4278,14 @@ void HttpServer::process_job(ServerJob * job) { return; } + std::unique_ptr tool_speculation; + if (req.tool_speculation.has_value()) { + tool_speculation = ToolSpeculationAttempt::create( + config_.tool_speculation, *req.tool_speculation, + req.response_id); + tool_speculation->start(); + } + auto & effective_prompt = prepared.tokens; const bool pflash_compressed = prepared.compressed; @@ -3882,8 +4415,69 @@ void HttpServer::process_job(ServerJob * job) { if (job->client_disconnected.load(std::memory_order_acquire)) { client_disconnected = true; } + auto finish_tool_speculation = [&](bool cancel) -> std::optional { + if (tool_speculation) { + json metadata = cancel + ? tool_speculation->cancel("client_disconnected") + : tool_speculation->resolve(emitter.tool_calls()); + metadata["prediction_source"] = "client"; + return metadata; + } + if (!req.automatic_tool_speculation.valid()) return std::nullopt; + try { + const ParsedRequest::AutomaticToolSpeculationLaunch & launch = + req.automatic_tool_speculation.get(); + json metadata; + if (launch.attempt) { + metadata = cancel + ? launch.attempt->cancel("client_disconnected") + : launch.attempt->resolve(emitter.tool_calls()); + } else { + metadata = { + {"protocol", "dflash.tool-speculation.v1"}, + {"status", "deferred"}, + {"reason", "predictor_unavailable"}, + }; + if (!launch.predictor_error.empty()) { + metadata["detail"] = launch.predictor_error; + } + } + metadata["prediction_source"] = + launch.prediction_source.empty() + ? "predictor" : launch.prediction_source; + metadata["predictor_wall_ms"] = launch.predictor_wall_ms; + return metadata; + } catch (const std::exception & error) { + return json{ + {"protocol", "dflash.tool-speculation.v1"}, + {"status", "failed"}, + {"reason", "predictor_future_failure"}, + {"detail", error.what()}, + {"prediction_source", "predictor"}, + }; + } catch (...) { + return json{ + {"protocol", "dflash.tool-speculation.v1"}, + {"status", "failed"}, + {"reason", "predictor_future_failure"}, + {"detail", "unknown error"}, + {"prediction_source", "predictor"}, + }; + } + }; if (req.stream && !client_disconnected) { auto final_chunks = emitter.emit_finish(completion_tokens, &gen_timings); + if (auto metadata = finish_tool_speculation(false)) { + const std::string extension = render_tool_speculation_sse( + req.format, req.response_id, req.model, *metadata); + // Keep the standard terminal event last: [DONE], message_stop, + // or response.completed. Only opted-in clients see this extension. + if (final_chunks.empty()) { + final_chunks.push_back(extension); + } else { + final_chunks.insert(final_chunks.end() - 1, extension); + } + } for (const auto & chunk : final_chunks) { if (!send_job_bytes(job, chunk.data(), chunk.size())) { client_disconnected = true; @@ -3891,13 +4485,18 @@ void HttpServer::process_job(ServerJob * job) { } } } else if (!req.stream && !client_disconnected) { - const json response = build_non_streaming_response( + json response = build_non_streaming_response( req, result, n_gen_cap, gen_timings, tokenizer_, emitter); + if (auto metadata = finish_tool_speculation(false)) { + response["dflash_tool_speculation"] = std::move(*metadata); + } // Streaming uses non-blocking sends; restore blocking mode before // writing a complete JSON response on this shared socket path. sock_set_block(fd); send_response(fd, 200, "application/json", response.dump() + "\n"); + } else { + finish_tool_speculation(true); } if (client_disconnected) { diff --git a/server/src/server/http_server.h b/server/src/server/http_server.h index 90e02412e..05fcf564c 100644 --- a/server/src/server/http_server.h +++ b/server/src/server/http_server.h @@ -19,6 +19,9 @@ #include "tokenizer.h" #include "chat_template.h" #include "tool_memory.h" +#include "tool_speculation.h" +#include "semantic_tool_hint.h" +#include "native_semantic_tool_predictor.h" #include "prefix_cache.h" #include "disk_prefix_cache.h" #include "freeze_history.h" @@ -38,8 +41,10 @@ #include #include #include +#include #include #include +#include #include #include #if !defined(_WIN32) @@ -232,6 +237,15 @@ struct ServerConfig { // Routing data collection (--collect-routing ): write binary per-token // routing data (hidden states + expert selections) for predictor training. std::string collect_routing_path; + + // Lossless external-tool speculation. Off unless the operator configures + // an executor, an empirical interference profile, and an explicit + // read-only/idempotent tool allowlist. + ToolSpeculationConfig tool_speculation; + // Model-agnostic tool-call prediction. Native predictors run before the + // model by default so shared accelerator compute cannot slow decoding; + // remote predictors may overlap. DS4 verifies the exact canonical call. + SemanticToolPredictorConfig semantic_tool_predictor; }; namespace http_detail { @@ -268,6 +282,22 @@ struct ParsedRequest { json messages; // Original request body (for upstream proxy forwarding) json raw_body; + // Concrete invocation predicted by a caller or future semantic sidecar. + // The engine may execute it privately, but never exposes its result until + // the model emits the exact canonical invocation. + std::optional tool_speculation; + // Engine-side Qwen prediction may start a private external tool as soon + // as it is ready. The model's eventual canonical call remains authoritative. + struct AutomaticToolSpeculationLaunch { + std::shared_ptr attempt; + double predictor_wall_ms = 0.0; + std::string prediction_source; + std::string predictor_error; + }; + bool automatic_tool_speculation_enabled = true; + std::shared_future + automatic_tool_speculation; + std::shared_future semantic_tool_prediction; // Response ID std::string response_id; // Thinking/reasoning state @@ -335,6 +365,10 @@ class HttpServer { // Set the chat template format (detected from model arch). void set_chat_format(ChatFormat fmt) { chat_format_ = fmt; } + // Start the optional native Qwen predictor after target construction. + // HTTP-only predictor configurations need no persistent initialization. + bool init_semantic_tool_predictor(std::string & error); + // Start listening. Blocks until shutdown() is called. int run(); @@ -483,6 +517,7 @@ class HttpServer { ParsedRequest & req); bool validate_request_context(SocketHandle fd, const ParsedRequest & req); void log_parsed_request(const ParsedRequest & req) const; + void launch_semantic_tool_prediction(ParsedRequest & req) const; void enqueue_request_and_wait(SocketHandle fd, ParsedRequest req); // Send HTTP response helpers. @@ -510,6 +545,7 @@ class HttpServer { ServerConfig config_; ChatFormat chat_format_; PFlashDrafterIpcClient pflash_remote_; + std::shared_ptr native_semantic_predictor_; ToolMemory tool_memory_; PrefixCache prefix_cache_; DiskPrefixCache disk_cache_; diff --git a/server/src/server/native_semantic_tool_predictor.cpp b/server/src/server/native_semantic_tool_predictor.cpp new file mode 100644 index 000000000..9551d549c --- /dev/null +++ b/server/src/server/native_semantic_tool_predictor.cpp @@ -0,0 +1,85 @@ +#include "native_semantic_tool_predictor.h" + +#include +#include +#include + +namespace dflash::common { + +std::shared_ptr +NativeSemanticToolPredictor::create( + const SemanticToolPredictorConfig & config, + std::string & error) { + error.clear(); + if (!config.native_enabled()) { + error = "native_predictor_config_incomplete"; + return nullptr; + } + auto predictor = std::shared_ptr( + new NativeSemanticToolPredictor(config)); + if (!predictor->tokenizer_.load_from_gguf( + config.native_model_path.c_str())) { + error = "native_predictor_tokenizer_load_failed"; + return nullptr; + } + if (!predictor->ipc_.start( + config.native_ipc_bin, config.native_model_path, + config.native_gpu, config.native_max_ctx, + config.native_work_dir)) { + error = "native_predictor_ipc_start_failed"; + return nullptr; + } + return predictor; +} + +SemanticToolPrediction NativeSemanticToolPredictor::predict( + const json & predictor_request, + const json & request_tools, + std::string * generated_text) { + const auto started = std::chrono::steady_clock::now(); + SemanticToolPrediction prediction; + prediction.source = "native-qwen3"; + auto finish = [&]() { + prediction.wall_ms = std::chrono::duration( + std::chrono::steady_clock::now() - started).count(); + return prediction; + }; + + std::string prompt_error; + const std::string prompt = build_native_semantic_tool_predictor_prompt( + predictor_request, prompt_error); + if (prompt.empty()) { + prediction.error = std::move(prompt_error); + return finish(); + } + const std::vector prompt_ids = tokenizer_.encode(prompt); + if (prompt_ids.empty()) { + prediction.error = "native_predictor_prompt_tokenization_failed"; + return finish(); + } + if (prompt_ids.size() + static_cast(config_.max_tokens) > + static_cast(config_.native_max_ctx)) { + prediction.error = "native_predictor_context_overflow"; + return finish(); + } + + std::vector output_ids; + if (!ipc_.predict(prompt_ids, config_.max_tokens, + output_ids, prediction.error)) { + return finish(); + } + const std::string generated = tokenizer_.decode(output_ids); + if (generated_text) *generated_text = generated; + if (!parse_native_semantic_tool_prediction( + generated, request_tools, prediction.call, prediction.error)) { + return finish(); + } + if (!materialize_declared_tool_defaults( + request_tools, prediction.call, prediction.error)) { + return finish(); + } + prediction.ok = true; + return finish(); +} + +} // namespace dflash::common diff --git a/server/src/server/native_semantic_tool_predictor.h b/server/src/server/native_semantic_tool_predictor.h new file mode 100644 index 000000000..088db4344 --- /dev/null +++ b/server/src/server/native_semantic_tool_predictor.h @@ -0,0 +1,40 @@ +// Native Qwen semantic tool predictor built from the PFlash/Qwen runtime. + +#pragma once + +#include "semantic_tool_hint.h" + +#include "common/qwen3_tool_predictor_ipc.h" +#include "tokenizer.h" + +#include +#include + +namespace dflash::common { + +class NativeSemanticToolPredictor { +public: + static std::shared_ptr create( + const SemanticToolPredictorConfig & config, + std::string & error); + + NativeSemanticToolPredictor(const NativeSemanticToolPredictor &) = delete; + NativeSemanticToolPredictor & operator=( + const NativeSemanticToolPredictor &) = delete; + + SemanticToolPrediction predict(const json & predictor_request, + const json & request_tools, + std::string * generated_text = nullptr); + + bool active() const { return ipc_.active(); } + +private: + explicit NativeSemanticToolPredictor( + const SemanticToolPredictorConfig & config) : config_(config) {} + + SemanticToolPredictorConfig config_; + Tokenizer tokenizer_; + Qwen3ToolPredictorIpcClient ipc_; +}; + +} // namespace dflash::common diff --git a/server/src/server/semantic_tool_hint.cpp b/server/src/server/semantic_tool_hint.cpp new file mode 100644 index 000000000..cf7e7c3de --- /dev/null +++ b/server/src/server/semantic_tool_hint.cpp @@ -0,0 +1,500 @@ +#include "semantic_tool_hint.h" + +#include "tool_parser.h" + +#include +#include +#include + +namespace dflash::common { + +const char * native_tool_predictor_schedule_name( + NativeToolPredictorSchedule schedule) { + switch (schedule) { + case NativeToolPredictorSchedule::BeforeModel: + return "before-model"; + case NativeToolPredictorSchedule::Overlap: + return "overlap"; + } + return "unknown"; +} + +bool parse_native_tool_predictor_schedule( + const std::string & value, + NativeToolPredictorSchedule & out) { + if (value == "before-model") { + out = NativeToolPredictorSchedule::BeforeModel; + return true; + } + if (value == "overlap") { + out = NativeToolPredictorSchedule::Overlap; + return true; + } + return false; +} + +namespace { + +bool request_has_function(const json & tools, const std::string & name) { + if (!tools.is_array() || name.empty()) return false; + for (const auto & tool : tools) { + if (!tool.is_object()) continue; + if (tool.value("name", "") == name) return true; + const auto function = tool.find("function"); + if (function != tool.end() && function->is_object() && + function->value("name", "") == name) { + return true; + } + } + return false; +} + +std::string sole_request_function(const json & tools) { + if (!tools.is_array()) return {}; + std::string sole; + for (const auto & tool : tools) { + if (!tool.is_object()) continue; + std::string name = tool.value("name", ""); + const auto function = tool.find("function"); + if (name.empty() && function != tool.end() && function->is_object()) { + name = function->value("name", ""); + } + if (name.empty()) continue; + if (!sole.empty() && sole != name) return {}; + sole = std::move(name); + } + return sole; +} + +bool parse_arguments(const json & value, ordered_json & out) { + try { + if (value.is_string()) { + out = ordered_json::parse(value.get()); + } else if (value.is_object()) { + out = ordered_json::parse(value.dump()); + } else { + return false; + } + } catch (...) { + return false; + } + return out.is_object(); +} + +bool parse_call_object(const json & value, SemanticToolCall & out) { + if (!value.is_object()) return false; + const std::string name = value.value( + "name", value.value("function", std::string{})); + if (name.empty()) return false; + + const json * arguments = nullptr; + for (const char * key : {"arguments", "parameters", "params"}) { + const auto it = value.find(key); + if (it != value.end()) { + arguments = &*it; + break; + } + } + ordered_json parsed; + if (!arguments || !parse_arguments(*arguments, parsed)) return false; + out.name = name; + out.arguments = std::move(parsed); + return true; +} + +bool parse_content_call(const std::string & content, SemanticToolCall & out) { + for (size_t offset = 0; offset < content.size(); ++offset) { + if (content[offset] != '{') continue; + try { + const auto value = json::parse( + content.begin() + static_cast(offset), + content.end(), nullptr, false); + if (!value.is_discarded() && parse_call_object(value, out)) { + return true; + } + } catch (...) { + // Continue scanning for a later strict object. + } + } + return false; +} + +std::string trim_copy(std::string value) { + const auto is_space = [](unsigned char ch) { return std::isspace(ch); }; + value.erase(value.begin(), std::find_if_not( + value.begin(), value.end(), is_space)); + value.erase(std::find_if_not(value.rbegin(), value.rend(), is_space).base(), + value.end()); + return value; +} + +bool parse_qwen_tagged_call_repair( + const std::string & generated_text, + SemanticToolCall & out) { + const std::string open = ""; + const size_t open_pos = generated_text.find(open); + if (open_pos == std::string::npos) return false; + const size_t content_pos = open_pos + open.size(); + size_t end_pos = generated_text.find("", content_pos); + if (end_pos == std::string::npos) { + end_pos = generated_text.find("<|im_end|>", content_pos); + } + if (end_pos == std::string::npos) end_pos = generated_text.size(); + + std::string payload = trim_copy( + generated_text.substr(content_pos, end_pos - content_pos)); + if (payload.empty()) return false; + + // Qwen3-0.6B Q8 occasionally drops only the outer opening brace and + // emits a stray quote before the matching closing brace, while keeping + // the name and argument object strict JSON. Repair only that narrow + // envelope error; all semantic fields still pass the normal schema gate. + if (payload.front() != '{') payload.insert(payload.begin(), '{'); + if (payload.size() >= 3 && payload.back() == '}') { + size_t quote = payload.size() - 2; + while (quote > 0 && std::isspace( + static_cast(payload[quote]))) { + --quote; + } + if (payload[quote] == '"') payload.erase(quote, 1); + } + const json value = json::parse(payload, nullptr, false); + return !value.is_discarded() && parse_call_object(value, out); +} + +bool parse_qwen_bare_single_tool_arguments( + const std::string & generated_text, + const json & request_tools, + SemanticToolCall & out) { + const std::string name = sole_request_function(request_tools); + if (name.empty()) return false; + + size_t begin = generated_text.find(""); + begin = begin == std::string::npos + ? 0 : begin + std::string("").size(); + begin = generated_text.find('{', begin); + if (begin == std::string::npos) return false; + size_t end = generated_text.rfind('}'); + if (end == std::string::npos || end < begin) return false; + + const json arguments = json::parse( + generated_text.begin() + static_cast(begin), + generated_text.begin() + static_cast(end + 1), + nullptr, false); + if (arguments.is_discarded() || !arguments.is_object()) return false; + out.name = name; + out.arguments = ordered_json::parse(arguments.dump()); + return true; +} + +std::string semantic_message_content(const json & message) { + const auto content = message.find("content"); + if (content == message.end() || content->is_null()) return {}; + if (content->is_string()) return content->get(); + if (!content->is_array()) return content->dump(); + + std::string text; + for (const auto & part : *content) { + if (part.is_string()) { + text += part.get(); + continue; + } + if (!part.is_object()) continue; + const std::string type = part.value("type", ""); + if (type == "text" || type == "input_text" || + type == "output_text") { + text += part.value("text", ""); + } + } + return text; +} + +std::string forced_tool_name(const json & choice) { + if (!choice.is_object()) return {}; + const auto function = choice.find("function"); + if (function != choice.end() && function->is_object()) { + return function->value("name", ""); + } + return choice.value("name", ""); +} + +} // namespace + +bool parse_semantic_tool_prediction( + const json & response, + const json & request_tools, + SemanticToolCall & out, + std::string & error) { + error.clear(); + const auto choices = response.find("choices"); + if (choices == response.end() || !choices->is_array() || + choices->size() != 1 || !(*choices)[0].is_object()) { + error = "predictor_response_missing_single_choice"; + return false; + } + const auto message = (*choices)[0].find("message"); + if (message == (*choices)[0].end() || !message->is_object()) { + error = "predictor_response_missing_message"; + return false; + } + + bool parsed = false; + const auto calls = message->find("tool_calls"); + if (calls != message->end() && calls->is_array() && calls->size() == 1) { + const auto function = (*calls)[0].find("function"); + if (function != (*calls)[0].end()) { + parsed = parse_call_object(*function, out); + } + } + if (!parsed) { + const auto content = message->find("content"); + if (content != message->end() && content->is_string()) { + parsed = parse_content_call(content->get(), out); + } + } + if (!parsed) { + error = "predictor_response_has_no_valid_call"; + return false; + } + if (!request_has_function(request_tools, out.name)) { + error = "predictor_selected_unknown_function"; + return false; + } + return true; +} + +bool materialize_declared_tool_defaults( + const json & request_tools, + SemanticToolCall & call, + std::string & error) { + error.clear(); + if (!call.arguments.is_object()) { + error = "predictor_arguments_not_object"; + return false; + } + if (!request_tools.is_array()) { + error = "predictor_tools_not_array"; + return false; + } + + const json * function = nullptr; + for (const auto & tool : request_tools) { + if (!tool.is_object()) continue; + const json & candidate = tool.contains("function") && + tool["function"].is_object() + ? tool["function"] : tool; + if (candidate.value("name", "") == call.name) { + function = &candidate; + break; + } + } + if (!function) { + error = "predictor_selected_unknown_function"; + return false; + } + + const json * parameters = nullptr; + for (const char * key : {"parameters", "input_schema"}) { + const auto found = function->find(key); + if (found != function->end() && found->is_object()) { + parameters = &*found; + break; + } + } + if (!parameters) return true; + const auto properties = parameters->find("properties"); + if (properties == parameters->end() || !properties->is_object()) { + return true; + } + for (const auto & property : properties->items()) { + if (call.arguments.contains(property.key()) || + !property.value().is_object() || + !property.value().contains("default")) { + continue; + } + call.arguments[property.key()] = property.value()["default"]; + } + return true; +} + +json build_semantic_tool_predictor_request( + const json & target_request, + const std::string & sidecar_model, + int max_tokens) { + json request = { + {"model", sidecar_model}, + {"stream", false}, + {"temperature", 0}, + {"max_tokens", max_tokens}, + }; + for (const char * key : {"messages", "tools", "tool_choice"}) { + const auto it = target_request.find(key); + if (it != target_request.end()) request[key] = *it; + } + if (!request.contains("tool_choice")) request["tool_choice"] = "auto"; + return request; +} + +std::string build_native_semantic_tool_predictor_prompt( + const json & predictor_request, + std::string & error) { + error.clear(); + const auto messages = predictor_request.find("messages"); + if (messages == predictor_request.end() || !messages->is_array() || + messages->empty()) { + error = "native_predictor_missing_messages"; + return {}; + } + const json tools = predictor_request.value("tools", json::array()); + if (!tools.is_array() || tools.empty()) { + error = "native_predictor_missing_tools"; + return {}; + } + + struct PredictorMessage { + std::string role; + std::string content; + json tool_calls; + }; + std::vector chat; + chat.reserve(messages->size()); + for (const auto & message : *messages) { + if (!message.is_object()) continue; + std::string role = message.value("role", "user"); + if (role == "developer") role = "system"; + chat.push_back({ + std::move(role), semantic_message_content(message), + message.value("tool_calls", json::array()), + }); + } + if (chat.empty()) { + error = "native_predictor_empty_messages"; + return {}; + } + + std::string constraint; + const json choice = predictor_request.value("tool_choice", json("auto")); + if (choice.is_string() && choice.get() == "required") { + constraint = "You must call exactly one available function."; + } else if (const std::string name = forced_tool_name(choice); + !name.empty()) { + constraint = "You must call the function " + name + "."; + } + // Render the exact tokenizer.chat_template contract embedded in the + // Qwen3-0.6B GGUF. PFlash's generic Qwen3.5 renderer uses parameter XML, + // while this model was trained to emit one JSON object inside + // ; using the wrong contract destroys multi-tool accuracy. + size_t begin = 0; + std::string system_content; + if (!chat.empty() && chat.front().role == "system") { + system_content = chat.front().content; + begin = 1; + } + if (!constraint.empty()) { + if (!system_content.empty()) system_content += "\n\n"; + system_content += constraint; + } + + std::string rendered = "<|im_start|>system\n"; + if (!system_content.empty()) { + rendered += system_content; + rendered += "\n\n"; + } + rendered += + "# Tools\n\n" + "You may call one or more functions to assist with the user query.\n\n" + "You are provided with function signatures within XML tags:\n" + ""; + for (const auto & tool : tools) rendered += tool.dump(); + rendered += + "\n\n\n" + "For each function call, return a json object with function name and " + "arguments within XML tags:\n" + "\n" + "{\"name\": , \"arguments\": }\n" + "<|im_end|>\n"; + + bool in_tool_response = false; + for (size_t index = begin; index < chat.size(); ++index) { + const auto & message = chat[index]; + if (message.role == "tool") { + if (!in_tool_response) { + rendered += "<|im_start|>user"; + in_tool_response = true; + } + rendered += "\n\n" + message.content + + "\n"; + const bool next_is_tool = index + 1 < chat.size() && + chat[index + 1].role == "tool"; + if (!next_is_tool) { + rendered += "<|im_end|>\n"; + in_tool_response = false; + } + continue; + } + + rendered += "<|im_start|>" + message.role + "\n" + message.content; + if (message.role == "assistant" && message.tool_calls.is_array()) { + for (const auto & raw_call : message.tool_calls) { + if (!raw_call.is_object()) continue; + const json & call = raw_call.contains("function") && + raw_call["function"].is_object() + ? raw_call["function"] : raw_call; + const std::string name = call.value("name", ""); + if (name.empty() || !call.contains("arguments")) continue; + if (!message.content.empty()) rendered += "\n"; + rendered += "\n{\"name\": \"" + name + + "\", \"arguments\": "; + rendered += call["arguments"].is_string() + ? call["arguments"].get() + : call["arguments"].dump(); + rendered += "}\n"; + } + } + rendered += "<|im_end|>\n"; + } + rendered += "<|im_start|>assistant\n\n\n\n\n"; + return rendered; +} + +bool parse_native_semantic_tool_prediction( + const std::string & generated_text, + const json & request_tools, + SemanticToolCall & out, + std::string & error) { + error.clear(); + const ToolParseResult parsed = parse_tool_calls( + generated_text, request_tools); + if (parsed.tool_calls.size() == 1) { + try { + ordered_json arguments = ordered_json::parse( + parsed.tool_calls.front().arguments); + if (!arguments.is_object()) { + error = "native_predictor_arguments_not_object"; + return false; + } + out.name = parsed.tool_calls.front().name; + out.arguments = std::move(arguments); + } catch (...) { + error = "native_predictor_arguments_invalid_json"; + return false; + } + } else if (parsed.tool_calls.empty()) { + if (!parse_qwen_tagged_call_repair(generated_text, out) && + !parse_qwen_bare_single_tool_arguments( + generated_text, request_tools, out)) { + error = "native_predictor_response_has_no_valid_call"; + return false; + } + } else { + error = "native_predictor_response_has_multiple_calls"; + return false; + } + if (!request_has_function(request_tools, out.name)) { + error = "predictor_selected_unknown_function"; + return false; + } + return true; +} + +} // namespace dflash::common diff --git a/server/src/server/semantic_tool_hint.h b/server/src/server/semantic_tool_hint.h new file mode 100644 index 000000000..69c6f962c --- /dev/null +++ b/server/src/server/semantic_tool_hint.h @@ -0,0 +1,105 @@ +// Model-agnostic tool-call predictions shared by HTTP and native predictors. + +#pragma once + +#include + +#include + +namespace dflash::common { + +using json = nlohmann::json; +using ordered_json = nlohmann::ordered_json; + +enum class NativeToolPredictorSchedule { + BeforeModel, + Overlap, +}; + +const char * native_tool_predictor_schedule_name( + NativeToolPredictorSchedule schedule); + +bool parse_native_tool_predictor_schedule( + const std::string & value, + NativeToolPredictorSchedule & out); + +struct SemanticToolPredictorConfig { + std::string url; + std::string model; + std::string native_model_path; + std::string native_ipc_bin; + std::string native_work_dir; + int native_gpu = 0; + int native_max_ctx = 4096; + int timeout_ms = 2000; + int max_tokens = 96; + NativeToolPredictorSchedule native_schedule = + NativeToolPredictorSchedule::BeforeModel; + // Conservative prior used by the measured tool-execution admission + // policy. The base predictor currently emits no calibrated probability. + double execution_confidence = 0.75; + + bool http_enabled() const { return !url.empty() && !model.empty(); } + bool native_enabled() const { + return !native_model_path.empty() && !native_ipc_bin.empty(); + } + bool enabled() const { return native_enabled() || http_enabled(); } + bool native_runs_before_model() const { + return native_enabled() && + native_schedule == NativeToolPredictorSchedule::BeforeModel; + } +}; + +struct SemanticToolCall { + std::string name; + ordered_json arguments = ordered_json::object(); +}; + +struct SemanticToolPrediction { + bool ok = false; + std::string error; + // Actual predictor used for this result. Native and HTTP fallback paths + // share one execution gate, so response metadata must not guess. + std::string source; + SemanticToolCall call; + double wall_ms = 0.0; +}; + +// Parse one OpenAI-compatible sidecar response and reject calls whose +// function name is absent from the request schema. Arguments remain decoded +// JSON values; sidecar token IDs are never accepted by the target. +bool parse_semantic_tool_prediction( + const json & response, + const json & request_tools, + SemanticToolCall & out, + std::string & error); + +// Materialize top-level defaults declared by the selected function before a +// prediction is executed. This turns an omitted optional default into the +// exact explicit invocation the target may emit; the normal exact-match gate +// still rejects the result if the authoritative call differs. +bool materialize_declared_tool_defaults( + const json & request_tools, + SemanticToolCall & call, + std::string & error); + +// Build the small OpenAI-compatible request sent to the predictor. Only +// dialogue/tool semantics are forwarded; target-only extensions are omitted. +json build_semantic_tool_predictor_request( + const json & target_request, + const std::string & sidecar_model, + int max_tokens); + +// Native predictor bridge. The prompt uses the Qwen tool template and the +// decoded response is parsed semantically before any target token IDs exist. +std::string build_native_semantic_tool_predictor_prompt( + const json & predictor_request, + std::string & error); + +bool parse_native_semantic_tool_prediction( + const std::string & generated_text, + const json & request_tools, + SemanticToolCall & out, + std::string & error); + +} // namespace dflash::common diff --git a/server/src/server/server_main.cpp b/server/src/server/server_main.cpp index 7373aace4..02b915f0f 100644 --- a/server/src/server/server_main.cpp +++ b/server/src/server/server_main.cpp @@ -12,6 +12,9 @@ // [--max-tokens 4096] [--target-device auto:0] #include "http_server.h" +#if defined(DFLASH27B_BACKEND_HIP) +#include "tool_speculation_hip_probe.h" +#endif #include "chat_template.h" #include "model_card.h" #include "common/backend_factory.h" @@ -27,6 +30,7 @@ #include "kvflash_pager.h" #include +#include #include #include #include @@ -37,6 +41,10 @@ #include #include +#if !defined(_WIN32) +#include +#endif + using namespace dflash::common; // Global server pointer for signal handling. @@ -66,6 +74,11 @@ static bool parse_double_list(const char * value, std::vector & out) { return !out.empty(); } +static bool environment_flag_enabled(const char * name) { + const char * value = std::getenv(name); + return value && *value && std::strcmp(value, "0") != 0; +} + static void print_usage(const char * prog) { std::fprintf(stderr, "Usage: %s [options]\n" @@ -175,6 +188,55 @@ static void print_usage(const char * prog) { " Drafter lifetime policy (default: auto)\n" " --lazy-draft Legacy alias for --draft-residency=request-scoped\n" "\n" + "Tool-call prediction (lossless exact verification):\n" + " --tool-hint-sidecar-url \n" + " OpenAI-compatible chat-completions endpoint.\n" + " --tool-hint-sidecar-model \n" + " Predictor model served by that endpoint.\n" + " --tool-hint-native-model \n" + " Qwen3-0.6B GGUF for the native PFlash runtime.\n" + " --tool-hint-native-ipc-bin \n" + " Matching backend_ipc_daemon executable.\n" + " --tool-hint-native-gpu Predictor GPU (default: 0).\n" + " --tool-hint-native-max-ctx \n" + " Predictor context capacity (default: 4096).\n" + " --tool-hint-native-schedule \n" + " before-model (default) runs Qwen before DS4\n" + " so shared-GPU decoding cannot be slowed;\n" + " overlap is an experimental throughput mode.\n" + " --tool-hint-native-work-dir \n" + " Optional private IPC scratch directory.\n" + " --tool-hint-sidecar-timeout-ms \n" + " Hard sidecar deadline (default: 2000).\n" + " --tool-hint-sidecar-max-tokens \n" + " Predictor completion cap (default: 96).\n" + " --tool-hint-execution-confidence

\n" + " Calibrated 0..1 prior for automatic external\n" + " tool admission (default: 0.75).\n" + " The target verifies every hint.\n" + "\n" + "Speculative external tools (opt-in, POSIX):\n" + " --tool-spec-executor Trusted executor adapter. Receives one\n" + " dflash.tool-speculation.v1 JSON request\n" + " on stdin; no shell is used.\n" +#if defined(DFLASH27B_BACKEND_HIP) + " --tool-spec-hip-sgemm-probe \n" + " Benchmark-only trusted in-process HIP\n" + " executor with a per-lane CU-masked stream.\n" +#endif + " --tool-spec-profile Measured resource-lane frontier JSON.\n" + " --tool-spec-allow Allow one read-only/idempotent tool; repeatable.\n" + " --tool-spec-cpu-affinity \n" + " Pin child tools to Linux CPUs/ranges, e.g.\n" + " 14-15,30-31. The model process affinity\n" + " must exclude every listed CPU.\n" + " --tool-spec-timeout-ms Executor result timeout (default: 60000).\n" + " --tool-spec-max-model-slowdown \n" + " Reject lanes slower than this inference\n" + " ratio (default: 1.20).\n" + " Every admitted lane must pass exact-output\n" + " decode-interference qualification.\n" + "\n" "PFlash upstream proxy (forward compressed prompt to a backend):\n" " --prefill-upstream-base OpenAI-compatible upstream. Compressed\n" " requests POST the raw prompt to\n" @@ -225,6 +287,9 @@ int main(int argc, char ** argv) { // Parse arguments. BackendArgs bargs; ServerConfig sconfig; + int tool_spec_hip_probe_device = -1; + int tool_spec_hip_probe_matrix = 0; + int tool_spec_hip_probe_total_cus = 0; bargs.model_path = argv[1]; bool spark_autotune = false; // --spark: self-tuning hot/cold MoE residency int spark_slots = -1; // --spark-slots: explicit cache slots/layer (-1=auto) @@ -236,6 +301,7 @@ int main(int argc, char ** argv) { bool fast_rollback_forced_off = false; bool target_split_fast_rollback_cli = false; bool adaptive_experts_set = false; // --adaptive-experts (MoE architectures only) + bool native_tool_predictor_schedule_set = false; // Track which thinking-budget tunables the operator set via CLI. // Those values win over the model card (spec §3.1: "Explicit CLI @@ -552,6 +618,142 @@ int main(int argc, char ** argv) { } else if (std::strcmp(argv[i], "--lazy-draft") == 0) { sconfig.lazy_draft = true; sconfig.draft_residency = DraftResidencyPolicy::RequestScoped; + } else if (std::strcmp(argv[i], "--tool-hint-sidecar-url") == 0 && + i + 1 < argc) { + sconfig.semantic_tool_predictor.url = argv[++i]; + } else if (std::strcmp(argv[i], "--tool-hint-sidecar-model") == 0 && + i + 1 < argc) { + sconfig.semantic_tool_predictor.model = argv[++i]; + } else if (std::strcmp(argv[i], "--tool-hint-native-model") == 0 && + i + 1 < argc) { + sconfig.semantic_tool_predictor.native_model_path = argv[++i]; + } else if (std::strcmp(argv[i], "--tool-hint-native-ipc-bin") == 0 && + i + 1 < argc) { + sconfig.semantic_tool_predictor.native_ipc_bin = argv[++i]; + } else if (std::strcmp(argv[i], "--tool-hint-native-work-dir") == 0 && + i + 1 < argc) { + sconfig.semantic_tool_predictor.native_work_dir = argv[++i]; + } else if (std::strcmp(argv[i], "--tool-hint-native-gpu") == 0 && + i + 1 < argc) { + sconfig.semantic_tool_predictor.native_gpu = std::atoi(argv[++i]); + if (sconfig.semantic_tool_predictor.native_gpu < 0) { + std::fprintf(stderr, + "[server] --tool-hint-native-gpu must be non-negative\n"); + return 2; + } + } else if (std::strcmp(argv[i], "--tool-hint-native-max-ctx") == 0 && + i + 1 < argc) { + sconfig.semantic_tool_predictor.native_max_ctx = std::atoi(argv[++i]); + if (sconfig.semantic_tool_predictor.native_max_ctx <= 0) { + std::fprintf(stderr, + "[server] --tool-hint-native-max-ctx must be positive\n"); + return 2; + } + } else if (std::strcmp(argv[i], "--tool-hint-native-schedule") == 0 && + i + 1 < argc) { + native_tool_predictor_schedule_set = true; + if (!parse_native_tool_predictor_schedule( + argv[++i], + sconfig.semantic_tool_predictor.native_schedule)) { + std::fprintf(stderr, + "[server] --tool-hint-native-schedule must be " + "before-model or overlap\n"); + return 2; + } + } else if (std::strcmp(argv[i], "--tool-hint-sidecar-timeout-ms") == 0 && + i + 1 < argc) { + sconfig.semantic_tool_predictor.timeout_ms = std::atoi(argv[++i]); + if (sconfig.semantic_tool_predictor.timeout_ms <= 0) { + std::fprintf(stderr, + "[server] --tool-hint-sidecar-timeout-ms must be positive\n"); + return 2; + } + } else if (std::strcmp(argv[i], "--tool-hint-sidecar-max-tokens") == 0 && + i + 1 < argc) { + sconfig.semantic_tool_predictor.max_tokens = std::atoi(argv[++i]); + if (sconfig.semantic_tool_predictor.max_tokens <= 0) { + std::fprintf(stderr, + "[server] --tool-hint-sidecar-max-tokens must be positive\n"); + return 2; + } + } else if (std::strcmp( + argv[i], "--tool-hint-execution-confidence") == 0 && + i + 1 < argc) { + sconfig.semantic_tool_predictor.execution_confidence = + std::atof(argv[++i]); + if (!std::isfinite( + sconfig.semantic_tool_predictor.execution_confidence) || + sconfig.semantic_tool_predictor.execution_confidence < 0.0 || + sconfig.semantic_tool_predictor.execution_confidence > 1.0) { + std::fprintf(stderr, + "[server] --tool-hint-execution-confidence must be between 0 and 1\n"); + return 2; + } + } else if (std::strcmp(argv[i], "--tool-spec-executor") == 0 && + i + 1 < argc) { + sconfig.tool_speculation.executor_path = argv[++i]; +#if defined(DFLASH27B_BACKEND_HIP) + } else if (std::strcmp(argv[i], "--tool-spec-hip-sgemm-probe") == 0 && + i + 1 < argc) { + const char * value = argv[++i]; + char * separator = nullptr; + const long device = std::strtol(value, &separator, 10); + if (separator == value || !separator || *separator != ':') { + std::fprintf(stderr, + "[server] --tool-spec-hip-sgemm-probe expects DEVICE:MATRIX\n"); + return 2; + } + char * end = nullptr; + const long matrix = std::strtol(separator + 1, &end, 10); + if (!end || *end != '\0' || device < 0 || + matrix <= 0 || matrix > 8192) { + std::fprintf(stderr, + "[server] invalid --tool-spec-hip-sgemm-probe DEVICE:MATRIX\n"); + return 2; + } + tool_spec_hip_probe_device = static_cast(device); + tool_spec_hip_probe_matrix = static_cast(matrix); +#endif + } else if (std::strcmp(argv[i], "--tool-spec-profile") == 0 && + i + 1 < argc) { + sconfig.tool_speculation.profile_path = argv[++i]; + } else if (std::strcmp(argv[i], "--tool-spec-allow") == 0 && + i + 1 < argc) { + const std::string name = argv[++i]; + if (name.empty()) { + std::fprintf(stderr, "[server] --tool-spec-allow needs a name\n"); + return 2; + } + sconfig.tool_speculation.allowed_tools.push_back(name); + } else if (std::strcmp(argv[i], "--tool-spec-cpu-affinity") == 0 && + i + 1 < argc) { + std::string affinity_error; + if (!parse_tool_speculation_cpu_affinity( + argv[++i], sconfig.tool_speculation.cpu_affinity, + affinity_error)) { + std::fprintf(stderr, "[server] %s\n", affinity_error.c_str()); + return 2; + } + } else if (std::strcmp(argv[i], "--tool-spec-timeout-ms") == 0 && + i + 1 < argc) { + sconfig.tool_speculation.timeout_ms = std::atoi(argv[++i]); + if (sconfig.tool_speculation.timeout_ms <= 0) { + std::fprintf(stderr, + "[server] --tool-spec-timeout-ms must be positive\n"); + return 2; + } + } else if (std::strcmp( + argv[i], "--tool-spec-max-model-slowdown") == 0 && + i + 1 < argc) { + sconfig.tool_speculation.max_model_slowdown_ratio = + std::atof(argv[++i]); + if (!std::isfinite( + sconfig.tool_speculation.max_model_slowdown_ratio) || + sconfig.tool_speculation.max_model_slowdown_ratio < 1.0) { + std::fprintf(stderr, + "[server] --tool-spec-max-model-slowdown must be >= 1\n"); + return 2; + } } else if (std::strcmp(argv[i], "--chat-template-file") == 0 && i + 1 < argc) { const char * path = argv[++i]; std::FILE * f = std::fopen(path, "rb"); @@ -616,6 +818,192 @@ int main(int argc, char ** argv) { return 2; } } + + if (tool_spec_hip_probe_device >= 0) { +#if defined(DFLASH27B_BACKEND_HIP) + std::string executor_error; + sconfig.tool_speculation.in_process_executor = + create_hip_sgemm_tool_speculation_executor( + tool_spec_hip_probe_device, + tool_spec_hip_probe_matrix, + tool_spec_hip_probe_total_cus, + executor_error); + if (!sconfig.tool_speculation.in_process_executor) { + std::fprintf(stderr, "[server] %s\n", executor_error.c_str()); + return 2; + } +#endif + } + const bool semantic_http_predictor_requested = + !sconfig.semantic_tool_predictor.url.empty() || + !sconfig.semantic_tool_predictor.model.empty(); + if (semantic_http_predictor_requested && + !sconfig.semantic_tool_predictor.http_enabled()) { + std::fprintf(stderr, + "[server] HTTP semantic tool hints require both " + "--tool-hint-sidecar-url and --tool-hint-sidecar-model\n"); + return 2; + } + const bool semantic_native_predictor_requested = + !sconfig.semantic_tool_predictor.native_model_path.empty() || + !sconfig.semantic_tool_predictor.native_ipc_bin.empty() || + !sconfig.semantic_tool_predictor.native_work_dir.empty() || + native_tool_predictor_schedule_set; + if (semantic_native_predictor_requested && + !sconfig.semantic_tool_predictor.native_enabled()) { + std::fprintf(stderr, + "[server] native semantic tool hints require both " + "--tool-hint-native-model and --tool-hint-native-ipc-bin\n"); + return 2; + } + const bool tool_speculation_requested = + !sconfig.tool_speculation.executor_path.empty() || + static_cast(sconfig.tool_speculation.in_process_executor) || + !sconfig.tool_speculation.profile_path.empty() || + !sconfig.tool_speculation.allowed_tools.empty() || + !sconfig.tool_speculation.cpu_affinity.empty(); + if (tool_speculation_requested) { + sconfig.tool_speculation.model_routing_static = + !environment_flag_enabled( + "DFLASH_MOE_TP_DYNAMIC_ROUTE_BALANCE") && + !environment_flag_enabled( + "DFLASH_DS4_TP_DYNAMIC_ROUTE_BALANCE"); + sconfig.tool_speculation.model_expert_ownership_unique = + !environment_flag_enabled("DFLASH_MOE_DUPLICATE_HOT_ON_COLD"); + const bool has_child_executor = + !sconfig.tool_speculation.executor_path.empty(); + const bool has_in_process_executor = + static_cast(sconfig.tool_speculation.in_process_executor); + if (has_child_executor == has_in_process_executor || + sconfig.tool_speculation.profile_path.empty() || + sconfig.tool_speculation.allowed_tools.empty()) { + std::fprintf(stderr, + "[server] tool speculation requires exactly one executor, " + "--tool-spec-profile, and at least one --tool-spec-allow\n"); + return 2; + } +#if !defined(_WIN32) + if (has_child_executor && + ::access(sconfig.tool_speculation.executor_path.c_str(), X_OK) != 0) { + std::fprintf(stderr, + "[server] tool speculation executor is not executable: %s\n", + sconfig.tool_speculation.executor_path.c_str()); + return 2; + } +#endif + std::sort(sconfig.tool_speculation.allowed_tools.begin(), + sconfig.tool_speculation.allowed_tools.end()); + sconfig.tool_speculation.allowed_tools.erase( + std::unique(sconfig.tool_speculation.allowed_tools.begin(), + sconfig.tool_speculation.allowed_tools.end()), + sconfig.tool_speculation.allowed_tools.end()); + std::string profile_error; + if (!sconfig.tool_speculation.policy.load_file( + sconfig.tool_speculation.profile_path, profile_error)) { + std::fprintf(stderr, "[server] %s\n", profile_error.c_str()); + return 2; + } + std::string cpu_affinity_error; + if (!qualify_tool_speculation_cpu_affinity( + sconfig.tool_speculation, cpu_affinity_error)) { + std::fprintf(stderr, "[server] %s\n", cpu_affinity_error.c_str()); + return 2; + } + if (sconfig.tool_speculation.cpu_affinity_isolated) { + std::fprintf(stderr, + "[server] disjoint CPU tool lane: %zu model logical CPUs, " + "%zu reserved tool logical CPUs\n", + sconfig.tool_speculation.model_cpu_affinity.size(), + sconfig.tool_speculation.cpu_affinity.size()); + } + const std::string & executor_contract = + sconfig.tool_speculation.policy.executor_contract(); + if (!executor_contract.empty() && + executor_contract != + sconfig.tool_speculation.execution_mode()) { + std::fprintf(stderr, + "[server] tool profile requires executor '%s', got '%s'\n", + executor_contract.c_str(), + sconfig.tool_speculation.execution_mode()); + return 2; + } + if (sconfig.tool_speculation.policy.benchmark_only() && + !has_in_process_executor) { + std::fprintf(stderr, + "[server] provisional_benchmark_only tool profiles cannot " + "enable an external production executor\n"); + return 2; + } + if (has_in_process_executor) { + int max_same_gpu_percentage = 0; + for (const auto & lane : + sconfig.tool_speculation.policy.lanes()) { + if (lane.decode_interference_qualified && + lane.accelerator_relation == "same_physical_gpu") { + max_same_gpu_percentage = std::max( + max_same_gpu_percentage, + lane.resource_percentage); + } + } + if (max_same_gpu_percentage > 0) { + const int reserved_cus = + (tool_spec_hip_probe_total_cus * + max_same_gpu_percentage + + 99) / + 100; + if (reserved_cus <= 0 || + reserved_cus >= tool_spec_hip_probe_total_cus) { + std::fprintf(stderr, + "[server] same-GPU HIP tool lane would reserve %d " + "of %d CUs; at least one model CU is required\n", + reserved_cus, tool_spec_hip_probe_total_cus); + return 2; + } + const std::string isolation = + std::to_string(tool_spec_hip_probe_device) + ":" + + std::to_string(reserved_cus); + const char * existing = + std::getenv("DFLASH_HIP_RESERVED_TOOL_LANE"); + if (existing && *existing && isolation != existing) { + std::fprintf(stderr, + "[server] DFLASH_HIP_RESERVED_TOOL_LANE=%s " + "conflicts with profile-required %s\n", + existing, isolation.c_str()); + return 2; + } + set_environment_variable( + "DFLASH_HIP_RESERVED_TOOL_LANE", + isolation.c_str(), true); + sconfig.tool_speculation.hip_tool_device = + tool_spec_hip_probe_device; + sconfig.tool_speculation.hip_reserved_tool_compute_units = + reserved_cus; + std::fprintf(stderr, + "[server] disjoint HIP tool lane: device %d reserves " + "%d/%d low CU(s); model streams use the complement\n", + tool_spec_hip_probe_device, reserved_cus, + tool_spec_hip_probe_total_cus); + } + } + if (sconfig.tool_speculation.policy.requires_static_model_routing() && + !sconfig.tool_speculation.model_routing_static) { + std::fprintf(stderr, + "[server] same-physical-GPU tool lanes require static model " + "routing; disable DFLASH_MOE_TP_DYNAMIC_ROUTE_BALANCE and " + "DFLASH_DS4_TP_DYNAMIC_ROUTE_BALANCE\n"); + return 2; + } + if (sconfig.tool_speculation.policy.requires_unique_expert_ownership() && + !sconfig.tool_speculation.model_expert_ownership_unique) { + std::fprintf(stderr, + "[server] same-physical-GPU tool lanes require unique expert " + "ownership; disable DFLASH_MOE_DUPLICATE_HOT_ON_COLD\n"); + return 2; + } + std::fprintf(stderr, + "[server] tool speculation preserves token speculation; " + "unqualified resource lanes are deferred\n"); + } if (fast_rollback_forced_off) { bargs.fast_rollback = false; target_split_fast_rollback_cli = false; @@ -1117,6 +1505,57 @@ int main(int argc, char ** argv) { std::fprintf(stderr, "[server] │ prefix_cache = %d slots\n", sconfig.prefix_cache_cap); std::fprintf(stderr, "[server] │ prefill_cache = %d slots\n", sconfig.prefill_cache_cap); std::fprintf(stderr, "[server] │ cors = %s\n", sconfig.enable_cors ? "ON" : "off"); + std::fprintf(stderr, "[server] │ tool_speculation= %s\n", + sconfig.tool_speculation.enabled() ? "ON" : "off"); + std::fprintf(stderr, "[server] │ tool_call_predictor= %s\n", + sconfig.semantic_tool_predictor.enabled() ? "ON" : "off"); + if (sconfig.semantic_tool_predictor.enabled()) { + const auto & predictor = sconfig.semantic_tool_predictor; + std::fprintf(stderr, "[server] │ tool_hint_transport= %s%s\n", + predictor.native_enabled() ? "native-qwen3" : "http", + predictor.native_enabled() && predictor.http_enabled() + ? "+http-fallback" : ""); + std::fprintf(stderr, "[server] │ tool_hint_model = %s\n", + predictor.native_enabled() + ? predictor.native_model_path.c_str() + : predictor.model.c_str()); + if (predictor.native_enabled()) { + std::fprintf(stderr, + "[server] │ tool_hint_gpu = %d (max_ctx=%d)\n", + predictor.native_gpu, predictor.native_max_ctx); + std::fprintf(stderr, + "[server] │ tool_hint_schedule= %s\n", + native_tool_predictor_schedule_name( + predictor.native_schedule)); + } + std::fprintf(stderr, "[server] │ tool_hint_timeout= %d ms\n", + predictor.timeout_ms); + std::fprintf(stderr, "[server] │ tool_hint_execute= %s (confidence=%.3f)\n", + sconfig.tool_speculation.enabled() ? "ON" : "off", + predictor.execution_confidence); + } + if (sconfig.tool_speculation.enabled()) { + std::fprintf(stderr, "[server] │ tool_spec_exec = %s\n", + sconfig.tool_speculation.execution_mode()); + std::fprintf(stderr, "[server] │ tool_spec_profile= %s\n", + sconfig.tool_speculation.profile_path.c_str()); + std::fprintf(stderr, "[server] │ tool_spec_decode = %s\n", + "spec preserved (unqualified lanes deferred)"); + std::fprintf(stderr, "[server] │ tool_spec_routing= %s\n", + sconfig.tool_speculation.model_routing_static + ? "static" : "dynamic"); + std::fprintf(stderr, "[server] │ tool_spec_experts= %s\n", + sconfig.tool_speculation.model_expert_ownership_unique + ? "unique ownership" : "duplicated ownership"); + std::fprintf(stderr, "[server] │ tool_spec_lanes ="); + for (const auto & lane : sconfig.tool_speculation.policy.lanes()) { + std::fprintf(stderr, " %d%%:%s", + lane.resource_percentage, + lane.decode_interference_qualified + ? "qualified" : "deferred"); + } + std::fprintf(stderr, "\n"); + } std::fprintf(stderr, "[server] │ cache_type_k = %s\n", #ifdef GGML_USE_HIP cache_type_k.empty() ? "q4_0 (default, HIP)" : cache_type_k.c_str()); @@ -1233,6 +1672,13 @@ int main(int argc, char ** argv) { HttpServer server(*backend, tokenizer, sconfig); server.set_chat_format(chat_format_for_arch(arch)); + std::string semantic_predictor_error; + if (!server.init_semantic_tool_predictor(semantic_predictor_error)) { + std::fprintf(stderr, + "[server] native semantic tool predictor initialization failed: %s\n", + semantic_predictor_error.c_str()); + return 1; + } g_server = &server; std::signal(SIGTERM, signal_handler); std::signal(SIGINT, signal_handler); diff --git a/server/src/server/tool_speculation.cpp b/server/src/server/tool_speculation.cpp new file mode 100644 index 000000000..c3d679a3c --- /dev/null +++ b/server/src/server/tool_speculation.cpp @@ -0,0 +1,1239 @@ +#include "tool_speculation.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if !defined(_WIN32) +# include +# include +# include +# include +# include +# include +# include +# if defined(__linux__) +# include +# endif +extern char ** environ; +#endif + +namespace dflash::common { +namespace { + +constexpr size_t kMaxExecutorRequestBytes = 64 * 1024; + +bool finite_positive(double value) { + return std::isfinite(value) && value > 0.0; +} + +double median(std::vector values) { + if (values.empty()) return 0.0; + std::sort(values.begin(), values.end()); + const size_t middle = values.size() / 2; + if (values.size() % 2 != 0) return values[middle]; + return (values[middle - 1] + values[middle]) / 2.0; +} + +bool request_declares_tool(const json & tools, const std::string & name) { + if (!tools.is_array() || name.empty()) return false; + for (const auto & tool : tools) { + if (!tool.is_object()) continue; + if (tool.value("name", "") == name) return true; + if (tool.contains("function") && tool["function"].is_object() && + tool["function"].value("name", "") == name) { + return true; + } + } + return false; +} + +std::string format_cpu_affinity(const std::vector & cpus) { + std::string value; + for (const int cpu : cpus) { + if (!value.empty()) value.push_back(','); + value += std::to_string(cpu); + } + return value; +} + +#if !defined(_WIN32) +bool send_all_socket(int fd, const void * data, size_t bytes) { + const char * cursor = static_cast(data); + while (bytes > 0) { + int flags = 0; +# if defined(MSG_NOSIGNAL) + flags = MSG_NOSIGNAL; +# endif + const ssize_t written = ::send(fd, cursor, bytes, flags); + if (written < 0) { + if (errno == EINTR) continue; + return false; + } + if (written == 0) return false; + cursor += written; + bytes -= static_cast(written); + } + return true; +} + +std::vector executor_environment( + int resource_percentage, + const std::string & accelerator_relation, + const std::vector & cpu_affinity) { + std::vector values; + static constexpr const char * kResourceKey = + "DFLASH_TOOL_SPECULATION_RESOURCE_PERCENTAGE="; + static constexpr size_t kResourceKeyLen = + sizeof("DFLASH_TOOL_SPECULATION_RESOURCE_PERCENTAGE=") - 1; + static constexpr const char * kRelationKey = + "DFLASH_TOOL_SPECULATION_ACCELERATOR_RELATION="; + static constexpr size_t kRelationKeyLen = + sizeof("DFLASH_TOOL_SPECULATION_ACCELERATOR_RELATION=") - 1; + static constexpr const char * kCpuAffinityKey = + "DFLASH_TOOL_SPECULATION_CPU_AFFINITY="; + static constexpr size_t kCpuAffinityKeyLen = + sizeof("DFLASH_TOOL_SPECULATION_CPU_AFFINITY=") - 1; + bool resource_replaced = false; + bool relation_replaced = false; + bool cpu_affinity_replaced = false; + for (char ** item = environ; item && *item; ++item) { + const std::string value(*item); + if (value.compare(0, kResourceKeyLen, kResourceKey) == 0) { + values.push_back( + std::string(kResourceKey) + + std::to_string(resource_percentage)); + resource_replaced = true; + } else if (value.compare(0, kRelationKeyLen, kRelationKey) == 0) { + values.push_back( + std::string(kRelationKey) + accelerator_relation); + relation_replaced = true; + } else if (value.compare( + 0, kCpuAffinityKeyLen, kCpuAffinityKey) == 0) { + // Drop a stale inherited value when this executor has no CPU + // lane. Otherwise replace it with the verified canonical list. + if (!cpu_affinity.empty()) { + values.push_back( + std::string(kCpuAffinityKey) + + format_cpu_affinity(cpu_affinity)); + cpu_affinity_replaced = true; + } + } else { + values.push_back(value); + } + } + if (!resource_replaced) { + values.push_back( + std::string(kResourceKey) + + std::to_string(resource_percentage)); + } + if (!relation_replaced) { + values.push_back(std::string(kRelationKey) + accelerator_relation); + } + if (!cpu_affinity.empty() && !cpu_affinity_replaced) { + values.push_back( + std::string(kCpuAffinityKey) + + format_cpu_affinity(cpu_affinity)); + } + values.push_back("DFLASH_TOOL_SPECULATION=1"); + return values; +} + +# if defined(__linux__) +bool pin_and_verify_child_cpu_affinity( + pid_t child, + const std::vector & cpus, + std::string & error) { + if (cpus.empty()) { + error.clear(); + return true; + } + cpu_set_t requested; + CPU_ZERO(&requested); + for (const int cpu : cpus) { + if (cpu < 0 || cpu >= CPU_SETSIZE) { + error = "executor CPU is outside CPU_SETSIZE: " + + std::to_string(cpu); + return false; + } + CPU_SET(cpu, &requested); + } + if (::sched_setaffinity(child, sizeof(requested), &requested) != 0) { + error = std::string("executor sched_setaffinity failed: ") + + std::strerror(errno); + return false; + } + cpu_set_t observed; + CPU_ZERO(&observed); + if (::sched_getaffinity(child, sizeof(observed), &observed) != 0) { + error = std::string("executor sched_getaffinity failed: ") + + std::strerror(errno); + return false; + } + for (int cpu = 0; cpu < CPU_SETSIZE; ++cpu) { + if (CPU_ISSET(cpu, &requested) != CPU_ISSET(cpu, &observed)) { + error = "executor CPU affinity verification mismatch"; + return false; + } + } + error.clear(); + return true; +} +# endif +#endif + +} // namespace + +bool CanonicalToolInvocation::from_parts( + const std::string & name, + const json & arguments, + CanonicalToolInvocation & out, + std::string & error) { + if (name.empty()) { + error = "tool name must not be empty"; + return false; + } + if (!arguments.is_object()) { + error = "tool arguments must be a JSON object"; + return false; + } + out.name = name; + out.arguments = arguments; + // nlohmann::json's default object type is key ordered, so dump() is a + // stable canonical identity independent of input object insertion order. + out.arguments_json = arguments.dump(); + error.clear(); + return true; +} + +bool CanonicalToolInvocation::from_tool_call( + const ToolCall & call, + CanonicalToolInvocation & out, + std::string & error) { + try { + const json arguments = call.arguments.empty() + ? json::object() + : json::parse(call.arguments); + return from_parts(call.name, arguments, out, error); + } catch (const std::exception & exception) { + error = std::string("authoritative tool arguments are invalid JSON: ") + + exception.what(); + return false; + } +} + +bool build_tool_speculation_prediction( + const std::string & name, + const json & arguments, + double confidence, + ToolSpeculationPrediction & out, + std::string & error) { + if (!std::isfinite(confidence) || confidence < 0.0 || confidence > 1.0) { + error = "tool prediction confidence must be between 0 and 1"; + return false; + } + CanonicalToolInvocation invocation; + if (!CanonicalToolInvocation::from_parts( + name, arguments, invocation, error)) { + return false; + } + out.call = std::move(invocation); + out.confidence = confidence; + error.clear(); + return true; +} + +bool parse_tool_speculation_prediction( + const json & value, + const json & tools, + ToolSpeculationPrediction & out, + std::string & error) { + if (!value.is_object()) { + error = "tool_speculation must be an object"; + return false; + } + if (!value.contains("call") || !value["call"].is_object()) { + error = "tool_speculation.call must be an object"; + return false; + } + if (!value.contains("confidence") || !value["confidence"].is_number()) { + error = "tool_speculation.confidence must be a number"; + return false; + } + const double confidence = value["confidence"].get(); + if (!std::isfinite(confidence) || confidence < 0.0 || confidence > 1.0) { + error = "tool_speculation.confidence must be between 0 and 1"; + return false; + } + + const json & call = value["call"]; + if (!call.contains("name") || !call["name"].is_string()) { + error = "tool_speculation.call.name must be a string"; + return false; + } + if (!call.contains("arguments")) { + error = "tool_speculation.call.arguments is required"; + return false; + } + ToolSpeculationPrediction prediction; + if (!build_tool_speculation_prediction( + call["name"].get(), call["arguments"], confidence, + prediction, error)) { + return false; + } + if (!request_declares_tool(tools, prediction.call.name)) { + error = "tool_speculation.call.name is not declared in tools"; + return false; + } + out = std::move(prediction); + error.clear(); + return true; +} + +bool parse_tool_speculation_cpu_affinity( + const std::string & value, + std::vector & out, + std::string & error) { + out.clear(); + if (value.empty()) { + error = "tool CPU affinity must not be empty"; + return false; + } + size_t cursor = 0; + while (cursor < value.size()) { + const size_t comma = value.find(',', cursor); + const size_t end = comma == std::string::npos ? value.size() : comma; + const std::string token = value.substr(cursor, end - cursor); + if (token.empty()) { + error = "tool CPU affinity contains an empty item"; + out.clear(); + return false; + } + const size_t dash = token.find('-'); + auto parse_cpu = [&](const std::string & item, int & cpu) { + if (item.empty() || !std::all_of( + item.begin(), item.end(), [](unsigned char character) { + return character >= '0' && character <= '9'; + })) { + return false; + } + char * parsed_end = nullptr; + errno = 0; + const long parsed = std::strtol(item.c_str(), &parsed_end, 10); + if (errno != 0 || !parsed_end || *parsed_end != '\0' || + parsed < 0 || parsed > std::numeric_limits::max()) { + return false; + } + cpu = static_cast(parsed); + return true; + }; + int first = -1; + int last = -1; + if (dash == std::string::npos) { + if (!parse_cpu(token, first)) { + error = "invalid tool CPU affinity item: " + token; + out.clear(); + return false; + } + last = first; + } else if (token.find('-', dash + 1) != std::string::npos || + !parse_cpu(token.substr(0, dash), first) || + !parse_cpu(token.substr(dash + 1), last) || + first > last) { + error = "invalid tool CPU affinity range: " + token; + out.clear(); + return false; + } + if (static_cast(last) - + static_cast(first) > 65535ULL) { + error = "tool CPU affinity range is too large: " + token; + out.clear(); + return false; + } + for (int cpu = first; cpu <= last; ++cpu) { + out.push_back(cpu); + if (cpu == std::numeric_limits::max()) break; + } + if (comma == std::string::npos) break; + cursor = comma + 1; + } + std::sort(out.begin(), out.end()); + out.erase(std::unique(out.begin(), out.end()), out.end()); + error.clear(); + return true; +} + +bool qualify_tool_speculation_cpu_affinity( + ToolSpeculationConfig & config, + std::string & error) { + config.model_cpu_affinity.clear(); + config.cpu_affinity_isolated = false; + if (config.cpu_affinity.empty()) { + error.clear(); + return true; + } +#if defined(__linux__) + if (config.executor_path.empty() || config.in_process_executor) { + error = "tool CPU affinity requires a child-process executor"; + return false; + } + const long configured_cpus = ::sysconf(_SC_NPROCESSORS_CONF); + if (configured_cpus <= 0) { + error = "cannot determine configured CPU count"; + return false; + } + cpu_set_t model_set; + CPU_ZERO(&model_set); + if (::sched_getaffinity(0, sizeof(model_set), &model_set) != 0) { + error = std::string("model sched_getaffinity failed: ") + + std::strerror(errno); + return false; + } + for (int cpu = 0; cpu < CPU_SETSIZE; ++cpu) { + if (CPU_ISSET(cpu, &model_set)) { + config.model_cpu_affinity.push_back(cpu); + } + } + if (config.model_cpu_affinity.empty()) { + error = "model CPU affinity is empty"; + return false; + } + for (const int cpu : config.cpu_affinity) { + if (cpu < 0 || cpu >= CPU_SETSIZE || cpu >= configured_cpus) { + error = "tool CPU is not configured on this host: " + + std::to_string(cpu); + return false; + } + if (CPU_ISSET(cpu, &model_set)) { + error = "tool CPU affinity overlaps model CPU " + + std::to_string(cpu); + return false; + } + } + config.cpu_affinity_isolated = true; + error.clear(); + return true; +#else + error = "tool CPU affinity isolation is supported only on Linux"; + return false; +#endif +} + +bool ToolSpeculationPolicy::load_file( + const std::string & path, std::string & error) { + std::ifstream input(path); + if (!input) { + error = "cannot open tool-speculation profile: " + path; + lanes_.clear(); + baseline_task_ms_ = 0.0; + profile_status_ = "qualified"; + executor_contract_.clear(); + return false; + } + try { + json report; + input >> report; + return load_json(report, error); + } catch (const std::exception & exception) { + error = std::string("invalid tool-speculation profile JSON: ") + + exception.what(); + lanes_.clear(); + baseline_task_ms_ = 0.0; + profile_status_ = "qualified"; + executor_contract_.clear(); + return false; + } +} + +bool ToolSpeculationPolicy::load_json( + const json & report, std::string & error) { + lanes_.clear(); + baseline_task_ms_ = 0.0; + profile_status_ = "qualified"; + executor_contract_.clear(); + if (!report.is_object() || !report.contains("path_summary") || + !report["path_summary"].is_object() || + report["path_summary"].empty()) { + error = "tool-speculation profile needs a non-empty path_summary"; + return false; + } + + if (report.contains("profile_status")) { + if (!report["profile_status"].is_string()) { + error = "tool-speculation profile_status must be a string"; + return false; + } + profile_status_ = report["profile_status"].get(); + if (profile_status_ != "qualified" && + profile_status_ != "provisional_benchmark_only") { + error = "tool-speculation profile_status must be qualified or " + "provisional_benchmark_only"; + return false; + } + } + if (report.contains("executor")) { + if (!report["executor"].is_string()) { + error = "tool-speculation executor contract must be a string"; + return false; + } + executor_contract_ = report["executor"].get(); + if (executor_contract_.empty()) { + error = "tool-speculation executor contract cannot be empty"; + return false; + } + } + + std::vector controls; + try { + for (auto item = report["path_summary"].begin(); + item != report["path_summary"].end(); ++item) { + size_t parsed = 0; + const int resource_percentage = std::stoi(item.key(), &parsed); + if (parsed != item.key().size() || + resource_percentage < 1 || resource_percentage > 100) { + throw std::runtime_error( + "invalid resource percentage " + item.key()); + } + const json & paths = item.value(); + const json & hit = paths.at("hit"); + const json & miss = paths.at("miss"); + const double hit_control = hit.at("control_task_mean_ms").get(); + const double miss_control = miss.at("control_task_mean_ms").get(); + const double hit_task = hit.at("speculative_task_mean_ms").get(); + const double miss_task = miss.at("speculative_task_mean_ms").get(); + const double slowdown_percent = std::max( + hit.at("model_slowdown_percent").get(), + miss.at("model_slowdown_percent").get()); + bool decode_interference_qualified = false; + if (paths.contains("decode_interference_qualified")) { + if (!paths["decode_interference_qualified"].is_boolean()) { + throw std::runtime_error( + "decode_interference_qualified must be boolean"); + } + decode_interference_qualified = + paths["decode_interference_qualified"].get(); + } + const std::string accelerator_relation = + paths.value("accelerator_relation", "unspecified"); + if (accelerator_relation != "unspecified" && + accelerator_relation != "non_accelerator" && + accelerator_relation != "separate_physical_gpu" && + accelerator_relation != "same_physical_gpu") { + throw std::runtime_error( + "accelerator_relation must be unspecified, " + "non_accelerator, separate_physical_gpu, or " + "same_physical_gpu"); + } + bool requires_static_model_routing = false; + if (paths.contains("requires_static_model_routing")) { + if (!paths["requires_static_model_routing"].is_boolean()) { + throw std::runtime_error( + "requires_static_model_routing must be boolean"); + } + requires_static_model_routing = + paths["requires_static_model_routing"].get(); + } + bool requires_unique_expert_ownership = false; + if (paths.contains("requires_unique_expert_ownership")) { + if (!paths["requires_unique_expert_ownership"].is_boolean()) { + throw std::runtime_error( + "requires_unique_expert_ownership must be boolean"); + } + requires_unique_expert_ownership = + paths["requires_unique_expert_ownership"].get(); + } + if (!finite_positive(hit_control) || + !finite_positive(miss_control) || + !finite_positive(hit_task) || + !finite_positive(miss_task) || + !std::isfinite(slowdown_percent) || slowdown_percent < -100.0) { + throw std::runtime_error("non-positive or non-finite profile latency"); + } + const double control = (hit_control + miss_control) / 2.0; + lanes_.push_back({ + resource_percentage, + control, + hit_task, + miss_task, + 1.0 + slowdown_percent / 100.0, + decode_interference_qualified, + accelerator_relation, + requires_static_model_routing, + requires_unique_expert_ownership, + }); + controls.push_back(control); + } + } catch (const std::exception & exception) { + error = std::string("invalid tool-speculation path_summary: ") + + exception.what(); + lanes_.clear(); + profile_status_ = "qualified"; + executor_contract_.clear(); + return false; + } + + std::sort(lanes_.begin(), lanes_.end(), + [](const auto & left, const auto & right) { + return left.resource_percentage < + right.resource_percentage; + }); + for (size_t index = 1; index < lanes_.size(); ++index) { + if (lanes_[index - 1].resource_percentage == + lanes_[index].resource_percentage) { + error = + "tool-speculation profile has duplicate resource percentages"; + lanes_.clear(); + return false; + } + } + baseline_task_ms_ = median(std::move(controls)); + error.clear(); + return true; +} + +bool ToolSpeculationPolicy::requires_static_model_routing() const { + return std::any_of( + lanes_.begin(), lanes_.end(), + [](const ToolSpeculationLane & lane) { + return lane.requires_static_model_routing; + }); +} + +bool ToolSpeculationPolicy::requires_unique_expert_ownership() const { + return std::any_of( + lanes_.begin(), lanes_.end(), + [](const ToolSpeculationLane & lane) { + return lane.requires_unique_expert_ownership; + }); +} + +ToolSpeculationAdmission ToolSpeculationPolicy::choose( + double confidence, + double max_model_slowdown_ratio) const { + ToolSpeculationAdmission decision; + decision.expected_task_ms = baseline_task_ms_; + if (lanes_.empty() || !finite_positive(baseline_task_ms_)) { + decision.reason = "profile_unavailable"; + return decision; + } + if (!std::isfinite(confidence) || confidence < 0.0 || confidence > 1.0) { + decision.reason = "invalid_confidence"; + return decision; + } + if (!std::isfinite(max_model_slowdown_ratio) || + max_model_slowdown_ratio < 1.0) { + decision.reason = "invalid_slowdown_guardrail"; + return decision; + } + + bool qualified_lane_available = false; + bool lane_passed_guardrail = false; + double best = baseline_task_ms_; + for (const ToolSpeculationLane & lane : lanes_) { + // Token speculation is an invariant, not a fallback choice. A lane + // may overlap DS4/DSpark only after its exact executor and placement + // passed the output-identity interference gate. + if (!lane.decode_interference_qualified) continue; + qualified_lane_available = true; + if (lane.model_slowdown_ratio > max_model_slowdown_ratio) continue; + lane_passed_guardrail = true; + const double expected = + confidence * lane.hit_task_ms + + (1.0 - confidence) * lane.miss_task_ms; + if (expected < best) { + best = expected; + decision.admitted = true; + decision.resource_percentage = lane.resource_percentage; + decision.expected_task_ms = expected; + decision.decode_interference_qualified = + lane.decode_interference_qualified; + decision.accelerator_relation = lane.accelerator_relation; + } + } + if (!decision.admitted) { + decision.reason = !qualified_lane_available + ? "decode_interference_unqualified" + : lane_passed_guardrail + ? "below_profile_break_even" + : "model_slowdown_guardrail"; + return decision; + } + decision.expected_speedup = baseline_task_ms_ / best; + decision.reason = "expected_latency_gain"; + return decision; +} + +bool ToolSpeculationConfig::allows(const std::string & name) const { + return std::find(allowed_tools.begin(), allowed_tools.end(), name) != + allowed_tools.end(); +} + +ToolSpeculationAttempt::ToolSpeculationAttempt( + const ToolSpeculationConfig & config, + const ToolSpeculationPrediction & prediction, + const std::string & request_id) + : config_(config) + , prediction_(prediction) + , request_id_(request_id) { + if (!config_.enabled()) { + admission_.reason = "engine_disabled"; + } else if (!config_.allows(prediction_.call.name)) { + admission_.reason = "tool_not_allowlisted"; + } else { + admission_ = config_.policy.choose( + prediction_.confidence, config_.max_model_slowdown_ratio); + } +} + +ToolSpeculationAttempt::~ToolSpeculationAttempt() { + if (!resolved_) terminate_executor(); +} + +std::unique_ptr ToolSpeculationAttempt::create( + const ToolSpeculationConfig & config, + const ToolSpeculationPrediction & prediction, + const std::string & request_id) { + return std::unique_ptr( + new ToolSpeculationAttempt(config, prediction, request_id)); +} + +void ToolSpeculationAttempt::start() { + if (started_) return; + started_ = true; + if (!admission_.admitted) return; + const json request = { + {"protocol", "dflash.tool-speculation.v1"}, + {"request_id", request_id_}, + {"mode", "speculative"}, + {"resource_percentage", admission_.resource_percentage}, + {"accelerator_relation", admission_.accelerator_relation}, + {"cpu_affinity", config_.cpu_affinity}, + {"cpu_affinity_isolated", config_.cpu_affinity_isolated}, + {"call", { + {"name", prediction_.call.name}, + {"arguments", prediction_.call.arguments}, + }}, + }; + started_at_ = std::chrono::steady_clock::now(); + if (config_.in_process_executor) { + in_process_execution_ = + config_.in_process_executor->start(request, launch_error_); + running_ = static_cast(in_process_execution_); + if (running_) { + std::fprintf(stderr, + "[tool-spec] launched request=%s tool=%s confidence=%.3f " + "resource=%d%% mode=%s\n", + request_id_.c_str(), prediction_.call.name.c_str(), + prediction_.confidence, admission_.resource_percentage, + config_.execution_mode()); + } + return; + } +#if defined(_WIN32) + launch_error_ = "tool speculation child executors are not implemented on Windows"; + return; +#else + const std::string payload = request.dump() + "\n"; + if (payload.size() > kMaxExecutorRequestBytes) { + launch_error_ = "executor request exceeds 64 KiB"; + return; + } + + int input_socket[2] = {-1, -1}; + if (::socketpair(AF_UNIX, SOCK_STREAM, 0, input_socket) != 0) { + launch_error_ = std::string("executor stdin socket failed: ") + + std::strerror(errno); + return; + } +# if defined(SO_NOSIGPIPE) + int no_sigpipe = 1; + ::setsockopt(input_socket[0], SOL_SOCKET, SO_NOSIGPIPE, + &no_sigpipe, sizeof(no_sigpipe)); +# endif + int output_pipe[2] = {-1, -1}; + if (::pipe(output_pipe) != 0) { + launch_error_ = std::string("executor stdout pipe failed: ") + + std::strerror(errno); + ::close(input_socket[0]); + ::close(input_socket[1]); + return; + } + + posix_spawn_file_actions_t actions; + int spawn_status = posix_spawn_file_actions_init(&actions); + const bool actions_initialized = spawn_status == 0; + if (spawn_status == 0) { + spawn_status = posix_spawn_file_actions_adddup2( + &actions, input_socket[1], STDIN_FILENO); + } + if (spawn_status == 0) { + spawn_status = posix_spawn_file_actions_adddup2( + &actions, output_pipe[1], STDOUT_FILENO); + } + if (spawn_status == 0) { + spawn_status = posix_spawn_file_actions_addclose(&actions, output_pipe[0]); + } + if (spawn_status == 0) { + spawn_status = posix_spawn_file_actions_addclose(&actions, input_socket[0]); + } + if (spawn_status == 0 && input_socket[1] != STDIN_FILENO) { + spawn_status = posix_spawn_file_actions_addclose(&actions, input_socket[1]); + } + if (spawn_status == 0 && output_pipe[1] != STDOUT_FILENO) { + spawn_status = posix_spawn_file_actions_addclose(&actions, output_pipe[1]); + } + + std::vector env_storage = executor_environment( + admission_.resource_percentage, admission_.accelerator_relation, + config_.cpu_affinity); + std::vector env; + env.reserve(env_storage.size() + 1); + for (std::string & value : env_storage) env.push_back(value.data()); + env.push_back(nullptr); + + std::string executable = config_.executor_path; + std::string protocol_arg = "--dflash-tool-spec-v1"; + char * argv[] = { + executable.data(), + protocol_arg.data(), + nullptr, + }; + pid_t child = -1; + if (spawn_status == 0) { + spawn_status = ::posix_spawn( + &child, executable.c_str(), &actions, nullptr, argv, env.data()); + } + if (actions_initialized) { + posix_spawn_file_actions_destroy(&actions); + } + ::close(input_socket[1]); + ::close(output_pipe[1]); + if (spawn_status != 0) { + launch_error_ = std::string("executor spawn failed: ") + + std::strerror(spawn_status); + ::close(input_socket[0]); + ::close(output_pipe[0]); + return; + } + +# if defined(__linux__) + if (!config_.cpu_affinity.empty()) { + std::string affinity_error; + if (!pin_and_verify_child_cpu_affinity( + child, config_.cpu_affinity, affinity_error)) { + ::kill(child, SIGKILL); + int child_status = 0; + while (::waitpid(child, &child_status, 0) < 0 && errno == EINTR) {} + ::close(input_socket[0]); + ::close(output_pipe[0]); + launch_error_ = std::move(affinity_error); + return; + } + } +# else + if (!config_.cpu_affinity.empty()) { + ::kill(child, SIGKILL); + int child_status = 0; + while (::waitpid(child, &child_status, 0) < 0 && errno == EINTR) {} + ::close(input_socket[0]); + ::close(output_pipe[0]); + launch_error_ = "tool CPU affinity isolation is supported only on Linux"; + return; + } +# endif + + child_pid_ = static_cast(child); + child_stdin_fd_ = input_socket[0]; + child_stdout_fd_ = output_pipe[0]; + const int flags = ::fcntl(child_stdout_fd_, F_GETFL, 0); + if (flags >= 0) { + ::fcntl(child_stdout_fd_, F_SETFL, flags | O_NONBLOCK); + } + running_ = true; + if (!send_all_socket(child_stdin_fd_, payload.data(), payload.size())) { + launch_error_ = std::string("executor request write failed: ") + + std::strerror(errno); + terminate_executor(); + return; + } + std::fprintf(stderr, + "[tool-spec] launched request=%s tool=%s confidence=%.3f " + "resource=%d%%\n", + request_id_.c_str(), prediction_.call.name.c_str(), + prediction_.confidence, admission_.resource_percentage); +#endif +} + +json ToolSpeculationAttempt::base_metadata() const { + json metadata = { + {"protocol", "dflash.tool-speculation.v1"}, + {"confidence", prediction_.confidence}, + {"prediction", { + {"name", prediction_.call.name}, + {"arguments", prediction_.call.arguments}, + }}, + {"resource_percentage", + admission_.admitted + ? json(admission_.resource_percentage) + : json(nullptr)}, + {"expected_speedup", + admission_.admitted ? json(admission_.expected_speedup) : json(nullptr)}, + {"decode_interference_qualified", + admission_.admitted + ? json(admission_.decode_interference_qualified) + : json(nullptr)}, + {"accelerator_relation", + admission_.admitted + ? json(admission_.accelerator_relation) + : json(nullptr)}, + {"cpu_affinity", config_.cpu_affinity}, + {"cpu_affinity_isolated", config_.cpu_affinity_isolated}, + }; + return metadata; +} + +bool ToolSpeculationAttempt::send_control(const char * operation) { + if (in_process_execution_) { + return operation && *operation && + in_process_execution_->send_control(operation); + } +#if defined(_WIN32) + (void)operation; + return false; +#else + if (child_stdin_fd_ < 0 || !operation || !*operation) return false; + const std::string command = json({ + {"protocol", "dflash.tool-speculation.v1"}, + {"request_id", request_id_}, + {"op", operation}, + {"authoritative_resource_percentage", 100}, + }).dump() + "\n"; + return send_all_socket( + child_stdin_fd_, command.data(), command.size()); +#endif +} + +void ToolSpeculationAttempt::terminate_executor(bool allow_control_grace) { + if (in_process_execution_) { + in_process_execution_->terminate(allow_control_grace); + in_process_execution_.reset(); + running_ = false; + return; + } +#if !defined(_WIN32) + if (child_stdin_fd_ >= 0) { + ::close(child_stdin_fd_); + child_stdin_fd_ = -1; + } + if (child_stdout_fd_ >= 0) { + ::close(child_stdout_fd_); + child_stdout_fd_ = -1; + } + if (child_pid_ > 0) { + const pid_t pid = static_cast(child_pid_); + auto wait_until = [&](std::chrono::steady_clock::time_point deadline) { + int status = 0; + while (std::chrono::steady_clock::now() < deadline) { + const pid_t waited = ::waitpid(pid, &status, WNOHANG); + if (waited == pid || (waited < 0 && errno == ECHILD)) { + child_pid_ = -1; + running_ = false; + return true; + } + if (waited < 0 && errno != EINTR) break; + std::this_thread::sleep_for(std::chrono::milliseconds(2)); + } + return false; + }; + const int grace_ms = std::max(0, config_.cancel_grace_ms); + if (allow_control_grace && wait_until( + std::chrono::steady_clock::now() + + std::chrono::milliseconds(grace_ms))) { + return; + } + ::kill(pid, SIGTERM); + const int term_grace_ms = allow_control_grace + ? std::min(20, grace_ms) : grace_ms; + if (wait_until(std::chrono::steady_clock::now() + + std::chrono::milliseconds(term_grace_ms))) { + return; + } + int status = 0; + ::kill(pid, SIGKILL); + while (::waitpid(pid, &status, 0) < 0 && errno == EINTR) {} + child_pid_ = -1; + } +#endif + running_ = false; +} + +bool ToolSpeculationAttempt::collect_executor_result( + json & result, + double & wait_ms, + std::string & error) { + if (in_process_execution_) { + const bool ok = in_process_execution_->collect_result( + config_.timeout_ms, config_.max_result_bytes, + result, wait_ms, error); + in_process_execution_.reset(); + running_ = false; + return ok; + } +#if defined(_WIN32) + (void)result; + wait_ms = 0.0; + error = "tool speculation executors are not implemented on Windows"; + return false; +#else + const auto wait_started = std::chrono::steady_clock::now(); + const auto deadline = wait_started + + std::chrono::milliseconds(std::max(1, config_.timeout_ms)); + std::string output; + bool eof = false; + while (!eof) { + const auto now = std::chrono::steady_clock::now(); + if (now >= deadline) { + error = "executor_timeout"; + terminate_executor(); + wait_ms = std::chrono::duration( + std::chrono::steady_clock::now() - wait_started).count(); + return false; + } + const int remaining_ms = std::max(1, static_cast( + std::chrono::duration_cast( + deadline - now).count())); + pollfd descriptor{child_stdout_fd_, POLLIN | POLLHUP, 0}; + const int polled = ::poll(&descriptor, 1, remaining_ms); + if (polled < 0) { + if (errno == EINTR) continue; + error = std::string("executor_poll_failed: ") + std::strerror(errno); + terminate_executor(); + return false; + } + if (polled == 0) continue; + if (descriptor.revents & (POLLERR | POLLNVAL)) { + error = "executor_stdout_failed"; + terminate_executor(); + return false; + } + if (descriptor.revents & (POLLIN | POLLHUP)) { + char buffer[8192]; + while (true) { + const ssize_t count = ::read( + child_stdout_fd_, buffer, sizeof(buffer)); + if (count > 0) { + if (output.size() + static_cast(count) > + config_.max_result_bytes) { + error = "executor_result_too_large"; + terminate_executor(); + return false; + } + output.append(buffer, static_cast(count)); + continue; + } + if (count == 0) { + eof = true; + break; + } + if (errno == EINTR) continue; + if (errno == EAGAIN || errno == EWOULDBLOCK) break; + error = std::string("executor_read_failed: ") + + std::strerror(errno); + terminate_executor(); + return false; + } + } + } + ::close(child_stdout_fd_); + child_stdout_fd_ = -1; + + int child_status = 0; + while (true) { + const pid_t waited = ::waitpid( + static_cast(child_pid_), &child_status, WNOHANG); + if (waited == static_cast(child_pid_)) break; + if (waited < 0) { + if (errno == EINTR) continue; + error = std::string("executor_wait_failed: ") + + std::strerror(errno); + child_pid_ = -1; + running_ = false; + return false; + } + if (std::chrono::steady_clock::now() >= deadline) { + error = "executor_exit_timeout"; + terminate_executor(); + return false; + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + child_pid_ = -1; + running_ = false; + wait_ms = std::chrono::duration( + std::chrono::steady_clock::now() - wait_started).count(); + if (!WIFEXITED(child_status) || WEXITSTATUS(child_status) != 0) { + error = WIFEXITED(child_status) + ? "executor_exit_" + std::to_string(WEXITSTATUS(child_status)) + : "executor_terminated"; + return false; + } + + try { + const json envelope = json::parse(output); + if (!envelope.is_object() || !envelope.value("ok", false) || + !envelope.contains("result")) { + error = "executor_rejected_or_invalid_envelope"; + return false; + } + result = envelope["result"]; + error.clear(); + return true; + } catch (const std::exception & exception) { + error = std::string("executor_invalid_json: ") + exception.what(); + return false; + } +#endif +} + +json ToolSpeculationAttempt::resolve( + const std::vector & authoritative_calls) { + if (resolved_) { + json metadata = base_metadata(); + metadata["status"] = "failed"; + metadata["reason"] = "already_resolved"; + return metadata; + } + resolved_ = true; + json metadata = base_metadata(); + if (!admission_.admitted) { + metadata["status"] = "deferred"; + metadata["reason"] = admission_.reason; + return metadata; + } + if (!launch_error_.empty() || !running_) { + metadata["status"] = "failed"; + metadata["reason"] = "executor_launch_failed"; + metadata["detail"] = launch_error_.empty() + ? "executor did not start" : launch_error_; + terminate_executor(); + return metadata; + } + if (authoritative_calls.size() != 1) { + send_control("cancel"); + terminate_executor(true); + metadata["status"] = "miss"; + metadata["reason"] = "authoritative_call_count"; + return metadata; + } + + CanonicalToolInvocation authoritative; + std::string canonical_error; + if (!CanonicalToolInvocation::from_tool_call( + authoritative_calls[0], authoritative, canonical_error)) { + send_control("cancel"); + terminate_executor(true); + metadata["status"] = "miss"; + metadata["reason"] = "invalid_authoritative_call"; + return metadata; + } + if (!(authoritative == prediction_.call)) { + send_control("cancel"); + terminate_executor(true); + metadata["status"] = "miss"; + metadata["reason"] = "invocation_mismatch"; + return metadata; + } + + json result; + double wait_ms = 0.0; + std::string executor_error; + const bool commit_signal_sent = send_control("commit"); +#if !defined(_WIN32) + if (child_stdin_fd_ >= 0) { + ::close(child_stdin_fd_); + child_stdin_fd_ = -1; + } +#endif + if (!collect_executor_result(result, wait_ms, executor_error)) { + metadata["status"] = "failed"; + metadata["reason"] = "speculative_executor_failure"; + metadata["detail"] = executor_error; + metadata["commit_signal_sent"] = commit_signal_sent; + metadata["commit_wait_ms"] = wait_ms; + return metadata; + } + const double wall_ms = std::chrono::duration( + std::chrono::steady_clock::now() - started_at_).count(); + metadata["status"] = "hit"; + metadata["call_id"] = authoritative_calls[0].id; + metadata["result"] = std::move(result); + metadata["commit_signal_sent"] = commit_signal_sent; + metadata["executor_wall_ms"] = wall_ms; + metadata["commit_wait_ms"] = wait_ms; + std::fprintf(stderr, + "[tool-spec] hit request=%s tool=%s resource=%d%% " + "wall_ms=%.1f wait_ms=%.1f\n", + request_id_.c_str(), prediction_.call.name.c_str(), + admission_.resource_percentage, wall_ms, wait_ms); + return metadata; +} + +json ToolSpeculationAttempt::cancel(const std::string & reason) { + if (!resolved_) { + resolved_ = true; + send_control("cancel"); + terminate_executor(true); + } + json metadata = base_metadata(); + metadata["status"] = "cancelled"; + metadata["reason"] = reason; + return metadata; +} + +std::string render_tool_speculation_sse( + ApiFormat api_format, + const std::string & request_id, + const std::string & model, + const json & metadata) { + switch (api_format) { + case ApiFormat::OPENAI_CHAT: { + const json event = { + {"id", request_id}, + {"object", "chat.completion.chunk"}, + {"model", model}, + {"choices", json::array()}, + {"dflash_tool_speculation", metadata}, + }; + return "data: " + event.dump() + "\n\n"; + } + case ApiFormat::ANTHROPIC: { + const json event = { + {"type", "dflash_tool_speculation"}, + {"dflash_tool_speculation", metadata}, + }; + return "event: dflash_tool_speculation\ndata: " + + event.dump() + "\n\n"; + } + case ApiFormat::RESPONSES: { + const json event = { + {"type", "response.dflash_tool_speculation"}, + {"response_id", request_id}, + {"dflash_tool_speculation", metadata}, + }; + return "event: response.dflash_tool_speculation\ndata: " + + event.dump() + "\n\n"; + } + default: + return "data: " + json({{"dflash_tool_speculation", metadata}}).dump() + + "\n\n"; + } +} + +} // namespace dflash::common diff --git a/server/src/server/tool_speculation.h b/server/src/server/tool_speculation.h new file mode 100644 index 000000000..2e38fadb9 --- /dev/null +++ b/server/src/server/tool_speculation.h @@ -0,0 +1,277 @@ +// Lossless, confidence-gated speculative tool execution. +// +// The model remains authoritative. A predicted read-only invocation may run +// while inference is in flight, but its result is returned only when the +// emitted tool name and canonical JSON arguments match exactly. + +#pragma once + +#include "api_types.h" +#include "tool_parser.h" + +#include + +#include +#include +#include +#include +#include + +namespace dflash::common { + +using json = nlohmann::json; + +struct CanonicalToolInvocation { + std::string name; + json arguments = json::object(); + std::string arguments_json; + + static bool from_parts(const std::string & name, + const json & arguments, + CanonicalToolInvocation & out, + std::string & error); + static bool from_tool_call(const ToolCall & call, + CanonicalToolInvocation & out, + std::string & error); + + bool operator==(const CanonicalToolInvocation & other) const { + return name == other.name && arguments_json == other.arguments_json; + } +}; + +struct ToolSpeculationPrediction { + CanonicalToolInvocation call; + double confidence = 0.0; +}; + +// Construct a canonical prediction from an engine-side predictor. This is +// the same validation boundary used for caller-supplied predictions, minus +// the request-schema check performed by the semantic predictor itself. +bool build_tool_speculation_prediction(const std::string & name, + const json & arguments, + double confidence, + ToolSpeculationPrediction & out, + std::string & error); + +// Parse the request extension: +// "tool_speculation": { +// "call": {"name": "...", "arguments": {...}}, +// "confidence": 0.0..1.0 +// } +// The predicted tool must also be present in the request's `tools` array. +bool parse_tool_speculation_prediction(const json & value, + const json & tools, + ToolSpeculationPrediction & out, + std::string & error); + +// Parse a Linux CPU-list such as "14-15,30-31". The result is sorted and +// deduplicated so it can be compared directly with an observed affinity mask. +bool parse_tool_speculation_cpu_affinity(const std::string & value, + std::vector & out, + std::string & error); + +struct ToolSpeculationLane { + // Backend-neutral executor capacity. A CUDA adapter may map this to an + // MPS share; a ROCm, CPU, I/O, or remote adapter may interpret it using + // its own measured resource contract. + int resource_percentage = 0; + double control_task_ms = 0.0; + double hit_task_ms = 0.0; + double miss_task_ms = 0.0; + double model_slowdown_ratio = 1.0; + // True only when this exact executor/resource lane passed the model-output + // interference gate. Missing profile metadata defers tool speculation; + // token speculation is never disabled or replaced with AR decode. + bool decode_interference_qualified = false; + // Physical relationship between the tool accelerator and the model's + // primary accelerator. Same-GPU lanes have stricter runtime requirements + // because stream priority and CU masks do not isolate shared kernels. + std::string accelerator_relation = "unspecified"; + bool requires_static_model_routing = false; + bool requires_unique_expert_ownership = false; +}; + +struct ToolSpeculationAdmission { + bool admitted = false; + int resource_percentage = 0; + double expected_task_ms = 0.0; + double expected_speedup = 1.0; + bool decode_interference_qualified = false; + std::string accelerator_relation = "unspecified"; + std::string reason; +}; + +// Optional trusted in-process executor. This avoids a second accelerator +// process/context on runtimes where process-level time-slicing defeats CU or +// stream isolation. Implementations remain behind the same allowlist, +// empirical admission policy, exact-call commit, and private-result boundary +// as the child-process adapter. +class ToolSpeculationExecution { +public: + virtual ~ToolSpeculationExecution() = default; + virtual bool send_control(const std::string & operation) = 0; + virtual bool collect_result(int timeout_ms, + size_t max_result_bytes, + json & result, + double & wait_ms, + std::string & error) = 0; + virtual void terminate(bool allow_control_grace) = 0; +}; + +class ToolSpeculationExecutor { +public: + virtual ~ToolSpeculationExecutor() = default; + virtual std::unique_ptr start( + const json & request, + std::string & error) = 0; + virtual const char * mode_name() const = 0; +}; + +// Runtime policy loaded from a qualification report's `path_summary`. This +// keeps backend-specific interference measurements out of hard-coded engine +// heuristics. +class ToolSpeculationPolicy { +public: + bool load_file(const std::string & path, std::string & error); + bool load_json(const json & report, std::string & error); + + ToolSpeculationAdmission choose( + double confidence, + double max_model_slowdown_ratio) const; + + bool empty() const { return lanes_.empty(); } + double baseline_task_ms() const { return baseline_task_ms_; } + const std::vector & lanes() const { return lanes_; } + const std::string & profile_status() const { return profile_status_; } + const std::string & executor_contract() const { return executor_contract_; } + bool benchmark_only() const { + return profile_status_ == "provisional_benchmark_only"; + } + bool requires_static_model_routing() const; + bool requires_unique_expert_ownership() const; + +private: + std::vector lanes_; + double baseline_task_ms_ = 0.0; + std::string profile_status_ = "qualified"; + std::string executor_contract_; +}; + +struct ToolSpeculationConfig { + std::string executor_path; + std::shared_ptr in_process_executor; + std::string profile_path; + std::vector allowed_tools; + ToolSpeculationPolicy policy; + int timeout_ms = 60000; + int cancel_grace_ms = 100; + size_t max_result_bytes = 1024 * 1024; + double max_model_slowdown_ratio = 1.20; + // Snapshot of the model routing mode used to validate profile/runtime + // compatibility at startup and expose it through /props. + bool model_routing_static = true; + bool model_expert_ownership_unique = true; + // Runtime evidence that the model and an in-process HIP tool use + // complementary CU masks. Zero means no model-side CU reservation. + int hip_tool_device = -1; + int hip_reserved_tool_compute_units = 0; + // Optional child-process CPU lane. Startup verifies that these logical + // CPUs are disjoint from the model process affinity; every child is pinned + // and re-read before its request payload is released. + std::vector cpu_affinity; + std::vector model_cpu_affinity; + bool cpu_affinity_isolated = false; + bool enabled() const { + return (!executor_path.empty() || in_process_executor) && + !allowed_tools.empty() && + !policy.empty(); + } + const char * execution_mode() const { + return in_process_executor + ? in_process_executor->mode_name() + : executor_path.empty() + ? "disabled" + : cpu_affinity.empty() + ? "child_process" + : "child_process_cpu_affinity"; + } + bool allows(const std::string & name) const; +}; + +// Capture the model process affinity and fail closed unless it is physically +// disjoint from the configured child executor CPUs. No-op when no CPU lane is +// requested. +bool qualify_tool_speculation_cpu_affinity(ToolSpeculationConfig & config, + std::string & error); + +// One request-scoped attempt. The configured executable is invoked without a +// shell and receives one JSON request on stdin. It must emit one JSON envelope +// on stdout: {"ok":true,"result":...}. Stdin remains open for a later +// `commit` (exact match; promote checkpointed remainder to the authoritative +// 100% lane) or `cancel` control record. A thin executable may forward this +// protocol to a persistent warm tool pool, keeping GPU initialization outside +// the request's critical path. +class ToolSpeculationAttempt { +public: + ToolSpeculationAttempt(const ToolSpeculationAttempt &) = delete; + ToolSpeculationAttempt & operator=(const ToolSpeculationAttempt &) = delete; + ~ToolSpeculationAttempt(); + + static std::unique_ptr create( + const ToolSpeculationConfig & config, + const ToolSpeculationPrediction & prediction, + const std::string & request_id); + + // Launch admitted work. Deferred and launch-failed attempts still return + // metadata through resolve(), so an opted-in client can see why it must + // execute the authoritative tool normally. + void start(); + + // Exact-match one authoritative call, expose a successful private result, + // or discard/cancel it. This method is single-use. + json resolve(const std::vector & authoritative_calls); + + // Cancel without exposing a result (disconnect, generation failure, etc.). + json cancel(const std::string & reason); + + bool admitted() const { return admission_.admitted; } + bool running() const { return running_; } +private: + ToolSpeculationAttempt(const ToolSpeculationConfig & config, + const ToolSpeculationPrediction & prediction, + const std::string & request_id); + + json base_metadata() const; + bool send_control(const char * operation); + void terminate_executor(bool allow_control_grace = false); + bool collect_executor_result(json & result, + double & wait_ms, + std::string & error); + + ToolSpeculationConfig config_; + ToolSpeculationPrediction prediction_; + std::string request_id_; + ToolSpeculationAdmission admission_; + std::chrono::steady_clock::time_point started_at_{}; + bool started_ = false; + bool running_ = false; + bool resolved_ = false; + std::string launch_error_; + std::unique_ptr in_process_execution_; + +#if !defined(_WIN32) + int child_stdin_fd_ = -1; + int child_stdout_fd_ = -1; + int child_pid_ = -1; +#endif +}; + +// Custom SSE extension emitted only for requests that supplied +// `tool_speculation`. Non-streaming responses use the same object under the +// top-level `dflash_tool_speculation` key. +std::string render_tool_speculation_sse(ApiFormat api_format, + const std::string & request_id, + const std::string & model, + const json & metadata); + +} // namespace dflash::common diff --git a/server/src/server/tool_speculation_hip_probe.cpp b/server/src/server/tool_speculation_hip_probe.cpp new file mode 100644 index 000000000..3c6155617 --- /dev/null +++ b/server/src/server/tool_speculation_hip_probe.cpp @@ -0,0 +1,456 @@ +#include "tool_speculation_hip_probe.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace dflash::common { +namespace { + +std::string hip_error(const char * operation, hipError_t status) { + return std::string(operation) + ": " + hipGetErrorString(status); +} + +std::string hipblas_error(const char * operation, hipblasStatus_t status) { + return std::string(operation) + " failed with status " + + std::to_string(static_cast(status)); +} + +// HIP's current device is thread-local process state. The HTTP worker that +// launches a tool may immediately continue into model inference, so a trusted +// executor must leave that state exactly as it found it. +class ScopedHipDevice final { +public: + explicit ScopedHipDevice(int device) { + status_ = hipGetDevice(&previous_device_); + if (status_ != hipSuccess) return; + if (previous_device_ == device) return; + status_ = hipSetDevice(device); + switched_ = status_ == hipSuccess; + } + + ~ScopedHipDevice() { + if (switched_) (void) hipSetDevice(previous_device_); + } + + bool ok() const { return status_ == hipSuccess; } + hipError_t status() const { return status_; } + +private: + int previous_device_ = 0; + hipError_t status_ = hipSuccess; + bool switched_ = false; +}; + +class HipSgemmState; + +class HipSgemmExecution final : public ToolSpeculationExecution { +public: + explicit HipSgemmExecution(std::shared_ptr state) + : state_(std::move(state)) {} + ~HipSgemmExecution() override; + + bool send_control(const std::string & operation) override; + bool collect_result(int timeout_ms, + size_t max_result_bytes, + json & result, + double & wait_ms, + std::string & error) override; + void terminate(bool allow_control_grace) override; + +private: + std::shared_ptr state_; + bool committed_ = false; + bool finished_ = false; +}; + +class HipSgemmState final { +public: + HipSgemmState(int device, int matrix_size, int total_cus) + : device_(device), matrix_size_(matrix_size), total_cus_(total_cus) {} + + ~HipSgemmState() { + std::lock_guard lock(mutex_); + ScopedHipDevice device(device_); + if (!device.ok()) return; + if (stream_) (void) hipStreamSynchronize(stream_); + if (finished_) (void) hipEventDestroy(finished_); + if (started_) (void) hipEventDestroy(started_); + if (handle_) (void) hipblasDestroy(handle_); + if (c_) (void) hipFree(c_); + if (b_) (void) hipFree(b_); + if (a_) (void) hipFree(a_); + if (stream_) (void) hipStreamDestroy(stream_); + } + + bool start(const json & request, std::string & error) { + ScopedHipDevice device(device_); + if (!device.ok()) { + error = hip_error("hipSetDevice(tool)", device.status()); + return false; + } + std::lock_guard lock(mutex_); + if (active_) { + error = "HIP probe already has active work"; + return false; + } + try { + const json & call = request.at("call"); + if (call.at("name").get() != "benchmark_hip_sgemm") { + error = "HIP probe only supports benchmark_hip_sgemm"; + return false; + } + const json & arguments = call.at("arguments"); + if (!arguments.is_object() || + !arguments.contains("iterations") || + !arguments["iterations"].is_number_integer()) { + error = "benchmark_hip_sgemm.iterations must be an integer"; + return false; + } + iterations_ = arguments["iterations"].get(); + if (iterations_ <= 0 || iterations_ > 1'000'000) { + error = "benchmark_hip_sgemm.iterations must be 1..1000000"; + return false; + } + const int resource_percentage = + request.at("resource_percentage").get(); + if (resource_percentage <= 0 || resource_percentage > 100) { + error = "resource_percentage must be 1..100"; + return false; + } + const int cu_count = std::clamp( + (total_cus_ * resource_percentage + 99) / 100, + 1, total_cus_); + if (!ensure_resources(cu_count, error)) return false; + + const float alpha = 1.0F; + const float beta = 0.0F; + hipError_t status = hipEventRecord(started_, stream_); + if (status != hipSuccess) { + error = hip_error("hipEventRecord(started)", status); + return false; + } + for (int iteration = 0; iteration < iterations_; ++iteration) { + const hipblasStatus_t blas_status = hipblasSgemm( + handle_, HIPBLAS_OP_N, HIPBLAS_OP_N, + matrix_size_, matrix_size_, matrix_size_, + &alpha, a_, matrix_size_, b_, matrix_size_, + &beta, c_, matrix_size_); + if (blas_status != HIPBLAS_STATUS_SUCCESS) { + error = hipblas_error("hipblasSgemm", blas_status); + (void) hipStreamSynchronize(stream_); + return false; + } + } + status = hipEventRecord(finished_, stream_); + if (status != hipSuccess) { + error = hip_error("hipEventRecord(finished)", status); + (void) hipStreamSynchronize(stream_); + return false; + } + active_ = true; + error.clear(); + return true; + } catch (const std::exception & exception) { + error = std::string("invalid HIP probe request: ") + exception.what(); + return false; + } + } + + bool collect(int timeout_ms, + size_t max_result_bytes, + json & result, + double & wait_ms, + std::string & error) { + ScopedHipDevice device(device_); + if (!device.ok()) { + error = hip_error("hipSetDevice(tool)", device.status()); + release_active(); + return false; + } + const auto wait_started = std::chrono::steady_clock::now(); + const auto deadline = wait_started + + std::chrono::milliseconds(std::max(1, timeout_ms)); + while (true) { + const hipError_t status = hipEventQuery(finished_); + if (status == hipSuccess) break; + if (status != hipErrorNotReady) { + error = hip_error("hipEventQuery", status); + release_active(); + return false; + } + if (std::chrono::steady_clock::now() >= deadline) { + error = "executor_timeout"; + synchronize_and_release(); + wait_ms = std::chrono::duration( + std::chrono::steady_clock::now() - wait_started).count(); + return false; + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + + float gpu_ms = 0.0F; + hipError_t status = hipEventElapsedTime(&gpu_ms, started_, finished_); + if (status != hipSuccess) { + error = hip_error("hipEventElapsedTime", status); + release_active(); + return false; + } + float sample = 0.0F; + status = hipMemcpyAsync( + &sample, c_, sizeof(sample), hipMemcpyDeviceToHost, stream_); + if (status == hipSuccess) status = hipStreamSynchronize(stream_); + if (status != hipSuccess || !std::isfinite(sample)) { + error = status == hipSuccess + ? "HIP probe produced a non-finite sample" + : hip_error("HIP probe result copy", status); + release_active(); + return false; + } + result = { + {"sample", sample}, + {"gpu_ms", gpu_ms}, + {"iterations", iterations_}, + {"matrix_size", matrix_size_}, + {"cu_count", current_cu_count_}, + }; + if (result.dump().size() > max_result_bytes) { + error = "executor_result_too_large"; + release_active(); + return false; + } + wait_ms = std::chrono::duration( + std::chrono::steady_clock::now() - wait_started).count(); + release_active(); + error.clear(); + return true; + } + + void synchronize_and_release() { + ScopedHipDevice device(device_); + std::lock_guard lock(mutex_); + if (device.ok() && active_ && stream_) { + (void) hipStreamSynchronize(stream_); + } + active_ = false; + } + +private: + bool ensure_resources(int cu_count, std::string & error) { + hipError_t status = hipSuccess; + if (!stream_ || !handle_ || current_cu_count_ != cu_count) { + if (stream_) { + (void) hipStreamSynchronize(stream_); + if (handle_) { + (void) hipblasDestroy(handle_); + handle_ = nullptr; + } + (void) hipStreamDestroy(stream_); + stream_ = nullptr; + } + const size_t mask_words = + static_cast((total_cus_ + 31) / 32); + std::vector mask(mask_words, 0); + for (int cu = 0; cu < cu_count; ++cu) { + mask[static_cast(cu / 32)] |= + uint32_t{1} << (cu % 32); + } + status = hipExtStreamCreateWithCUMask( + &stream_, static_cast(mask.size()), mask.data()); + if (status != hipSuccess) { + error = hip_error("hipExtStreamCreateWithCUMask", status); + return false; + } + const hipblasStatus_t create_status = hipblasCreate(&handle_); + if (create_status != HIPBLAS_STATUS_SUCCESS) { + error = hipblas_error("hipblasCreate", create_status); + return false; + } + const hipblasStatus_t stream_status = + hipblasSetStream(handle_, stream_); + if (stream_status != HIPBLAS_STATUS_SUCCESS) { + error = hipblas_error("hipblasSetStream", stream_status); + return false; + } + current_cu_count_ = cu_count; + } + if (!a_ || !b_ || !c_) { + if (c_) (void) hipFree(c_); + if (b_) (void) hipFree(b_); + if (a_) (void) hipFree(a_); + a_ = nullptr; + b_ = nullptr; + c_ = nullptr; + const size_t elements = + static_cast(matrix_size_) * matrix_size_; + const size_t bytes = elements * sizeof(float); + if ((status = hipMalloc(&a_, bytes)) != hipSuccess || + (status = hipMalloc(&b_, bytes)) != hipSuccess || + (status = hipMalloc(&c_, bytes)) != hipSuccess) { + error = hip_error("hipMalloc", status); + if (c_) (void) hipFree(c_); + if (b_) (void) hipFree(b_); + if (a_) (void) hipFree(a_); + a_ = nullptr; + b_ = nullptr; + c_ = nullptr; + return false; + } + if ((status = hipMemsetAsync(a_, 0x01, bytes, stream_)) != hipSuccess || + (status = hipMemsetAsync(b_, 0x02, bytes, stream_)) != hipSuccess || + (status = hipMemsetAsync(c_, 0, bytes, stream_)) != hipSuccess) { + error = hip_error("hipMemsetAsync", status); + return false; + } + const float alpha = 1.0F; + const float beta = 0.0F; + const hipblasStatus_t warm_status = hipblasSgemm( + handle_, HIPBLAS_OP_N, HIPBLAS_OP_N, + matrix_size_, matrix_size_, matrix_size_, + &alpha, a_, matrix_size_, b_, matrix_size_, + &beta, c_, matrix_size_); + if (warm_status != HIPBLAS_STATUS_SUCCESS) { + error = hipblas_error("hipblasSgemm(warmup)", warm_status); + return false; + } + if ((status = hipStreamSynchronize(stream_)) != hipSuccess) { + error = hip_error("hipStreamSynchronize(warmup)", status); + return false; + } + } + if (!started_ && + (status = hipEventCreate(&started_)) != hipSuccess) { + error = hip_error("hipEventCreate(started)", status); + return false; + } + if (!finished_ && + (status = hipEventCreate(&finished_)) != hipSuccess) { + error = hip_error("hipEventCreate(finished)", status); + return false; + } + return true; + } + + void release_active() { + std::lock_guard lock(mutex_); + active_ = false; + } + + std::mutex mutex_; + int device_ = 0; + int matrix_size_ = 0; + int total_cus_ = 0; + int current_cu_count_ = 0; + int iterations_ = 0; + bool active_ = false; + hipStream_t stream_ = nullptr; + hipblasHandle_t handle_ = nullptr; + hipEvent_t started_ = nullptr; + hipEvent_t finished_ = nullptr; + float * a_ = nullptr; + float * b_ = nullptr; + float * c_ = nullptr; +}; + +HipSgemmExecution::~HipSgemmExecution() { + if (!finished_) terminate(false); +} + +bool HipSgemmExecution::send_control(const std::string & operation) { + if (operation == "commit") { + committed_ = true; + return true; + } + if (operation == "cancel") { + committed_ = false; + return true; + } + return false; +} + +bool HipSgemmExecution::collect_result( + int timeout_ms, + size_t max_result_bytes, + json & result, + double & wait_ms, + std::string & error) { + if (finished_) { + error = "executor already collected"; + return false; + } + if (!committed_) { + error = "executor result requested before commit"; + terminate(false); + return false; + } + finished_ = true; + return state_->collect( + timeout_ms, max_result_bytes, result, wait_ms, error); +} + +void HipSgemmExecution::terminate(bool allow_control_grace) { + (void) allow_control_grace; + if (finished_) return; + finished_ = true; + state_->synchronize_and_release(); +} + +class HipSgemmExecutor final : public ToolSpeculationExecutor { +public: + explicit HipSgemmExecutor(std::shared_ptr state) + : state_(std::move(state)) {} + + std::unique_ptr start( + const json & request, std::string & error) override { + if (!state_->start(request, error)) return nullptr; + return std::make_unique(state_); + } + + const char * mode_name() const override { + return "in_process_hip_cu_mask"; + } + +private: + std::shared_ptr state_; +}; + +} // namespace + +std::shared_ptr +create_hip_sgemm_tool_speculation_executor( + int device, + int matrix_size, + int & total_compute_units, + std::string & error) { + total_compute_units = 0; + if (device < 0 || matrix_size <= 0 || matrix_size > 8192) { + error = "HIP probe needs DEVICE >= 0 and MATRIX_SIZE in 1..8192"; + return nullptr; + } + hipDeviceProp_t properties{}; + const hipError_t status = hipGetDeviceProperties(&properties, device); + if (status != hipSuccess) { + error = hip_error("hipGetDeviceProperties", status); + return nullptr; + } + if (properties.multiProcessorCount <= 0) { + error = "HIP device reports no compute units"; + return nullptr; + } + total_compute_units = properties.multiProcessorCount; + error.clear(); + auto state = std::make_shared( + device, matrix_size, properties.multiProcessorCount); + return std::make_shared(std::move(state)); +} + +} // namespace dflash::common diff --git a/server/src/server/tool_speculation_hip_probe.h b/server/src/server/tool_speculation_hip_probe.h new file mode 100644 index 000000000..673376521 --- /dev/null +++ b/server/src/server/tool_speculation_hip_probe.h @@ -0,0 +1,21 @@ +#pragma once + +#include "tool_speculation.h" + +#include +#include + +namespace dflash::common { + +// Benchmark-only trusted executor used to qualify same-process HIP sharing. +// It accepts the allowlisted `benchmark_hip_sgemm` tool with +// {"iterations": N}. The matrix size is fixed at startup so allocation and +// warmup stay outside measured requests. +std::shared_ptr +create_hip_sgemm_tool_speculation_executor( + int device, + int matrix_size, + int & total_compute_units, + std::string & error); + +} // namespace dflash::common diff --git a/server/test/smoke_qwen3_tool_predictor_ipc.cpp b/server/test/smoke_qwen3_tool_predictor_ipc.cpp new file mode 100644 index 000000000..866bbaf96 --- /dev/null +++ b/server/test/smoke_qwen3_tool_predictor_ipc.cpp @@ -0,0 +1,191 @@ +#include "server/native_semantic_tool_predictor.h" + +#include +#include +#include +#include +#include + +using namespace dflash::common; + +namespace { + +struct Case { + const char * id; + const char * prompt; + const char * expected_name; + ordered_json expected_arguments; +}; + +json production_tools() { + return json::parse(R"json( +[ + {"type":"function","function":{"name":"get_weather","description":"Get current weather for one city.","parameters":{"type":"object","properties":{"city":{"type":"string"},"unit":{"type":"string","enum":["celsius","fahrenheit"]}},"required":["city","unit"],"additionalProperties":false}}}, + {"type":"function","function":{"name":"get_stock_quote","description":"Get the latest market quote for a ticker symbol.","parameters":{"type":"object","properties":{"symbol":{"type":"string"}},"required":["symbol"],"additionalProperties":false}}}, + {"type":"function","function":{"name":"search_documents","description":"Search indexed documents.","parameters":{"type":"object","properties":{"query":{"type":"string"},"limit":{"type":"integer","minimum":1,"maximum":20}},"required":["query","limit"],"additionalProperties":false}}}, + {"type":"function","function":{"name":"calculate","description":"Evaluate one arithmetic expression.","parameters":{"type":"object","properties":{"expression":{"type":"string"}},"required":["expression"],"additionalProperties":false}}}, + {"type":"function","function":{"name":"lookup_order","description":"Look up an order by its identifier.","parameters":{"type":"object","properties":{"order_id":{"type":"string"}},"required":["order_id"],"additionalProperties":false}}}, + {"type":"function","function":{"name":"translate_text","description":"Translate text to a target language.","parameters":{"type":"object","properties":{"text":{"type":"string"},"target_language":{"type":"string"}},"required":["text","target_language"],"additionalProperties":false}}}, + {"type":"function","function":{"name":"plan_route","description":"Plan a route between two places.","parameters":{"type":"object","properties":{"origin":{"type":"string"},"destination":{"type":"string"},"mode":{"type":"string","enum":["car","walk","transit"]}},"required":["origin","destination","mode"],"additionalProperties":false}}}, + {"type":"function","function":{"name":"read_file","description":"Read a UTF-8 text file from the workspace.","parameters":{"type":"object","properties":{"path":{"type":"string"}},"required":["path"],"additionalProperties":false}}} +] +)json"); +} + +json make_request(const std::string & prompt, const json & tools, + int32_t max_tokens) { + return { + {"model", "native-qwen3"}, + {"messages", json::array({{ + {"role", "user"}, + {"content", prompt}, + }})}, + {"tools", tools}, + {"tool_choice", "required"}, + {"temperature", 0}, + {"max_tokens", max_tokens}, + }; +} + +std::vector production_cases() { + return { + {"weather_rome", "What is the weather in Rome? Use Celsius.", + "get_weather", {{"city", "Rome"}, {"unit", "celsius"}}}, + {"weather_boston", "Check Boston weather in Fahrenheit.", + "get_weather", {{"city", "Boston"}, {"unit", "fahrenheit"}}}, + {"stock_nvda", "Get the latest quote for NVDA.", + "get_stock_quote", {{"symbol", "NVDA"}}}, + {"stock_amd", "Look up AMD's current stock quote.", + "get_stock_quote", {{"symbol", "AMD"}}}, + {"search_rocm", + "Search documents for 'ROCm graph replay' and return at most 5 results.", + "search_documents", {{"query", "ROCm graph replay"}, {"limit", 5}}}, + {"search_tool", + "Find the top 3 documents about speculative tool execution.", + "search_documents", + {{"query", "speculative tool execution"}, {"limit", 3}}}, + {"calculate", "Calculate (73.5 * 4) / 7.", + "calculate", {{"expression", "(73.5 * 4) / 7"}}}, + {"order", "Look up order LBX-2048-A.", + "lookup_order", {{"order_id", "LBX-2048-A"}}}, + {"translate", "Translate 'the server is ready' to Italian.", + "translate_text", + {{"text", "the server is ready"}, {"target_language", "Italian"}}}, + {"route", + "Plan a walking route from Termini Station to the Colosseum.", + "plan_route", + {{"origin", "Termini Station"}, + {"destination", "the Colosseum"}, + {"mode", "walk"}}}, + {"read_file", "Read the file docs/production.md.", + "read_file", {{"path", "docs/production.md"}}}, + {"punctuation", + "Search for the exact phrase 'R9700 + Strix: 0731/DS4' with limit 4.", + "search_documents", + {{"query", "R9700 + Strix: 0731/DS4"}, {"limit", 4}}}, + }; +} + +bool arguments_equal(const ordered_json & left, const ordered_json & right) { + // Object key order is irrelevant to tool-call semantics. Convert both + // ordered objects to the canonical map-backed representation first. + return json::parse(left.dump()) == json::parse(right.dump()); +} + +void print_prediction(const char * id, const SemanticToolPrediction & prediction, + const char * expected_name, + const ordered_json & expected_arguments, + const std::string & generated) { + const json output = { + {"id", id}, + {"ok", prediction.ok}, + {"error", prediction.error}, + {"wall_ms", prediction.wall_ms}, + {"name", prediction.call.name}, + {"arguments", prediction.call.arguments}, + {"generated", generated}, + {"name_match", prediction.ok && prediction.call.name == expected_name}, + {"exact_match", prediction.ok && prediction.call.name == expected_name && + arguments_equal(prediction.call.arguments, + expected_arguments)}, + }; + std::printf("%s\n", output.dump().c_str()); + std::fflush(stdout); +} + +} // namespace + +int main(int argc, char ** argv) { + if (argc < 4) { + std::fprintf(stderr, + "usage: %s [prompt]\n", + argv[0]); + return 2; + } + + SemanticToolPredictorConfig config; + config.native_model_path = argv[1]; + config.native_ipc_bin = argv[2]; + config.native_gpu = std::atoi(argv[3]); + config.native_max_ctx = 4096; + config.max_tokens = 96; + + std::string error; + auto predictor = NativeSemanticToolPredictor::create(config, error); + if (!predictor) { + std::fprintf(stderr, "predictor start failed: %s\n", error.c_str()); + return 1; + } + + const json tools = production_tools(); + if (argc > 4) { + const std::string prompt = argv[4]; + std::string generated; + const auto prediction = predictor->predict( + make_request(prompt, tools, config.max_tokens), tools, &generated); + print_prediction("custom", prediction, "", ordered_json::object(), + generated); + return prediction.ok ? 0 : 1; + } + + const std::vector cases = production_cases(); + size_t valid = 0; + size_t name_matches = 0; + size_t exact_matches = 0; + std::vector walls; + for (const Case & test_case : cases) { + std::string generated; + const auto prediction = predictor->predict( + make_request(test_case.prompt, tools, config.max_tokens), tools, + &generated); + print_prediction(test_case.id, prediction, test_case.expected_name, + test_case.expected_arguments, generated); + valid += prediction.ok ? 1 : 0; + name_matches += prediction.ok && + prediction.call.name == test_case.expected_name ? 1 : 0; + exact_matches += prediction.ok && + prediction.call.name == test_case.expected_name && + arguments_equal(prediction.call.arguments, + test_case.expected_arguments) + ? 1 : 0; + walls.push_back(prediction.wall_ms); + } + std::sort(walls.begin(), walls.end()); + const double wall_p50 = walls.empty() + ? 0.0 : 0.5 * (walls[(walls.size() - 1) / 2] + walls[walls.size() / 2]); + const json summary = { + {"requests", cases.size()}, + {"valid", valid}, + {"name_matches", name_matches}, + {"exact_matches", exact_matches}, + {"name_accuracy", cases.empty() ? 0.0 + : static_cast(name_matches) / + static_cast(cases.size())}, + {"exact_accuracy", cases.empty() ? 0.0 + : static_cast(exact_matches) / + static_cast(cases.size())}, + {"wall_p50_ms", wall_p50}, + }; + std::printf("%s\n", summary.dump().c_str()); + return valid == cases.size() && name_matches == cases.size() ? 0 : 1; +} diff --git a/server/test/test_semantic_tool_hint.cpp b/server/test/test_semantic_tool_hint.cpp new file mode 100644 index 000000000..6fa96b792 --- /dev/null +++ b/server/test/test_semantic_tool_hint.cpp @@ -0,0 +1,248 @@ +#include "CppUnitTestFramework.hpp" + +#include "server/semantic_tool_hint.h" + +#include + +namespace { +struct SemanticToolHintFixture {}; +} + +using namespace dflash::common; + +static json weather_tools() { + return json::array({{ + {"type", "function"}, + {"function", { + {"name", "get_weather"}, + {"parameters", { + {"type", "object"}, + {"properties", { + {"city", {{"type", "string"}}}, + {"unit", {{"type", "string"}}}, + }}, + }}, + }}, + }}); +} + +TEST_CASE(SemanticToolHintFixture, parses_qwen_openai_tool_call_semantics) { + const json response = { + {"choices", json::array({{ + {"message", { + {"role", "assistant"}, + {"content", ""}, + {"tool_calls", json::array({{ + {"type", "function"}, + {"function", { + {"name", "get_weather"}, + {"arguments", "{\"city\":\"Rome\",\"unit\":\"celsius\"}"}, + }}, + }})}, + }}, + }})}, + }; + SemanticToolCall call; + std::string error; + CHECK(parse_semantic_tool_prediction( + response, weather_tools(), call, error)); + CHECK(error.empty()); + CHECK(call.name == "get_weather"); + CHECK(call.arguments.dump() == + "{\"city\":\"Rome\",\"unit\":\"celsius\"}"); +} + +TEST_CASE(SemanticToolHintFixture, rejects_unknown_predicted_function) { + const json response = { + {"choices", json::array({{ + {"message", { + {"tool_calls", json::array({{ + {"function", { + {"name", "delete_everything"}, + {"arguments", "{}"}, + }}, + }})}, + }}, + }})}, + }; + SemanticToolCall call; + std::string error; + CHECK(!parse_semantic_tool_prediction( + response, weather_tools(), call, error)); + CHECK(error == "predictor_selected_unknown_function"); +} + +TEST_CASE(SemanticToolHintFixture, materializes_declared_optional_defaults) { + json tools = weather_tools(); + tools[0]["function"]["parameters"]["properties"]["unit"]["default"] = + "celsius"; + SemanticToolCall call; + call.name = "get_weather"; + call.arguments = ordered_json::parse(R"({"city":"Rome"})"); + std::string error; + + CHECK(materialize_declared_tool_defaults(tools, call, error)); + CHECK(error.empty()); + CHECK(call.arguments.dump() == + R"({"city":"Rome","unit":"celsius"})"); +} + +TEST_CASE(SemanticToolHintFixture, explicit_prediction_beats_schema_default) { + json tools = weather_tools(); + tools[0]["function"]["parameters"]["properties"]["unit"]["default"] = + "celsius"; + SemanticToolCall call; + call.name = "get_weather"; + call.arguments = ordered_json::parse( + R"({"city":"Rome","unit":"fahrenheit"})"); + std::string error; + + CHECK(materialize_declared_tool_defaults(tools, call, error)); + CHECK(call.arguments["unit"] == "fahrenheit"); +} + +TEST_CASE(SemanticToolHintFixture, predictor_request_forwards_only_semantics) { + const json target = { + {"model", "deepseek-v4-flash"}, + {"messages", json::array({{{"role", "user"}, {"content", "weather"}}})}, + {"tools", weather_tools()}, + {"tool_choice", "required"}, + {"tool_speculation", {{"name", "unsafe"}}}, + {"prefix_cache", {{"scope", "full"}}}, + }; + const json request = build_semantic_tool_predictor_request( + target, "Qwen3-0.6B", 32); + CHECK(request["model"] == "Qwen3-0.6B"); + CHECK(request["max_tokens"] == 32); + CHECK(request["tool_choice"] == "required"); + CHECK(!request.contains("tool_speculation")); + CHECK(!request.contains("prefix_cache")); +} + +TEST_CASE(SemanticToolHintFixture, native_predictor_config_is_independent_of_http) { + SemanticToolPredictorConfig config; + config.native_model_path = "/models/qwen3-0.6b.gguf"; + config.native_ipc_bin = "/opt/lucebox/backend_ipc_daemon"; + CHECK(config.native_enabled()); + CHECK(!config.http_enabled()); + CHECK(config.enabled()); + CHECK(config.native_runs_before_model()); + CHECK(std::string(native_tool_predictor_schedule_name( + config.native_schedule)) == "before-model"); +} + +TEST_CASE(SemanticToolHintFixture, native_predictor_overlap_is_explicit) { + NativeToolPredictorSchedule schedule = + NativeToolPredictorSchedule::BeforeModel; + CHECK(parse_native_tool_predictor_schedule("overlap", schedule)); + CHECK(schedule == NativeToolPredictorSchedule::Overlap); + CHECK(std::string(native_tool_predictor_schedule_name(schedule)) == + "overlap"); + CHECK(!parse_native_tool_predictor_schedule("automatic", schedule)); +} + +TEST_CASE(SemanticToolHintFixture, native_prompt_uses_qwen_tool_contract) { + const json request = { + {"messages", json::array({{ + {"role", "user"}, + {"content", "What is the weather in Rome?"}, + }})}, + {"tools", weather_tools()}, + {"tool_choice", "required"}, + }; + std::string error; + const std::string prompt = + build_native_semantic_tool_predictor_prompt(request, error); + CHECK(error.empty()); + CHECK(prompt.find("You must call exactly one available function.") != + std::string::npos); + CHECK(prompt.find("get_weather") != std::string::npos); + CHECK(prompt.find("What is the weather in Rome?") != std::string::npos); + CHECK(prompt.find("{\"name\": , \"arguments\":") != + std::string::npos); + CHECK(prompt.find("") == + std::string::npos); + CHECK(prompt.find("\n\n") != std::string::npos); +} + +TEST_CASE(SemanticToolHintFixture, parses_native_qwen_xml_semantics) { + const std::string generated = + "\n" + "\n" + "\nRome\n\n" + "\ncelsius\n\n" + "\n" + ""; + SemanticToolCall call; + std::string error; + CHECK(parse_native_semantic_tool_prediction( + generated, weather_tools(), call, error)); + CHECK(error.empty()); + CHECK(call.name == "get_weather"); + CHECK(call.arguments.dump() == + "{\"city\":\"Rome\",\"unit\":\"celsius\"}"); +} + +TEST_CASE(SemanticToolHintFixture, repairs_qwen_missing_outer_call_brace) { + const json tools = json::array({{ + {"type", "function"}, + {"function", { + {"name", "get_stock_quote"}, + {"parameters", { + {"type", "object"}, + {"properties", {{"symbol", {{"type", "string"}}}}}, + {"required", json::array({"symbol"})}, + }}, + }}, + }}); + const std::string generated = + "\n" + " \"name\": \"get_stock_quote\",\n" + " \"arguments\": {\"symbol\": \"NVDA\"}\n" + "\"}<|im_end|>"; + SemanticToolCall call; + std::string error; + CHECK(parse_native_semantic_tool_prediction( + generated, tools, call, error)); + CHECK(error.empty()); + CHECK(call.name == "get_stock_quote"); + CHECK(call.arguments.dump() == "{\"symbol\":\"NVDA\"}"); +} + +TEST_CASE(SemanticToolHintFixture, maps_bare_arguments_only_for_one_tool) { + const json one_tool = json::array({{ + {"name", "benchmark_cpu_sparse"}, + {"parameters", { + {"type", "object"}, + {"properties", {{"iterations", {{"type", "integer"}}}}}, + {"required", json::array({"iterations"})}, + }}, + }}); + SemanticToolCall call; + std::string error; + CHECK(parse_native_semantic_tool_prediction( + "\n{\"iterations\":172452}\n", + one_tool, call, error)); + CHECK(error.empty()); + CHECK(call.name == "benchmark_cpu_sparse"); + CHECK(call.arguments.dump() == "{\"iterations\":172452}"); + + json two_tools = one_tool; + two_tools.push_back({ + {"name", "other"}, + {"parameters", {{"type", "object"}}}, + }); + CHECK(!parse_native_semantic_tool_prediction( + "{\"iterations\":172452}", two_tools, call, error)); +} + +TEST_CASE(SemanticToolHintFixture, native_parser_rejects_multiple_calls) { + const std::string call = + "Rome" + "celsius"; + SemanticToolCall prediction; + std::string error; + CHECK(!parse_native_semantic_tool_prediction( + call + call, weather_tools(), prediction, error)); + CHECK(error == "native_predictor_response_has_multiple_calls"); +} diff --git a/server/test/test_server_unit.cpp b/server/test/test_server_unit.cpp index 8373859dc..b80833c6e 100644 --- a/server/test/test_server_unit.cpp +++ b/server/test/test_server_unit.cpp @@ -2486,6 +2486,11 @@ TEST_CASE(ServerUnitFixture, test_pflash_raw_body_preserved) { TEST_ASSERT(req.raw_body["temperature"].get() > 0.6f); } +TEST_CASE(ServerUnitFixture, test_tool_speculation_defaults_to_automatic_prediction) { + ParsedRequest req; + TEST_ASSERT(req.automatic_tool_speculation_enabled); +} + TEST_CASE(ServerUnitFixture, test_parse_request_sampler_applies_defaults_and_overrides) { SamplingDefaults defaults; defaults.has_temperature = true; @@ -2743,6 +2748,43 @@ TEST_CASE(ServerUnitFixture, test_deepseek4_render_empty_chat_gen_prompt) { TEST_ASSERT(out == expected); } +TEST_CASE(ServerUnitFixture, test_deepseek4_render_required_tool_instructions) { + std::vector msgs = { + {"user", "What is the weather?", ""}, + }; + const std::string tools = + R"([{"type":"function","function":{"name":"weather.get","parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}}])"; + const std::string out = render_chat_template( + msgs, ChatFormat::DEEPSEEK4, + /*add_generation_prompt=*/true, + /*enable_thinking=*/false, + tools, + /*tool_call_required=*/true); + + TEST_ASSERT(out.find("\n") != std::string::npos); + TEST_ASSERT(out.find("\"name\":\"weather.get\"") != std::string::npos); + TEST_ASSERT(out.find("") != std::string::npos); + TEST_ASSERT(out.find("") != std::string::npos); + TEST_ASSERT(out.find("MUST call exactly one") != std::string::npos); + TEST_ASSERT(out.find("<｜User｜>What is the weather?") != + std::string::npos); + const std::string suffix = "<｜Assistant｜>"; + TEST_ASSERT(out.size() >= suffix.size()); + TEST_ASSERT(out.compare(out.size() - suffix.size(), suffix.size(), suffix) == 0); +} + +TEST_CASE(ServerUnitFixture, test_deepseek4_auto_tool_is_not_forced) { + std::vector msgs = {{"user", "Hello", ""}}; + const std::string tools = + R"([{"type":"function","function":{"name":"weather.get","parameters":{"type":"object","properties":{}}}}])"; + const std::string out = render_chat_template( + msgs, ChatFormat::DEEPSEEK4, true, false, tools, + /*tool_call_required=*/false); + + TEST_ASSERT(out.find("You may call functions") != std::string::npos); + TEST_ASSERT(out.find("MUST call exactly one") == std::string::npos); +} + TEST_CASE(ServerUnitFixture, test_jinja_render_basic) { std::vector msgs = { {"system", "you are helpful", ""}, @@ -4836,6 +4878,149 @@ TEST_CASE(ServerUnitFixture, test_props_budget_envelope_shape) { TEST_ASSERT(body["server"]["props_schema"].get() == 2); } +TEST_CASE(ServerUnitFixture, test_props_tool_speculation_shape) { + ServerConfig cfg; + Tokenizer tok; + PrefixCache pc(0, tok); + ToolMemory tm; + + json body = build_props_body(cfg, pc, tm); + TEST_ASSERT(body.contains("tool_speculation")); + const json & disabled = body["tool_speculation"]; + TEST_ASSERT(!disabled["enabled"].get()); + TEST_ASSERT(!disabled["automatic_prediction_enabled"].get()); + TEST_ASSERT(disabled["prediction_source"].is_null()); + TEST_ASSERT(disabled["prediction_confidence"].is_null()); + TEST_ASSERT(disabled["profile_status"].is_null()); + TEST_ASSERT(disabled["executor_contract"].is_null()); + TEST_ASSERT(disabled["protocol"].get() == + "dflash.tool-speculation.v1"); + TEST_ASSERT(disabled["requires_client_support"].get()); + TEST_ASSERT(disabled["preserves_token_speculation"].get()); + TEST_ASSERT(disabled["unqualified_lane_policy"].get() == + "defer"); + TEST_ASSERT(disabled["allowed_tools"].empty()); + TEST_ASSERT(disabled["model_routing_static"].get()); + TEST_ASSERT(disabled["model_expert_ownership_unique"].get()); + TEST_ASSERT(disabled["compute_isolation"].get() == "none"); + TEST_ASSERT(!disabled["cpu_affinity_isolated"].get()); + TEST_ASSERT(disabled["tool_cpu_affinity"].empty()); + TEST_ASSERT(disabled["model_cpu_affinity"].empty()); + TEST_ASSERT(disabled["hip_tool_device"].is_null()); + TEST_ASSERT(disabled["hip_reserved_tool_compute_units"].get() == 0); + TEST_ASSERT(disabled["profile_lanes"].empty()); + + cfg.tool_speculation.executor_path = "/trusted/tool-adapter"; + cfg.tool_speculation.profile_path = "/measured/frontier.json"; + cfg.tool_speculation.allowed_tools = {"lookup"}; + std::string profile_error; + TEST_ASSERT(cfg.tool_speculation.policy.load_json(json{ + {"path_summary", { + {"25", { + {"decode_interference_qualified", true}, + {"hit", { + {"control_task_mean_ms", 100.0}, + {"speculative_task_mean_ms", 80.0}, + {"model_slowdown_percent", 2.0}, + }}, + {"miss", { + {"control_task_mean_ms", 100.0}, + {"speculative_task_mean_ms", 101.0}, + {"model_slowdown_percent", 2.0}, + }}, + }}, + }}, + }, profile_error)); + + body = build_props_body(cfg, pc, tm); + const json & enabled = body["tool_speculation"]; + TEST_ASSERT(enabled["enabled"].get()); + TEST_ASSERT(!enabled["automatic_prediction_enabled"].get()); + TEST_ASSERT(!enabled["predictor_decode_isolated"].get()); + TEST_ASSERT(enabled["profile_status"].get() == "qualified"); + TEST_ASSERT(enabled["allowed_tools"] == json::array({"lookup"})); + TEST_ASSERT(enabled["preserves_token_speculation"].get()); + TEST_ASSERT(enabled["unqualified_lane_policy"].get() == + "defer"); + TEST_ASSERT(enabled["profile_lanes"].size() == 1); + TEST_ASSERT(enabled["profile_lanes"][0] + ["resource_percentage"].get() == 25); + TEST_ASSERT(std::fabs(enabled["profile_lanes"][0] + ["model_slowdown_ratio"].get() - 1.02) < + 1e-9); + TEST_ASSERT(enabled["profile_lanes"][0] + ["decode_interference_qualified"].get()); + TEST_ASSERT(enabled["profile_lanes"][0] + ["accelerator_relation"].get() == + "unspecified"); + TEST_ASSERT(!enabled["profile_lanes"][0] + ["requires_static_model_routing"].get()); + TEST_ASSERT(!enabled["profile_lanes"][0] + ["requires_unique_expert_ownership"].get()); + + cfg.semantic_tool_predictor.native_model_path = "/models/qwen3-0.6b.gguf"; + cfg.semantic_tool_predictor.native_ipc_bin = "/bin/backend-ipc"; + body = build_props_body(cfg, pc, tm); + const json & automatic = body["tool_speculation"]; + TEST_ASSERT(automatic["automatic_prediction_enabled"].get()); + TEST_ASSERT(!automatic["requires_client_support"].get()); + TEST_ASSERT(automatic["prediction_source"].get() == + "native-qwen3"); + TEST_ASSERT(std::fabs( + automatic["prediction_confidence"].get() - 0.75) < 1e-9); + TEST_ASSERT(automatic["predictor_schedule"].get() == + "before-model"); + TEST_ASSERT(automatic["predictor_decode_isolated"].get()); + + cfg.semantic_tool_predictor.native_schedule = + NativeToolPredictorSchedule::Overlap; + body = build_props_body(cfg, pc, tm); + const json & overlapping = body["tool_speculation"]; + TEST_ASSERT(overlapping["predictor_schedule"].get() == + "overlap"); + TEST_ASSERT(!overlapping["predictor_decode_isolated"].get()); + cfg.semantic_tool_predictor.native_schedule = + NativeToolPredictorSchedule::BeforeModel; + + cfg.semantic_tool_predictor.native_model_path.clear(); + cfg.semantic_tool_predictor.native_ipc_bin.clear(); + cfg.semantic_tool_predictor.url = "http://127.0.0.1:9000/v1/chat/completions"; + cfg.semantic_tool_predictor.model = "remote-predictor"; + body = build_props_body(cfg, pc, tm); + const json & remote = body["tool_speculation"]; + TEST_ASSERT(remote["automatic_prediction_enabled"].get()); + TEST_ASSERT(remote["prediction_source"].get() == + "remote-predictor"); + TEST_ASSERT(!remote["predictor_decode_isolated"].get()); + cfg.semantic_tool_predictor.url.clear(); + cfg.semantic_tool_predictor.model.clear(); + cfg.semantic_tool_predictor.native_model_path = "/models/qwen3-0.6b.gguf"; + cfg.semantic_tool_predictor.native_ipc_bin = "/bin/backend-ipc"; + + cfg.tool_speculation.cpu_affinity = {14, 30}; + cfg.tool_speculation.model_cpu_affinity = {0, 1, 2, 3}; + cfg.tool_speculation.cpu_affinity_isolated = true; + body = build_props_body(cfg, pc, tm); + const json & cpu_isolated = body["tool_speculation"]; + TEST_ASSERT(cpu_isolated["compute_isolation"].get() == + "disjoint_cpu_affinity"); + TEST_ASSERT(cpu_isolated["cpu_affinity_isolated"].get()); + TEST_ASSERT(cpu_isolated["tool_cpu_affinity"] == + json::array({14, 30})); + TEST_ASSERT(cpu_isolated["model_cpu_affinity"] == + json::array({0, 1, 2, 3})); + + cfg.tool_speculation.cpu_affinity_isolated = false; + cfg.tool_speculation.hip_tool_device = 1; + cfg.tool_speculation.hip_reserved_tool_compute_units = 1; + body = build_props_body(cfg, pc, tm); + const json & isolated = body["tool_speculation"]; + TEST_ASSERT(isolated["compute_isolation"].get() == + "disjoint_hip_cu_masks"); + TEST_ASSERT(isolated["hip_tool_device"].get() == 1); + TEST_ASSERT(isolated["hip_reserved_tool_compute_units"].get() == 1); +} + // ─── /props.runtime captures full config (§4.16) ────────────────────── // Snapshot/bench tooling reads /props.runtime wholesale into // result.json.server_info; this test pins the field set so additions @@ -5074,6 +5259,40 @@ TEST_CASE(ServerUnitFixture, test_model_backend_retries_empty_spec_restore_once_ TEST_ASSERT(backend.restore_saw_force_ar); } +TEST_CASE(ServerUnitFixture, test_model_backend_can_forbid_ar_retry) { + EmptySpecRetryBackend backend; + GenerateRequest req; + req.prompt = {1, 2, 3}; + req.n_gen = 4; + req.allow_decode_mode_retry = false; + DaemonIO io; + + GenerateResult result = backend.generate(req, io); + + TEST_ASSERT(result.ok()); + TEST_ASSERT(result.tokens.empty()); + TEST_ASSERT(result.spec_decode_ran); + TEST_ASSERT(backend.generate_calls == 1); + TEST_ASSERT(!backend.generate_saw_force_ar); +} + +TEST_CASE(ServerUnitFixture, test_model_backend_restore_can_forbid_ar_retry) { + EmptySpecRetryBackend backend; + GenerateRequest req; + req.prompt = {1, 2, 3}; + req.n_gen = 4; + req.allow_decode_mode_retry = false; + DaemonIO io; + + GenerateResult result = backend.restore_and_generate(7, req, io); + + TEST_ASSERT(result.ok()); + TEST_ASSERT(result.tokens.empty()); + TEST_ASSERT(result.spec_decode_ran); + TEST_ASSERT(backend.restore_calls == 1); + TEST_ASSERT(!backend.restore_saw_force_ar); +} + TEST_CASE(ServerUnitFixture, test_model_backend_retries_empty_visible_spec_generate_once_with_ar) { EmptySpecRetryBackend backend; backend.generate_first_empty_visible = true; diff --git a/server/test/test_tool_speculation.cpp b/server/test/test_tool_speculation.cpp new file mode 100644 index 000000000..e9686e611 --- /dev/null +++ b/server/test/test_tool_speculation.cpp @@ -0,0 +1,630 @@ +#include "CppUnitTestFramework.hpp" +#include "server/tool_speculation.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#if !defined(_WIN32) +# include +# include +# if defined(__linux__) +# include +# endif +#endif + +using dflash::common::ApiFormat; +using dflash::common::CanonicalToolInvocation; +using dflash::common::ToolCall; +using dflash::common::ToolSpeculationAttempt; +using dflash::common::ToolSpeculationConfig; +using dflash::common::ToolSpeculationExecution; +using dflash::common::ToolSpeculationExecutor; +using dflash::common::ToolSpeculationPolicy; +using dflash::common::ToolSpeculationPrediction; +using dflash::common::build_tool_speculation_prediction; +using dflash::common::json; +using dflash::common::parse_tool_speculation_prediction; +using dflash::common::parse_tool_speculation_cpu_affinity; +using dflash::common::qualify_tool_speculation_cpu_affinity; +using dflash::common::render_tool_speculation_sse; + +namespace { +struct ToolSpeculationFixture {}; + +struct FakeExecutionState { + json request; + std::vector controls; + bool terminated = false; + bool collected = false; +}; + +class FakeExecution final : public ToolSpeculationExecution { +public: + explicit FakeExecution(std::shared_ptr state) + : state_(std::move(state)) {} + + bool send_control(const std::string & operation) override { + state_->controls.push_back(operation); + return operation == "commit" || operation == "cancel"; + } + + bool collect_result(int timeout_ms, + size_t max_result_bytes, + json & result, + double & wait_ms, + std::string & error) override { + (void) timeout_ms; + state_->collected = true; + result = {{"value", 42}}; + wait_ms = 0.0; + if (result.dump().size() > max_result_bytes) { + error = "executor_result_too_large"; + return false; + } + error.clear(); + return true; + } + + void terminate(bool allow_control_grace) override { + (void) allow_control_grace; + state_->terminated = true; + } + +private: + std::shared_ptr state_; +}; + +class FakeExecutor final : public ToolSpeculationExecutor { +public: + explicit FakeExecutor(std::shared_ptr state) + : state_(std::move(state)) {} + + std::unique_ptr start( + const json & request, std::string & error) override { + state_->request = request; + error.clear(); + return std::make_unique(state_); + } + + const char * mode_name() const override { + return "fake_in_process"; + } + +private: + std::shared_ptr state_; +}; + +json policy_fixture(bool decode_interference_qualified = true) { + auto path = [decode_interference_qualified]( + double hit_task, double miss_task, + double slowdown_percent) { + return json{ + {"decode_interference_qualified", + decode_interference_qualified}, + {"hit", { + {"control_task_mean_ms", 100.0}, + {"speculative_task_mean_ms", hit_task}, + {"model_slowdown_percent", slowdown_percent}, + }}, + {"miss", { + {"control_task_mean_ms", 100.0}, + {"speculative_task_mean_ms", miss_task}, + {"model_slowdown_percent", slowdown_percent}, + }}, + }; + }; + json fixture = { + {"path_summary", { + {"25", path(80.0, 101.0, 2.0)}, + {"50", path(60.0, 110.0, 7.0)}, + {"100", path(50.0, 130.0, 16.0)}, + }}, + }; + return fixture; +} + +ToolSpeculationConfig test_config(const std::string & executor = {}) { + ToolSpeculationConfig config; + config.executor_path = executor.empty() ? "/unused/executor" : executor; + config.profile_path = "fixture.json"; + config.allowed_tools = {"lookup"}; + config.timeout_ms = 1000; + config.cancel_grace_ms = 20; + config.max_model_slowdown_ratio = 1.20; + std::string error; + if (!config.policy.load_json(policy_fixture(), error)) { + throw std::runtime_error(error); + } + return config; +} + +ToolSpeculationPrediction prediction(double confidence = 0.9) { + ToolSpeculationPrediction value; + std::string error; + if (!CanonicalToolInvocation::from_parts( + "lookup", json{{"a", 1}, {"b", 2}}, value.call, error)) { + throw std::runtime_error(error); + } + value.confidence = confidence; + return value; +} + +#if !defined(_WIN32) +std::string make_executor_script(const std::string & body) { + char path[] = "/tmp/dflash-tool-spec-test-XXXXXX"; + const int fd = ::mkstemp(path); + if (fd < 0) throw std::runtime_error("mkstemp failed"); + const std::string script = "#!/bin/sh\nIFS= read -r request\n" + body; + size_t offset = 0; + while (offset < script.size()) { + const ssize_t count = ::write( + fd, script.data() + offset, script.size() - offset); + if (count <= 0) { + ::close(fd); + ::unlink(path); + throw std::runtime_error("script write failed"); + } + offset += static_cast(count); + } + if (::fchmod(fd, 0700) != 0) { + ::close(fd); + ::unlink(path); + throw std::runtime_error("chmod failed"); + } + ::close(fd); + return path; +} + +std::string make_temp_path() { + char path[] = "/tmp/dflash-tool-spec-observed-XXXXXX"; + const int fd = ::mkstemp(path); + if (fd < 0) throw std::runtime_error("mkstemp failed"); + ::close(fd); + return path; +} + +std::string read_text_file(const std::string & path) { + FILE * file = std::fopen(path.c_str(), "rb"); + if (!file) return {}; + std::string value; + char buffer[256]; + while (const size_t count = std::fread(buffer, 1, sizeof(buffer), file)) { + value.append(buffer, count); + } + std::fclose(file); + return value; +} +#endif +} // namespace + +TEST_CASE(ToolSpeculationFixture, canonical_identity_ignores_argument_order) { + CanonicalToolInvocation first; + CanonicalToolInvocation second; + std::string error; + CHECK(CanonicalToolInvocation::from_parts( + "lookup", json{{"b", 2}, {"a", 1}}, first, error)); + CHECK(CanonicalToolInvocation::from_parts( + "lookup", json{{"a", 1}, {"b", 2}}, second, error)); + CHECK(first == second); + CHECK(first.arguments_json == R"({"a":1,"b":2})"); +} + +TEST_CASE(ToolSpeculationFixture, engine_prediction_uses_canonical_boundary) { + ToolSpeculationPrediction value; + std::string error; + CHECK(build_tool_speculation_prediction( + "lookup", json{{"b", 2}, {"a", 1}}, 0.75, value, error)); + CHECK(error.empty()); + CHECK(value.call.name == "lookup"); + CHECK(value.call.arguments_json == "{\"a\":1,\"b\":2}"); + CHECK(std::fabs(value.confidence - 0.75) < 1e-9); + CHECK(!build_tool_speculation_prediction( + "lookup", json::array(), 0.75, value, error)); + CHECK(!build_tool_speculation_prediction( + "lookup", json::object(), 1.01, value, error)); +} + +TEST_CASE(ToolSpeculationFixture, prediction_requires_declared_tool) { + const json tools = json::array({{ + {"type", "function"}, + {"function", {{"name", "lookup"}}}, + }}); + const json request = { + {"call", { + {"name", "lookup"}, + {"arguments", {{"key", "x"}}}, + }}, + {"confidence", 0.75}, + }; + ToolSpeculationPrediction parsed; + std::string error; + CHECK(parse_tool_speculation_prediction(request, tools, parsed, error)); + CHECK(parsed.call.name == "lookup"); + + json undeclared = request; + undeclared["call"]["name"] = "write_file"; + CHECK(!parse_tool_speculation_prediction( + undeclared, tools, parsed, error)); + CHECK(error.find("not declared") != std::string::npos); +} + +TEST_CASE(ToolSpeculationFixture, cpu_affinity_parser_canonicalizes_ranges) { + std::vector cpus; + std::string error; + CHECK(parse_tool_speculation_cpu_affinity( + "30-31,15,14-15", cpus, error)); + CHECK(cpus == std::vector({14, 15, 30, 31})); + + CHECK(!parse_tool_speculation_cpu_affinity("14,,15", cpus, error)); + CHECK(cpus.empty()); + CHECK(!parse_tool_speculation_cpu_affinity("15-14", cpus, error)); + CHECK(cpus.empty()); +} + +TEST_CASE(ToolSpeculationFixture, empirical_policy_selects_resource_by_confidence) { + ToolSpeculationPolicy policy; + std::string error; + CHECK(policy.load_json(policy_fixture(), error)); + + const auto deferred = policy.choose(0.0, 1.20); + CHECK(!deferred.admitted); + CHECK(deferred.reason == "below_profile_break_even"); + + const auto low = policy.choose(0.10, 1.20); + CHECK(low.admitted); + CHECK(low.resource_percentage == 25); + + const auto medium = policy.choose(0.50, 1.20); + CHECK(medium.admitted); + CHECK(medium.resource_percentage == 50); + + const auto high = policy.choose(0.90, 1.20); + CHECK(high.admitted); + CHECK(high.resource_percentage == 100); + + const auto guarded = policy.choose(0.90, 1.10); + CHECK(guarded.admitted); + CHECK(guarded.resource_percentage == 50); +} + +TEST_CASE(ToolSpeculationFixture, unqualified_resource_lanes_are_deferred) { + ToolSpeculationPolicy policy; + std::string error; + CHECK(policy.load_json(policy_fixture(false), error)); + + const auto decision = policy.choose(1.0, 1.20); + CHECK(!decision.admitted); + CHECK(decision.reason == "decode_interference_unqualified"); +} + +TEST_CASE(ToolSpeculationFixture, profile_metadata_is_fail_closed) { + ToolSpeculationPolicy policy; + std::string error; + json fixture = policy_fixture(); + fixture["profile_status"] = "provisional_benchmark_only"; + fixture["executor"] = "in_process_hip_cu_mask"; + CHECK(policy.load_json(fixture, error)); + CHECK(policy.benchmark_only()); + CHECK(policy.executor_contract() == "in_process_hip_cu_mask"); + + fixture["profile_status"] = "unknown"; + CHECK(!policy.load_json(fixture, error)); + CHECK(policy.empty()); +} + +TEST_CASE(ToolSpeculationFixture, same_gpu_profile_declares_routing_requirements) { + ToolSpeculationPolicy policy; + std::string error; + json fixture = policy_fixture(); + for (auto & lane : fixture["path_summary"]) { + lane["accelerator_relation"] = "same_physical_gpu"; + } + CHECK(policy.load_json(fixture, error)); + CHECK(!policy.requires_static_model_routing()); + CHECK(!policy.requires_unique_expert_ownership()); + + for (auto & lane : fixture["path_summary"]) { + lane["requires_static_model_routing"] = true; + } + CHECK(policy.load_json(fixture, error)); + CHECK(policy.requires_static_model_routing()); + CHECK(!policy.requires_unique_expert_ownership()); + + for (auto & lane : fixture["path_summary"]) { + lane["requires_unique_expert_ownership"] = true; + } + CHECK(policy.load_json(fixture, error)); + CHECK(policy.requires_static_model_routing()); + CHECK(policy.requires_unique_expert_ownership()); + const auto decision = policy.choose(1.0, 2.0); + CHECK(decision.admitted); + CHECK(decision.accelerator_relation == "same_physical_gpu"); +} + +TEST_CASE(ToolSpeculationFixture, same_gpu_profile_obeys_measured_break_even) { + ToolSpeculationPolicy policy; + std::string error; + CHECK(policy.load_json(json{ + {"path_summary", { + {"100", { + {"accelerator_relation", "same_physical_gpu"}, + {"requires_static_model_routing", true}, + {"requires_unique_expert_ownership", true}, + {"decode_interference_qualified", true}, + {"hit", { + {"control_task_mean_ms", 2670.178}, + {"speculative_task_mean_ms", 2389.530}, + {"model_slowdown_percent", 169.109}, + }}, + {"miss", { + {"control_task_mean_ms", 2670.178}, + {"speculative_task_mean_ms", 4171.884}, + {"model_slowdown_percent", 169.109}, + }}, + }}, + }}, + }, error)); + + const auto below = policy.choose(0.84, 3.0); + CHECK(!below.admitted); + CHECK(below.reason == "below_profile_break_even"); + + const auto above = policy.choose(0.85, 3.0); + CHECK(above.admitted); + CHECK(above.resource_percentage == 100); + + const auto guarded = policy.choose(1.0, 1.20); + CHECK(!guarded.admitted); + CHECK(guarded.reason == "model_slowdown_guardrail"); +} + +TEST_CASE(ToolSpeculationFixture, non_allowlisted_tool_is_deferred) { + ToolSpeculationConfig config = test_config(); + config.allowed_tools = {"other"}; + auto attempt = ToolSpeculationAttempt::create( + config, prediction(), "request_allowlist"); + attempt->start(); + const json metadata = attempt->resolve({ + ToolCall{"call_1", "lookup", R"({"a":1,"b":2})"}, + }); + CHECK(metadata["status"] == "deferred"); + CHECK(metadata["reason"] == "tool_not_allowlisted"); + CHECK(!metadata.contains("result")); +} + +TEST_CASE(ToolSpeculationFixture, in_process_exact_match_commits_private_result) { + auto state = std::make_shared(); + ToolSpeculationConfig config = test_config(); + config.executor_path.clear(); + config.in_process_executor = std::make_shared(state); + CHECK(config.enabled()); + CHECK(std::string(config.execution_mode()) == "fake_in_process"); + + auto attempt = ToolSpeculationAttempt::create( + config, prediction(), "request_in_process_hit"); + attempt->start(); + CHECK(attempt->running()); + CHECK(state->request["call"]["name"] == "lookup"); + CHECK(state->request["resource_percentage"] == 100); + + const json metadata = attempt->resolve({ + ToolCall{"call_1", "lookup", R"({"b":2,"a":1})"}, + }); + CHECK(metadata["status"] == "hit"); + CHECK(metadata["result"]["value"] == 42); + CHECK(state->collected); + CHECK(state->controls.size() == 1); + CHECK(state->controls[0] == "commit"); + CHECK(!state->terminated); +} + +TEST_CASE(ToolSpeculationFixture, in_process_mismatch_cancels_private_result) { + auto state = std::make_shared(); + ToolSpeculationConfig config = test_config(); + config.executor_path.clear(); + config.in_process_executor = std::make_shared(state); + + auto attempt = ToolSpeculationAttempt::create( + config, prediction(), "request_in_process_miss"); + attempt->start(); + const json metadata = attempt->resolve({ + ToolCall{"call_1", "lookup", R"({"a":999})"}, + }); + CHECK(metadata["status"] == "miss"); + CHECK(metadata["reason"] == "invocation_mismatch"); + CHECK(!metadata.contains("result")); + CHECK(!state->collected); + CHECK(state->terminated); + CHECK(state->controls.size() == 1); + CHECK(state->controls[0] == "cancel"); +} + +#if !defined(_WIN32) +TEST_CASE(ToolSpeculationFixture, cpu_affinity_reaches_child_executor) { +#if defined(__linux__) + cpu_set_t allowed; + CPU_ZERO(&allowed); + CHECK(::sched_getaffinity(0, sizeof(allowed), &allowed) == 0); + int selected_cpu = -1; + for (int cpu = 0; cpu < CPU_SETSIZE; ++cpu) { + if (CPU_ISSET(cpu, &allowed)) { + selected_cpu = cpu; + break; + } + } + CHECK(selected_cpu >= 0); + const std::string selected = std::to_string(selected_cpu); + const std::string path = make_executor_script( + "IFS= read -r control\n" + "observed=$(awk '/Cpus_allowed_list/{print $2}' /proc/self/status)\n" + "printf '{\"ok\":true,\"result\":{\"configured\":\"%s\"," + "\"observed\":\"%s\"}}\\n' " + "\"$DFLASH_TOOL_SPECULATION_CPU_AFFINITY\" \"$observed\"\n"); + ToolSpeculationConfig config = test_config(path); + config.cpu_affinity = {selected_cpu}; + config.cpu_affinity_isolated = true; + CHECK(std::string(config.execution_mode()) == + "child_process_cpu_affinity"); + auto attempt = ToolSpeculationAttempt::create( + config, prediction(), "request_cpu_affinity"); + attempt->start(); + const json metadata = attempt->resolve({ + ToolCall{"call_1", "lookup", R"({"a":1,"b":2})"}, + }); + ::unlink(path.c_str()); + CHECK(metadata["status"] == "hit"); + CHECK(metadata["cpu_affinity_isolated"].get()); + CHECK(metadata["result"]["configured"] == selected); + CHECK(metadata["result"]["observed"] == selected); +#else + CHECK(true); +#endif +} + +TEST_CASE(ToolSpeculationFixture, cpu_affinity_qualification_rejects_overlap) { +#if defined(__linux__) + cpu_set_t allowed; + CPU_ZERO(&allowed); + CHECK(::sched_getaffinity(0, sizeof(allowed), &allowed) == 0); + int selected_cpu = -1; + for (int cpu = 0; cpu < CPU_SETSIZE; ++cpu) { + if (CPU_ISSET(cpu, &allowed)) { + selected_cpu = cpu; + break; + } + } + CHECK(selected_cpu >= 0); + ToolSpeculationConfig config = test_config(); + config.cpu_affinity = {selected_cpu}; + std::string error; + CHECK(!qualify_tool_speculation_cpu_affinity(config, error)); + CHECK(error.find("overlaps model CPU") != std::string::npos); + CHECK(!config.cpu_affinity_isolated); +#else + CHECK(true); +#endif +} + +TEST_CASE(ToolSpeculationFixture, exact_match_exposes_result_and_resource_share) { + const std::string control_path = make_temp_path(); + const std::string path = make_executor_script( + "IFS= read -r control\nprintf '%s' \"$control\" > " + control_path + "\n" + "printf '{\"ok\":true,\"result\":{\"resource\":\"%s\"," + "\"relation\":\"%s\",\"value\":42}}\\n' " + "\"$DFLASH_TOOL_SPECULATION_RESOURCE_PERCENTAGE\" " + "\"$DFLASH_TOOL_SPECULATION_ACCELERATOR_RELATION\"\n"); + ToolSpeculationConfig config = test_config(path); + auto attempt = ToolSpeculationAttempt::create( + config, prediction(), "request_hit"); + attempt->start(); + const json metadata = attempt->resolve({ + ToolCall{"call_1", "lookup", R"({"b":2,"a":1})"}, + }); + ::unlink(path.c_str()); + const std::string control = read_text_file(control_path); + ::unlink(control_path.c_str()); + + CHECK(metadata["status"] == "hit"); + CHECK(metadata["resource_percentage"] == 100); + CHECK(metadata["result"]["resource"] == "100"); + CHECK(metadata["result"]["relation"] == "unspecified"); + CHECK(metadata["result"]["value"] == 42); + CHECK(control.find("\"op\":\"commit\"") != std::string::npos); + CHECK(control.find("\"authoritative_resource_percentage\":100") != + std::string::npos); +} + +TEST_CASE(ToolSpeculationFixture, unqualified_tool_never_launches_or_changes_decode) { + const std::string path = make_executor_script( + "IFS= read -r control\n" + "exit 0\n"); + ToolSpeculationConfig config = test_config(path); + std::string error; + CHECK(config.policy.load_json(policy_fixture(false), error)); + auto deferred = ToolSpeculationAttempt::create( + config, prediction(), "request_guarded_decode"); + deferred->start(); + CHECK(!deferred->running()); + const json metadata = deferred->resolve({ + ToolCall{"call_1", "lookup", R"({"a":1,"b":2})"}, + }); + CHECK(metadata["status"] == "deferred"); + CHECK(metadata["reason"] == "decode_interference_unqualified"); + ::unlink(path.c_str()); +} + +TEST_CASE(ToolSpeculationFixture, executor_failure_is_private) { + ToolSpeculationConfig config = test_config( + "/definitely/missing/dflash-tool-spec-executor"); + auto attempt = ToolSpeculationAttempt::create( + config, prediction(), "request_executor_failure"); + attempt->start(); + const json metadata = attempt->resolve({ + ToolCall{"call_1", "lookup", R"({"a":1,"b":2})"}, + }); + CHECK(metadata["status"] == "failed"); + CHECK(metadata["reason"] == "executor_launch_failed"); + CHECK(!metadata.contains("result")); +} + +TEST_CASE(ToolSpeculationFixture, qualified_lane_keeps_speculative_decode) { + const std::string path = make_executor_script( + "IFS= read -r control\n" + "exit 0\n"); + ToolSpeculationConfig config = test_config(path); + std::string error; + CHECK(config.policy.load_json(policy_fixture(), error)); + auto attempt = ToolSpeculationAttempt::create( + config, prediction(), "request_qualified_lane"); + attempt->start(); + CHECK(attempt->running()); + const json metadata = attempt->cancel("test_complete"); + CHECK(metadata["decode_interference_qualified"].get()); + ::unlink(path.c_str()); +} + +TEST_CASE(ToolSpeculationFixture, mismatch_cancels_and_never_exposes_result) { + const std::string control_path = make_temp_path(); + const std::string path = make_executor_script( + "IFS= read -r control\nprintf '%s' \"$control\" > " + control_path + "\n" + "exit 3\n"); + ToolSpeculationConfig config = test_config(path); + auto attempt = ToolSpeculationAttempt::create( + config, prediction(), "request_miss"); + attempt->start(); + const auto started = std::chrono::steady_clock::now(); + const json metadata = attempt->resolve({ + ToolCall{"call_1", "lookup", R"({"a":999})"}, + }); + const double elapsed_ms = std::chrono::duration( + std::chrono::steady_clock::now() - started).count(); + ::unlink(path.c_str()); + const std::string control = read_text_file(control_path); + ::unlink(control_path.c_str()); + + CHECK(metadata["status"] == "miss"); + CHECK(metadata["reason"] == "invocation_mismatch"); + CHECK(!metadata.contains("result")); + CHECK(elapsed_ms < 1000.0); + CHECK(control.find("\"op\":\"cancel\"") != std::string::npos); +} +#endif + +TEST_CASE(ToolSpeculationFixture, streaming_extension_keeps_result_explicit) { + const json metadata = { + {"status", "hit"}, + {"result", {{"value", 42}}}, + }; + const std::string event = render_tool_speculation_sse( + ApiFormat::OPENAI_CHAT, "req_1", "model", metadata); + CHECK(event.find("dflash_tool_speculation") != std::string::npos); + CHECK(event.find("\"value\":42") != std::string::npos); +}