From 8fc35ae8f516fb9aa98d0eb016d69162d397341f Mon Sep 17 00:00:00 2001 From: mrciffa <49000955+davide221@users.noreply.github.com> Date: Thu, 13 Aug 2026 17:18:19 +0200 Subject: [PATCH 01/11] feat(server): add lossless tool speculation lanes --- server/CMakeLists.txt | 11 +- .../llama.cpp/ggml/src/ggml-cuda/common.cuh | 87 ++ server/src/common/model_backend.h | 5 + server/src/server/http_server.cpp | 92 +- server/src/server/http_server.h | 11 + server/src/server/server_main.cpp | 264 +++++ server/src/server/tool_speculation.cpp | 984 ++++++++++++++++++ server/src/server/tool_speculation.h | 246 +++++ .../src/server/tool_speculation_hip_probe.cpp | 456 ++++++++ .../src/server/tool_speculation_hip_probe.h | 21 + server/test/test_server_unit.cpp | 116 +++ server/test/test_tool_speculation.cpp | 530 ++++++++++ 12 files changed, 2821 insertions(+), 2 deletions(-) create mode 100644 server/src/server/tool_speculation.cpp create mode 100644 server/src/server/tool_speculation.h create mode 100644 server/src/server/tool_speculation_hip_probe.cpp create mode 100644 server/src/server/tool_speculation_hip_probe.h create mode 100644 server/test/test_tool_speculation.cpp diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index 0ed5743fb..b4876f957 100644 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -1466,6 +1466,7 @@ if(DFLASH27B_TESTS) set(_server_unit_sources test/test_unit_main.cpp test/test_server_unit.cpp + test/test_tool_speculation.cpp test/test_anchor_params.cpp test/test_derived_scalars.cpp test/test_adaptive_keep_ratio.cpp @@ -1497,6 +1498,7 @@ if(DFLASH27B_TESTS) add_executable(test_server_unit ${_server_unit_sources}) target_sources(test_server_unit PRIVATE src/server/http_server.cpp + src/server/tool_speculation.cpp src/server/model_card.cpp src/server/prompt_normalize.cpp src/qwen3/anchor_scan.cpp) @@ -1746,12 +1748,15 @@ if(DFLASH27B_SERVER) add_executable(dflash_server src/server/server_main.cpp src/server/http_server.cpp + src/server/tool_speculation.cpp src/server/model_card.cpp src/server/prompt_normalize.cpp ) target_include_directories(dflash_server PRIVATE ${DFLASH27B_SRC_INCLUDE_DIRS}) if(DFLASH27B_GPU_BACKEND STREQUAL "hip") target_compile_definitions(dflash_server PRIVATE DFLASH27B_BACKEND_HIP=1 GGML_USE_HIP) + target_sources(dflash_server PRIVATE + src/server/tool_speculation_hip_probe.cpp) if(DFLASH27B_ENABLE_MIXED_CUDA_HIP) target_compile_definitions(dflash_server PRIVATE DFLASH27B_BACKEND_MIXED=1) @@ -1778,7 +1783,11 @@ if(DFLASH27B_SERVER) find_package(CUDAToolkit REQUIRED) target_link_libraries(dflash_server PRIVATE CUDA::cudart) else() - target_link_libraries(dflash_server PRIVATE hip::host) + # ggml-hip finds hipBLAS in a child-directory scope. The trusted + # in-process tool adapter also calls hipBLAS directly, so import + # the target in this scope before linking the server executable. + find_package(hipblas REQUIRED) + target_link_libraries(dflash_server PRIVATE hip::host roc::hipblas) endif() # Copy share/status.html next to the binary so it can be found at runtime. diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/common.cuh b/server/deps/llama.cpp/ggml/src/ggml-cuda/common.cuh index 31b04cd14..0680df55e 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cuda/common.cuh +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/common.cuh @@ -25,8 +25,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -161,6 +163,60 @@ static int ggml_cuda_highest_compiled_arch(const int arch) { #define GGML_CUDA_MAX_STREAMS 8 +#if defined(GGML_USE_HIP) +// Optional Lucebox experiment: reserve the lowest CUs on one HIP device for a +// trusted in-process tool stream. Every lazily-created ggml stream on that +// device receives the complementary CU mask, creating a real disjoint lane +// inside one HIP context. Default behavior is unchanged when the variable is +// absent. Format: DFLASH_HIP_RESERVED_TOOL_LANE=DEVICE:CUS (for example 0:1). +static inline bool dflash_hip_model_stream_mask( + int device, std::vector & mask, int & reserved_cus) { + const char * raw = std::getenv("DFLASH_HIP_RESERVED_TOOL_LANE"); + if (!raw || !*raw) return false; + + errno = 0; + char * separator = nullptr; + const long configured_device = std::strtol(raw, &separator, 10); + if (errno != 0 || separator == raw || !separator || *separator != ':') { + GGML_ABORT( + "DFLASH_HIP_RESERVED_TOOL_LANE must be DEVICE:CUS, got '%s'\n", + raw); + } + char * end = nullptr; + errno = 0; + const long configured_cus = std::strtol(separator + 1, &end, 10); + if (errno != 0 || !end || *end != '\0' || configured_device < 0 || + configured_cus <= 0) { + GGML_ABORT( + "DFLASH_HIP_RESERVED_TOOL_LANE must be DEVICE:CUS with positive " + "integers, got '%s'\n", raw); + } + if (configured_device != device) return false; + + hipDeviceProp_t properties{}; + const hipError_t status = hipGetDeviceProperties(&properties, device); + if (status != hipSuccess) { + GGML_ABORT( + "DFLASH_HIP_RESERVED_TOOL_LANE could not inspect HIP device %d: " + "%s\n", + device, hipGetErrorString(status)); + } + if (configured_cus >= properties.multiProcessorCount) { + GGML_ABORT( + "DFLASH_HIP_RESERVED_TOOL_LANE reserves %ld of %d CUs on device " + "%d; at least one model CU is required\n", + configured_cus, properties.multiProcessorCount, device); + } + reserved_cus = static_cast(configured_cus); + mask.assign( + static_cast((properties.multiProcessorCount + 31) / 32), 0); + for (int cu = reserved_cus; cu < properties.multiProcessorCount; ++cu) { + mask[static_cast(cu / 32)] |= uint32_t{1} << (cu % 32); + } + return true; +} +#endif + [[noreturn]] void ggml_cuda_error(const char * stmt, const char * func, const char * file, int line, const char * msg); @@ -1486,6 +1542,37 @@ struct ggml_backend_cuda_context { cudaStream_t stream(int device, int stream) { if (streams[device][stream] == nullptr) { ggml_cuda_set_device(device); +#if defined(GGML_USE_HIP) + std::vector model_cu_mask; + int reserved_cus = 0; + const bool disjoint_tool_lane = dflash_hip_model_stream_mask( + device, model_cu_mask, reserved_cus); + if (disjoint_tool_lane) { + CUDA_CHECK(hipExtStreamCreateWithCUMask( + &streams[device][stream], + static_cast(model_cu_mask.size()), + model_cu_mask.data())); + if (low_priority_streams) { +#if HIP_VERSION_MAJOR >= 7 + hipStreamAttrValue priority{}; + priority.priority = stream_priority; + CUDA_CHECK(hipStreamSetAttribute( + streams[device][stream], + hipStreamAttributePriority, &priority)); +#else + GGML_ABORT( + "DFLASH_HIP_RESERVED_TOOL_LANE requires ROCm 7+ " + "to preserve low-priority DSpark streams\n"); +#endif + } + if (stream == 0) { + std::fprintf(stderr, + "ggml_hip: device %d model streams exclude %d " + "low CU(s) reserved for in-process tools\n", + device, reserved_cus); + } + } else +#endif if (low_priority_streams) { CUDA_CHECK(cudaStreamCreateWithPriority( &streams[device][stream], cudaStreamNonBlocking, diff --git a/server/src/common/model_backend.h b/server/src/common/model_backend.h index 0a4d6e00d..c82021505 100644 --- a/server/src/common/model_backend.h +++ b/server/src/common/model_backend.h @@ -196,6 +196,10 @@ struct GenerateRequest { // path returns success but emits no tokens, so each backend can route the // retry through its existing AR path without copying retry policy. bool force_ar_decode = false; + // Opt out of the common speculative-to-AR empty-output retry. Tool + // speculation sets this false so the external optimization can never + // change the request's model decode strategy. + bool allow_decode_mode_retry = true; }; // Stable, backend-independent generation failure categories. Backends should @@ -355,6 +359,7 @@ struct ModelBackend { static bool should_retry_empty_spec_decode(const GenerateRequest & req, const GenerateResult & result) { return req.n_gen > 0 + && req.allow_decode_mode_retry && !req.force_ar_decode && result.ok() && result.spec_decode_ran diff --git a/server/src/server/http_server.cpp b/server/src/server/http_server.cpp index 2cad0e142..bf8b0ad43 100644 --- a/server/src/server/http_server.cpp +++ b/server/src/server/http_server.cpp @@ -706,6 +706,20 @@ json build_props_body(const ServerConfig & config, // benchmarks to silently run at temp=0 (degenerate-decode collapse) // when the model card specifies temp=1.0/top_p=0.95/top_k=64. const auto & smp = config.sampler_defaults; + json tool_spec_lanes = json::array(); + for (const auto & lane : config.tool_speculation.policy.lanes()) { + tool_spec_lanes.push_back({ + {"resource_percentage", lane.resource_percentage}, + {"model_slowdown_ratio", lane.model_slowdown_ratio}, + {"decode_interference_qualified", + lane.decode_interference_qualified}, + {"accelerator_relation", lane.accelerator_relation}, + {"requires_static_model_routing", + lane.requires_static_model_routing}, + {"requires_unique_expert_ownership", + lane.requires_unique_expert_ownership}, + }); + } json body = { {"default_generation_settings", { {"n_ctx", config.max_ctx}, @@ -780,6 +794,39 @@ json build_props_body(const ServerConfig & config, {"ddtree_budget", config.speculative_enabled ? json(config.ddtree_budget) : json(nullptr)}, }}, + {"tool_speculation", { + {"enabled", config.tool_speculation.enabled()}, + {"execution_mode", config.tool_speculation.execution_mode()}, + {"profile_status", + config.tool_speculation.policy.empty() + ? json(nullptr) + : json(config.tool_speculation.policy.profile_status())}, + {"executor_contract", + config.tool_speculation.policy.executor_contract().empty() + ? json(nullptr) + : json(config.tool_speculation.policy.executor_contract())}, + {"protocol", "dflash.tool-speculation.v1"}, + {"requires_client_support", true}, + {"preserves_token_speculation", true}, + {"unqualified_lane_policy", "defer"}, + {"allowed_tools", config.tool_speculation.allowed_tools}, + {"max_model_slowdown_ratio", + config.tool_speculation.max_model_slowdown_ratio}, + {"model_routing_static", + config.tool_speculation.model_routing_static}, + {"model_expert_ownership_unique", + config.tool_speculation.model_expert_ownership_unique}, + {"compute_isolation", + config.tool_speculation.hip_reserved_tool_compute_units > 0 + ? "disjoint_hip_cu_masks" : "none"}, + {"hip_tool_device", + config.tool_speculation.hip_tool_device >= 0 + ? json(config.tool_speculation.hip_tool_device) + : json(nullptr)}, + {"hip_reserved_tool_compute_units", + config.tool_speculation.hip_reserved_tool_compute_units}, + {"profile_lanes", tool_spec_lanes}, + }}, {"sampling", { {"capabilities", { {"supports_temperature", true}, @@ -1610,6 +1657,18 @@ bool HttpServer::parse_common_request_fields( // Tool choice constraint for hint generation. if (body.contains("tool_choice")) req.tool_choice = body["tool_choice"]; + if (body.contains("tool_speculation")) { + ToolSpeculationPrediction prediction; + std::string prediction_error; + if (!parse_tool_speculation_prediction( + body["tool_speculation"], req.tools, prediction, + prediction_error)) { + send_error(fd, 400, prediction_error); + return false; + } + req.tool_speculation = std::move(prediction); + } + if (body.contains("prefix_cache") && body["prefix_cache"].is_object()) { const auto & prefix_cache = body["prefix_cache"]; if (prefix_cache.contains("scope") && prefix_cache["scope"].is_string() && @@ -3339,6 +3398,11 @@ void HttpServer::prepare_generation_inputs( inputs.request.n_gen = inputs.generation_cap; inputs.request.sampler = req.sampler; inputs.request.do_sample = req.sampler.needs_logit_processing(); + // An opted-in external tool may overlap the chosen decoder, but it must + // never cause speculative decoding to be retried as AR. If the selected + // decoder cannot produce output, surface that outcome unchanged. + inputs.request.allow_decode_mode_retry = + !req.tool_speculation.has_value(); // Tokens are delivered through DaemonIO so all API formats share the // same disconnect and streaming state machine. inputs.request.stream = false; @@ -3566,6 +3630,14 @@ void HttpServer::process_job(ServerJob * job) { return; } + std::unique_ptr tool_speculation; + if (req.tool_speculation.has_value()) { + tool_speculation = ToolSpeculationAttempt::create( + config_.tool_speculation, *req.tool_speculation, + req.response_id); + tool_speculation->start(); + } + auto & effective_prompt = prepared.tokens; const bool pflash_compressed = prepared.compressed; @@ -3697,6 +3769,18 @@ void HttpServer::process_job(ServerJob * job) { } if (req.stream && !client_disconnected) { auto final_chunks = emitter.emit_finish(completion_tokens, &gen_timings); + if (tool_speculation) { + const json metadata = tool_speculation->resolve(emitter.tool_calls()); + const std::string extension = render_tool_speculation_sse( + req.format, req.response_id, req.model, metadata); + // Keep the standard terminal event last: [DONE], message_stop, + // or response.completed. Only opted-in clients see this extension. + if (final_chunks.empty()) { + final_chunks.push_back(extension); + } else { + final_chunks.insert(final_chunks.end() - 1, extension); + } + } for (const auto & chunk : final_chunks) { if (!send_job_bytes(job, chunk.data(), chunk.size())) { client_disconnected = true; @@ -3704,14 +3788,20 @@ void HttpServer::process_job(ServerJob * job) { } } } else if (!req.stream && !client_disconnected) { - const json response = build_non_streaming_response( + json response = build_non_streaming_response( req, result, n_gen_cap, gen_timings, tokenizer_, emitter); + if (tool_speculation) { + response["dflash_tool_speculation"] = + tool_speculation->resolve(emitter.tool_calls()); + } // Streaming uses non-blocking sends; restore blocking mode before // writing a complete JSON response on this shared socket path. const int flags = sock_get_flags(fd); if (flags >= 0) sock_set_block(fd); send_response(fd, 200, "application/json", response.dump() + "\n"); + } else if (tool_speculation) { + tool_speculation->cancel("client_disconnected"); } if (client_disconnected) { diff --git a/server/src/server/http_server.h b/server/src/server/http_server.h index caf5b6946..14a5a82cb 100644 --- a/server/src/server/http_server.h +++ b/server/src/server/http_server.h @@ -18,6 +18,7 @@ #include "tokenizer.h" #include "chat_template.h" #include "tool_memory.h" +#include "tool_speculation.h" #include "prefix_cache.h" #include "disk_prefix_cache.h" #include "freeze_history.h" @@ -37,6 +38,7 @@ #include #include #include +#include #include #include #if !defined(_WIN32) @@ -217,6 +219,11 @@ struct ServerConfig { // Routing data collection (--collect-routing ): write binary per-token // routing data (hidden states + expert selections) for predictor training. std::string collect_routing_path; + + // Lossless external-tool speculation. Off unless the operator configures + // an executor, an empirical interference profile, and an explicit + // read-only/idempotent tool allowlist. + ToolSpeculationConfig tool_speculation; }; // ─── Parsed request ───────────────────────────────────────────────────── @@ -236,6 +243,10 @@ struct ParsedRequest { json messages; // Original request body (for upstream proxy forwarding) json raw_body; + // Concrete invocation predicted by a caller or future semantic sidecar. + // The engine may execute it privately, but never exposes its result until + // the model emits the exact canonical invocation. + std::optional tool_speculation; // Response ID std::string response_id; // Thinking/reasoning state diff --git a/server/src/server/server_main.cpp b/server/src/server/server_main.cpp index adfedc300..936a7f41c 100644 --- a/server/src/server/server_main.cpp +++ b/server/src/server/server_main.cpp @@ -12,6 +12,9 @@ // [--max-tokens 4096] [--target-device auto:0] #include "http_server.h" +#if defined(DFLASH27B_BACKEND_HIP) +#include "tool_speculation_hip_probe.h" +#endif #include "chat_template.h" #include "model_card.h" #include "common/backend_factory.h" @@ -26,6 +29,7 @@ #include "placement/draft_residency.h" #include +#include #include #include #include @@ -35,6 +39,10 @@ #include #include +#if !defined(_WIN32) +#include +#endif + using namespace dflash::common; // Global server pointer for signal handling. @@ -64,6 +72,11 @@ static bool parse_double_list(const char * value, std::vector & out) { return !out.empty(); } +static bool environment_flag_enabled(const char * name) { + const char * value = std::getenv(name); + return value && *value && std::strcmp(value, "0") != 0; +} + static void print_usage(const char * prog) { std::fprintf(stderr, "Usage: %s [options]\n" @@ -165,6 +178,24 @@ static void print_usage(const char * prog) { " Drafter lifetime policy (default: auto)\n" " --lazy-draft Legacy alias for --draft-residency=request-scoped\n" "\n" + "Speculative external tools (opt-in, POSIX):\n" + " --tool-spec-executor Trusted executor adapter. Receives one\n" + " dflash.tool-speculation.v1 JSON request\n" + " on stdin; no shell is used.\n" +#if defined(DFLASH27B_BACKEND_HIP) + " --tool-spec-hip-sgemm-probe \n" + " Benchmark-only trusted in-process HIP\n" + " executor with a per-lane CU-masked stream.\n" +#endif + " --tool-spec-profile Measured resource-lane frontier JSON.\n" + " --tool-spec-allow Allow one read-only/idempotent tool; repeatable.\n" + " --tool-spec-timeout-ms Executor result timeout (default: 60000).\n" + " --tool-spec-max-model-slowdown \n" + " Reject lanes slower than this inference\n" + " ratio (default: 1.20).\n" + " Every admitted lane must pass exact-output\n" + " decode-interference qualification.\n" + "\n" "PFlash upstream proxy (forward compressed prompt to a backend):\n" " --prefill-upstream-base OpenAI-compatible upstream. Compressed\n" " requests POST the raw prompt to\n" @@ -216,6 +247,9 @@ int main(int argc, char ** argv) { // Parse arguments. BackendArgs bargs; ServerConfig sconfig; + int tool_spec_hip_probe_device = -1; + int tool_spec_hip_probe_matrix = 0; + int tool_spec_hip_probe_total_cus = 0; bargs.model_path = argv[1]; bool spark_autotune = false; // --spark: self-tuning hot/cold MoE residency int spark_slots = -1; // --spark-slots: explicit cache slots/layer (-1=auto) @@ -506,6 +540,62 @@ int main(int argc, char ** argv) { } else if (std::strcmp(argv[i], "--lazy-draft") == 0) { sconfig.lazy_draft = true; sconfig.draft_residency = DraftResidencyPolicy::RequestScoped; + } else if (std::strcmp(argv[i], "--tool-spec-executor") == 0 && + i + 1 < argc) { + sconfig.tool_speculation.executor_path = argv[++i]; +#if defined(DFLASH27B_BACKEND_HIP) + } else if (std::strcmp(argv[i], "--tool-spec-hip-sgemm-probe") == 0 && + i + 1 < argc) { + const char * value = argv[++i]; + char * separator = nullptr; + const long device = std::strtol(value, &separator, 10); + if (separator == value || !separator || *separator != ':') { + std::fprintf(stderr, + "[server] --tool-spec-hip-sgemm-probe expects DEVICE:MATRIX\n"); + return 2; + } + char * end = nullptr; + const long matrix = std::strtol(separator + 1, &end, 10); + if (!end || *end != '\0' || device < 0 || + matrix <= 0 || matrix > 8192) { + std::fprintf(stderr, + "[server] invalid --tool-spec-hip-sgemm-probe DEVICE:MATRIX\n"); + return 2; + } + tool_spec_hip_probe_device = static_cast(device); + tool_spec_hip_probe_matrix = static_cast(matrix); +#endif + } else if (std::strcmp(argv[i], "--tool-spec-profile") == 0 && + i + 1 < argc) { + sconfig.tool_speculation.profile_path = argv[++i]; + } else if (std::strcmp(argv[i], "--tool-spec-allow") == 0 && + i + 1 < argc) { + const std::string name = argv[++i]; + if (name.empty()) { + std::fprintf(stderr, "[server] --tool-spec-allow needs a name\n"); + return 2; + } + sconfig.tool_speculation.allowed_tools.push_back(name); + } else if (std::strcmp(argv[i], "--tool-spec-timeout-ms") == 0 && + i + 1 < argc) { + sconfig.tool_speculation.timeout_ms = std::atoi(argv[++i]); + if (sconfig.tool_speculation.timeout_ms <= 0) { + std::fprintf(stderr, + "[server] --tool-spec-timeout-ms must be positive\n"); + return 2; + } + } else if (std::strcmp( + argv[i], "--tool-spec-max-model-slowdown") == 0 && + i + 1 < argc) { + sconfig.tool_speculation.max_model_slowdown_ratio = + std::atof(argv[++i]); + if (!std::isfinite( + sconfig.tool_speculation.max_model_slowdown_ratio) || + sconfig.tool_speculation.max_model_slowdown_ratio < 1.0) { + std::fprintf(stderr, + "[server] --tool-spec-max-model-slowdown must be >= 1\n"); + return 2; + } } else if (std::strcmp(argv[i], "--chat-template-file") == 0 && i + 1 < argc) { const char * path = argv[++i]; std::FILE * f = std::fopen(path, "rb"); @@ -570,6 +660,156 @@ int main(int argc, char ** argv) { return 2; } } + + if (tool_spec_hip_probe_device >= 0) { +#if defined(DFLASH27B_BACKEND_HIP) + std::string executor_error; + sconfig.tool_speculation.in_process_executor = + create_hip_sgemm_tool_speculation_executor( + tool_spec_hip_probe_device, + tool_spec_hip_probe_matrix, + tool_spec_hip_probe_total_cus, + executor_error); + if (!sconfig.tool_speculation.in_process_executor) { + std::fprintf(stderr, "[server] %s\n", executor_error.c_str()); + return 2; + } +#endif + } + const bool tool_speculation_requested = + !sconfig.tool_speculation.executor_path.empty() || + static_cast(sconfig.tool_speculation.in_process_executor) || + !sconfig.tool_speculation.profile_path.empty() || + !sconfig.tool_speculation.allowed_tools.empty(); + if (tool_speculation_requested) { + sconfig.tool_speculation.model_routing_static = + !environment_flag_enabled( + "DFLASH_MOE_TP_DYNAMIC_ROUTE_BALANCE") && + !environment_flag_enabled( + "DFLASH_DS4_TP_DYNAMIC_ROUTE_BALANCE"); + sconfig.tool_speculation.model_expert_ownership_unique = + !environment_flag_enabled("DFLASH_MOE_DUPLICATE_HOT_ON_COLD"); + const bool has_child_executor = + !sconfig.tool_speculation.executor_path.empty(); + const bool has_in_process_executor = + static_cast(sconfig.tool_speculation.in_process_executor); + if (has_child_executor == has_in_process_executor || + sconfig.tool_speculation.profile_path.empty() || + sconfig.tool_speculation.allowed_tools.empty()) { + std::fprintf(stderr, + "[server] tool speculation requires exactly one executor, " + "--tool-spec-profile, and at least one --tool-spec-allow\n"); + return 2; + } +#if !defined(_WIN32) + if (has_child_executor && + ::access(sconfig.tool_speculation.executor_path.c_str(), X_OK) != 0) { + std::fprintf(stderr, + "[server] tool speculation executor is not executable: %s\n", + sconfig.tool_speculation.executor_path.c_str()); + return 2; + } +#endif + std::sort(sconfig.tool_speculation.allowed_tools.begin(), + sconfig.tool_speculation.allowed_tools.end()); + sconfig.tool_speculation.allowed_tools.erase( + std::unique(sconfig.tool_speculation.allowed_tools.begin(), + sconfig.tool_speculation.allowed_tools.end()), + sconfig.tool_speculation.allowed_tools.end()); + std::string profile_error; + if (!sconfig.tool_speculation.policy.load_file( + sconfig.tool_speculation.profile_path, profile_error)) { + std::fprintf(stderr, "[server] %s\n", profile_error.c_str()); + return 2; + } + const std::string & executor_contract = + sconfig.tool_speculation.policy.executor_contract(); + if (!executor_contract.empty() && + executor_contract != + sconfig.tool_speculation.execution_mode()) { + std::fprintf(stderr, + "[server] tool profile requires executor '%s', got '%s'\n", + executor_contract.c_str(), + sconfig.tool_speculation.execution_mode()); + return 2; + } + if (sconfig.tool_speculation.policy.benchmark_only() && + !has_in_process_executor) { + std::fprintf(stderr, + "[server] provisional_benchmark_only tool profiles cannot " + "enable an external production executor\n"); + return 2; + } + if (has_in_process_executor) { + int max_same_gpu_percentage = 0; + for (const auto & lane : + sconfig.tool_speculation.policy.lanes()) { + if (lane.decode_interference_qualified && + lane.accelerator_relation == "same_physical_gpu") { + max_same_gpu_percentage = std::max( + max_same_gpu_percentage, + lane.resource_percentage); + } + } + if (max_same_gpu_percentage > 0) { + const int reserved_cus = + (tool_spec_hip_probe_total_cus * + max_same_gpu_percentage + + 99) / + 100; + if (reserved_cus <= 0 || + reserved_cus >= tool_spec_hip_probe_total_cus) { + std::fprintf(stderr, + "[server] same-GPU HIP tool lane would reserve %d " + "of %d CUs; at least one model CU is required\n", + reserved_cus, tool_spec_hip_probe_total_cus); + return 2; + } + const std::string isolation = + std::to_string(tool_spec_hip_probe_device) + ":" + + std::to_string(reserved_cus); + const char * existing = + std::getenv("DFLASH_HIP_RESERVED_TOOL_LANE"); + if (existing && *existing && isolation != existing) { + std::fprintf(stderr, + "[server] DFLASH_HIP_RESERVED_TOOL_LANE=%s " + "conflicts with profile-required %s\n", + existing, isolation.c_str()); + return 2; + } + set_environment_variable( + "DFLASH_HIP_RESERVED_TOOL_LANE", + isolation.c_str(), true); + sconfig.tool_speculation.hip_tool_device = + tool_spec_hip_probe_device; + sconfig.tool_speculation.hip_reserved_tool_compute_units = + reserved_cus; + std::fprintf(stderr, + "[server] disjoint HIP tool lane: device %d reserves " + "%d/%d low CU(s); model streams use the complement\n", + tool_spec_hip_probe_device, reserved_cus, + tool_spec_hip_probe_total_cus); + } + } + if (sconfig.tool_speculation.policy.requires_static_model_routing() && + !sconfig.tool_speculation.model_routing_static) { + std::fprintf(stderr, + "[server] same-physical-GPU tool lanes require static model " + "routing; disable DFLASH_MOE_TP_DYNAMIC_ROUTE_BALANCE and " + "DFLASH_DS4_TP_DYNAMIC_ROUTE_BALANCE\n"); + return 2; + } + if (sconfig.tool_speculation.policy.requires_unique_expert_ownership() && + !sconfig.tool_speculation.model_expert_ownership_unique) { + std::fprintf(stderr, + "[server] same-physical-GPU tool lanes require unique expert " + "ownership; disable DFLASH_MOE_DUPLICATE_HOT_ON_COLD\n"); + return 2; + } + std::fprintf(stderr, + "[server] tool speculation preserves token speculation; " + "unqualified resource lanes are deferred\n"); + } if (fast_rollback_forced_off) { bargs.fast_rollback = false; target_split_fast_rollback_cli = false; @@ -1057,6 +1297,30 @@ int main(int argc, char ** argv) { std::fprintf(stderr, "[server] │ prefix_cache = %d slots\n", sconfig.prefix_cache_cap); std::fprintf(stderr, "[server] │ prefill_cache = %d slots\n", sconfig.prefill_cache_cap); std::fprintf(stderr, "[server] │ cors = %s\n", sconfig.enable_cors ? "ON" : "off"); + std::fprintf(stderr, "[server] │ tool_speculation= %s\n", + sconfig.tool_speculation.enabled() ? "ON" : "off"); + if (sconfig.tool_speculation.enabled()) { + std::fprintf(stderr, "[server] │ tool_spec_exec = %s\n", + sconfig.tool_speculation.execution_mode()); + std::fprintf(stderr, "[server] │ tool_spec_profile= %s\n", + sconfig.tool_speculation.profile_path.c_str()); + std::fprintf(stderr, "[server] │ tool_spec_decode = %s\n", + "spec preserved (unqualified lanes deferred)"); + std::fprintf(stderr, "[server] │ tool_spec_routing= %s\n", + sconfig.tool_speculation.model_routing_static + ? "static" : "dynamic"); + std::fprintf(stderr, "[server] │ tool_spec_experts= %s\n", + sconfig.tool_speculation.model_expert_ownership_unique + ? "unique ownership" : "duplicated ownership"); + std::fprintf(stderr, "[server] │ tool_spec_lanes ="); + for (const auto & lane : sconfig.tool_speculation.policy.lanes()) { + std::fprintf(stderr, " %d%%:%s", + lane.resource_percentage, + lane.decode_interference_qualified + ? "qualified" : "deferred"); + } + std::fprintf(stderr, "\n"); + } std::fprintf(stderr, "[server] │ cache_type_k = %s\n", #ifdef GGML_USE_HIP cache_type_k.empty() ? "q4_0 (default, HIP)" : cache_type_k.c_str()); diff --git a/server/src/server/tool_speculation.cpp b/server/src/server/tool_speculation.cpp new file mode 100644 index 000000000..f5fdd221f --- /dev/null +++ b/server/src/server/tool_speculation.cpp @@ -0,0 +1,984 @@ +#include "tool_speculation.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if !defined(_WIN32) +# include +# include +# include +# include +# include +# include +# include +extern char ** environ; +#endif + +namespace dflash::common { +namespace { + +constexpr size_t kMaxExecutorRequestBytes = 64 * 1024; + +bool finite_positive(double value) { + return std::isfinite(value) && value > 0.0; +} + +double median(std::vector values) { + if (values.empty()) return 0.0; + std::sort(values.begin(), values.end()); + const size_t middle = values.size() / 2; + if (values.size() % 2 != 0) return values[middle]; + return (values[middle - 1] + values[middle]) / 2.0; +} + +bool request_declares_tool(const json & tools, const std::string & name) { + if (!tools.is_array() || name.empty()) return false; + for (const auto & tool : tools) { + if (!tool.is_object()) continue; + if (tool.value("name", "") == name) return true; + if (tool.contains("function") && tool["function"].is_object() && + tool["function"].value("name", "") == name) { + return true; + } + } + return false; +} + +#if !defined(_WIN32) +bool send_all_socket(int fd, const void * data, size_t bytes) { + const char * cursor = static_cast(data); + while (bytes > 0) { + int flags = 0; +# if defined(MSG_NOSIGNAL) + flags = MSG_NOSIGNAL; +# endif + const ssize_t written = ::send(fd, cursor, bytes, flags); + if (written < 0) { + if (errno == EINTR) continue; + return false; + } + if (written == 0) return false; + cursor += written; + bytes -= static_cast(written); + } + return true; +} + +std::vector executor_environment( + int resource_percentage, + const std::string & accelerator_relation) { + std::vector values; + static constexpr const char * kResourceKey = + "DFLASH_TOOL_SPECULATION_RESOURCE_PERCENTAGE="; + static constexpr size_t kResourceKeyLen = + sizeof("DFLASH_TOOL_SPECULATION_RESOURCE_PERCENTAGE=") - 1; + static constexpr const char * kRelationKey = + "DFLASH_TOOL_SPECULATION_ACCELERATOR_RELATION="; + static constexpr size_t kRelationKeyLen = + sizeof("DFLASH_TOOL_SPECULATION_ACCELERATOR_RELATION=") - 1; + bool resource_replaced = false; + bool relation_replaced = false; + for (char ** item = environ; item && *item; ++item) { + const std::string value(*item); + if (value.compare(0, kResourceKeyLen, kResourceKey) == 0) { + values.push_back( + std::string(kResourceKey) + + std::to_string(resource_percentage)); + resource_replaced = true; + } else if (value.compare(0, kRelationKeyLen, kRelationKey) == 0) { + values.push_back( + std::string(kRelationKey) + accelerator_relation); + relation_replaced = true; + } else { + values.push_back(value); + } + } + if (!resource_replaced) { + values.push_back( + std::string(kResourceKey) + + std::to_string(resource_percentage)); + } + if (!relation_replaced) { + values.push_back(std::string(kRelationKey) + accelerator_relation); + } + values.push_back("DFLASH_TOOL_SPECULATION=1"); + return values; +} +#endif + +} // namespace + +bool CanonicalToolInvocation::from_parts( + const std::string & name, + const json & arguments, + CanonicalToolInvocation & out, + std::string & error) { + if (name.empty()) { + error = "tool name must not be empty"; + return false; + } + if (!arguments.is_object()) { + error = "tool arguments must be a JSON object"; + return false; + } + out.name = name; + out.arguments = arguments; + // nlohmann::json's default object type is key ordered, so dump() is a + // stable canonical identity independent of input object insertion order. + out.arguments_json = arguments.dump(); + error.clear(); + return true; +} + +bool CanonicalToolInvocation::from_tool_call( + const ToolCall & call, + CanonicalToolInvocation & out, + std::string & error) { + try { + const json arguments = call.arguments.empty() + ? json::object() + : json::parse(call.arguments); + return from_parts(call.name, arguments, out, error); + } catch (const std::exception & exception) { + error = std::string("authoritative tool arguments are invalid JSON: ") + + exception.what(); + return false; + } +} + +bool parse_tool_speculation_prediction( + const json & value, + const json & tools, + ToolSpeculationPrediction & out, + std::string & error) { + if (!value.is_object()) { + error = "tool_speculation must be an object"; + return false; + } + if (!value.contains("call") || !value["call"].is_object()) { + error = "tool_speculation.call must be an object"; + return false; + } + if (!value.contains("confidence") || !value["confidence"].is_number()) { + error = "tool_speculation.confidence must be a number"; + return false; + } + const double confidence = value["confidence"].get(); + if (!std::isfinite(confidence) || confidence < 0.0 || confidence > 1.0) { + error = "tool_speculation.confidence must be between 0 and 1"; + return false; + } + + const json & call = value["call"]; + if (!call.contains("name") || !call["name"].is_string()) { + error = "tool_speculation.call.name must be a string"; + return false; + } + if (!call.contains("arguments")) { + error = "tool_speculation.call.arguments is required"; + return false; + } + CanonicalToolInvocation invocation; + if (!CanonicalToolInvocation::from_parts( + call["name"].get(), call["arguments"], invocation, + error)) { + return false; + } + if (!request_declares_tool(tools, invocation.name)) { + error = "tool_speculation.call.name is not declared in tools"; + return false; + } + out.call = std::move(invocation); + out.confidence = confidence; + error.clear(); + return true; +} + +bool ToolSpeculationPolicy::load_file( + const std::string & path, std::string & error) { + std::ifstream input(path); + if (!input) { + error = "cannot open tool-speculation profile: " + path; + lanes_.clear(); + baseline_task_ms_ = 0.0; + profile_status_ = "qualified"; + executor_contract_.clear(); + return false; + } + try { + json report; + input >> report; + return load_json(report, error); + } catch (const std::exception & exception) { + error = std::string("invalid tool-speculation profile JSON: ") + + exception.what(); + lanes_.clear(); + baseline_task_ms_ = 0.0; + profile_status_ = "qualified"; + executor_contract_.clear(); + return false; + } +} + +bool ToolSpeculationPolicy::load_json( + const json & report, std::string & error) { + lanes_.clear(); + baseline_task_ms_ = 0.0; + profile_status_ = "qualified"; + executor_contract_.clear(); + if (!report.is_object() || !report.contains("path_summary") || + !report["path_summary"].is_object() || + report["path_summary"].empty()) { + error = "tool-speculation profile needs a non-empty path_summary"; + return false; + } + + if (report.contains("profile_status")) { + if (!report["profile_status"].is_string()) { + error = "tool-speculation profile_status must be a string"; + return false; + } + profile_status_ = report["profile_status"].get(); + if (profile_status_ != "qualified" && + profile_status_ != "provisional_benchmark_only") { + error = "tool-speculation profile_status must be qualified or " + "provisional_benchmark_only"; + return false; + } + } + if (report.contains("executor")) { + if (!report["executor"].is_string()) { + error = "tool-speculation executor contract must be a string"; + return false; + } + executor_contract_ = report["executor"].get(); + if (executor_contract_.empty()) { + error = "tool-speculation executor contract cannot be empty"; + return false; + } + } + + std::vector controls; + try { + for (auto item = report["path_summary"].begin(); + item != report["path_summary"].end(); ++item) { + size_t parsed = 0; + const int resource_percentage = std::stoi(item.key(), &parsed); + if (parsed != item.key().size() || + resource_percentage < 1 || resource_percentage > 100) { + throw std::runtime_error( + "invalid resource percentage " + item.key()); + } + const json & paths = item.value(); + const json & hit = paths.at("hit"); + const json & miss = paths.at("miss"); + const double hit_control = hit.at("control_task_mean_ms").get(); + const double miss_control = miss.at("control_task_mean_ms").get(); + const double hit_task = hit.at("speculative_task_mean_ms").get(); + const double miss_task = miss.at("speculative_task_mean_ms").get(); + const double slowdown_percent = std::max( + hit.at("model_slowdown_percent").get(), + miss.at("model_slowdown_percent").get()); + bool decode_interference_qualified = false; + if (paths.contains("decode_interference_qualified")) { + if (!paths["decode_interference_qualified"].is_boolean()) { + throw std::runtime_error( + "decode_interference_qualified must be boolean"); + } + decode_interference_qualified = + paths["decode_interference_qualified"].get(); + } + const std::string accelerator_relation = + paths.value("accelerator_relation", "unspecified"); + if (accelerator_relation != "unspecified" && + accelerator_relation != "non_accelerator" && + accelerator_relation != "separate_physical_gpu" && + accelerator_relation != "same_physical_gpu") { + throw std::runtime_error( + "accelerator_relation must be unspecified, " + "non_accelerator, separate_physical_gpu, or " + "same_physical_gpu"); + } + bool requires_static_model_routing = false; + if (paths.contains("requires_static_model_routing")) { + if (!paths["requires_static_model_routing"].is_boolean()) { + throw std::runtime_error( + "requires_static_model_routing must be boolean"); + } + requires_static_model_routing = + paths["requires_static_model_routing"].get(); + } + bool requires_unique_expert_ownership = false; + if (paths.contains("requires_unique_expert_ownership")) { + if (!paths["requires_unique_expert_ownership"].is_boolean()) { + throw std::runtime_error( + "requires_unique_expert_ownership must be boolean"); + } + requires_unique_expert_ownership = + paths["requires_unique_expert_ownership"].get(); + } + if (!finite_positive(hit_control) || + !finite_positive(miss_control) || + !finite_positive(hit_task) || + !finite_positive(miss_task) || + !std::isfinite(slowdown_percent) || slowdown_percent < -100.0) { + throw std::runtime_error("non-positive or non-finite profile latency"); + } + const double control = (hit_control + miss_control) / 2.0; + lanes_.push_back({ + resource_percentage, + control, + hit_task, + miss_task, + 1.0 + slowdown_percent / 100.0, + decode_interference_qualified, + accelerator_relation, + requires_static_model_routing, + requires_unique_expert_ownership, + }); + controls.push_back(control); + } + } catch (const std::exception & exception) { + error = std::string("invalid tool-speculation path_summary: ") + + exception.what(); + lanes_.clear(); + profile_status_ = "qualified"; + executor_contract_.clear(); + return false; + } + + std::sort(lanes_.begin(), lanes_.end(), + [](const auto & left, const auto & right) { + return left.resource_percentage < + right.resource_percentage; + }); + for (size_t index = 1; index < lanes_.size(); ++index) { + if (lanes_[index - 1].resource_percentage == + lanes_[index].resource_percentage) { + error = + "tool-speculation profile has duplicate resource percentages"; + lanes_.clear(); + return false; + } + } + baseline_task_ms_ = median(std::move(controls)); + error.clear(); + return true; +} + +bool ToolSpeculationPolicy::requires_static_model_routing() const { + return std::any_of( + lanes_.begin(), lanes_.end(), + [](const ToolSpeculationLane & lane) { + return lane.requires_static_model_routing; + }); +} + +bool ToolSpeculationPolicy::requires_unique_expert_ownership() const { + return std::any_of( + lanes_.begin(), lanes_.end(), + [](const ToolSpeculationLane & lane) { + return lane.requires_unique_expert_ownership; + }); +} + +ToolSpeculationAdmission ToolSpeculationPolicy::choose( + double confidence, + double max_model_slowdown_ratio) const { + ToolSpeculationAdmission decision; + decision.expected_task_ms = baseline_task_ms_; + if (lanes_.empty() || !finite_positive(baseline_task_ms_)) { + decision.reason = "profile_unavailable"; + return decision; + } + if (!std::isfinite(confidence) || confidence < 0.0 || confidence > 1.0) { + decision.reason = "invalid_confidence"; + return decision; + } + if (!std::isfinite(max_model_slowdown_ratio) || + max_model_slowdown_ratio < 1.0) { + decision.reason = "invalid_slowdown_guardrail"; + return decision; + } + + bool qualified_lane_available = false; + bool lane_passed_guardrail = false; + double best = baseline_task_ms_; + for (const ToolSpeculationLane & lane : lanes_) { + // Token speculation is an invariant, not a fallback choice. A lane + // may overlap DS4/DSpark only after its exact executor and placement + // passed the output-identity interference gate. + if (!lane.decode_interference_qualified) continue; + qualified_lane_available = true; + if (lane.model_slowdown_ratio > max_model_slowdown_ratio) continue; + lane_passed_guardrail = true; + const double expected = + confidence * lane.hit_task_ms + + (1.0 - confidence) * lane.miss_task_ms; + if (expected < best) { + best = expected; + decision.admitted = true; + decision.resource_percentage = lane.resource_percentage; + decision.expected_task_ms = expected; + decision.decode_interference_qualified = + lane.decode_interference_qualified; + decision.accelerator_relation = lane.accelerator_relation; + } + } + if (!decision.admitted) { + decision.reason = !qualified_lane_available + ? "decode_interference_unqualified" + : lane_passed_guardrail + ? "below_profile_break_even" + : "model_slowdown_guardrail"; + return decision; + } + decision.expected_speedup = baseline_task_ms_ / best; + decision.reason = "expected_latency_gain"; + return decision; +} + +bool ToolSpeculationConfig::allows(const std::string & name) const { + return std::find(allowed_tools.begin(), allowed_tools.end(), name) != + allowed_tools.end(); +} + +ToolSpeculationAttempt::ToolSpeculationAttempt( + const ToolSpeculationConfig & config, + const ToolSpeculationPrediction & prediction, + const std::string & request_id) + : config_(config) + , prediction_(prediction) + , request_id_(request_id) { + if (!config_.enabled()) { + admission_.reason = "engine_disabled"; + } else if (!config_.allows(prediction_.call.name)) { + admission_.reason = "tool_not_allowlisted"; + } else { + admission_ = config_.policy.choose( + prediction_.confidence, config_.max_model_slowdown_ratio); + } +} + +ToolSpeculationAttempt::~ToolSpeculationAttempt() { + if (!resolved_) terminate_executor(); +} + +std::unique_ptr ToolSpeculationAttempt::create( + const ToolSpeculationConfig & config, + const ToolSpeculationPrediction & prediction, + const std::string & request_id) { + return std::unique_ptr( + new ToolSpeculationAttempt(config, prediction, request_id)); +} + +void ToolSpeculationAttempt::start() { + if (started_) return; + started_ = true; + if (!admission_.admitted) return; + const json request = { + {"protocol", "dflash.tool-speculation.v1"}, + {"request_id", request_id_}, + {"mode", "speculative"}, + {"resource_percentage", admission_.resource_percentage}, + {"accelerator_relation", admission_.accelerator_relation}, + {"call", { + {"name", prediction_.call.name}, + {"arguments", prediction_.call.arguments}, + }}, + }; + started_at_ = std::chrono::steady_clock::now(); + if (config_.in_process_executor) { + in_process_execution_ = + config_.in_process_executor->start(request, launch_error_); + running_ = static_cast(in_process_execution_); + if (running_) { + std::fprintf(stderr, + "[tool-spec] launched request=%s tool=%s confidence=%.3f " + "resource=%d%% mode=%s\n", + request_id_.c_str(), prediction_.call.name.c_str(), + prediction_.confidence, admission_.resource_percentage, + config_.execution_mode()); + } + return; + } +#if defined(_WIN32) + launch_error_ = "tool speculation child executors are not implemented on Windows"; + return; +#else + const std::string payload = request.dump() + "\n"; + if (payload.size() > kMaxExecutorRequestBytes) { + launch_error_ = "executor request exceeds 64 KiB"; + return; + } + + int input_socket[2] = {-1, -1}; + if (::socketpair(AF_UNIX, SOCK_STREAM, 0, input_socket) != 0) { + launch_error_ = std::string("executor stdin socket failed: ") + + std::strerror(errno); + return; + } +# if defined(SO_NOSIGPIPE) + int no_sigpipe = 1; + ::setsockopt(input_socket[0], SOL_SOCKET, SO_NOSIGPIPE, + &no_sigpipe, sizeof(no_sigpipe)); +# endif + int output_pipe[2] = {-1, -1}; + if (::pipe(output_pipe) != 0) { + launch_error_ = std::string("executor stdout pipe failed: ") + + std::strerror(errno); + ::close(input_socket[0]); + ::close(input_socket[1]); + return; + } + + posix_spawn_file_actions_t actions; + int spawn_status = posix_spawn_file_actions_init(&actions); + const bool actions_initialized = spawn_status == 0; + if (spawn_status == 0) { + spawn_status = posix_spawn_file_actions_adddup2( + &actions, input_socket[1], STDIN_FILENO); + } + if (spawn_status == 0) { + spawn_status = posix_spawn_file_actions_adddup2( + &actions, output_pipe[1], STDOUT_FILENO); + } + if (spawn_status == 0) { + spawn_status = posix_spawn_file_actions_addclose(&actions, output_pipe[0]); + } + if (spawn_status == 0) { + spawn_status = posix_spawn_file_actions_addclose(&actions, input_socket[0]); + } + if (spawn_status == 0 && input_socket[1] != STDIN_FILENO) { + spawn_status = posix_spawn_file_actions_addclose(&actions, input_socket[1]); + } + if (spawn_status == 0 && output_pipe[1] != STDOUT_FILENO) { + spawn_status = posix_spawn_file_actions_addclose(&actions, output_pipe[1]); + } + + std::vector env_storage = executor_environment( + admission_.resource_percentage, admission_.accelerator_relation); + std::vector env; + env.reserve(env_storage.size() + 1); + for (std::string & value : env_storage) env.push_back(value.data()); + env.push_back(nullptr); + + std::string executable = config_.executor_path; + std::string protocol_arg = "--dflash-tool-spec-v1"; + char * argv[] = { + executable.data(), + protocol_arg.data(), + nullptr, + }; + pid_t child = -1; + if (spawn_status == 0) { + spawn_status = ::posix_spawn( + &child, executable.c_str(), &actions, nullptr, argv, env.data()); + } + if (actions_initialized) { + posix_spawn_file_actions_destroy(&actions); + } + ::close(input_socket[1]); + ::close(output_pipe[1]); + if (spawn_status != 0) { + launch_error_ = std::string("executor spawn failed: ") + + std::strerror(spawn_status); + ::close(input_socket[0]); + ::close(output_pipe[0]); + return; + } + + child_pid_ = static_cast(child); + child_stdin_fd_ = input_socket[0]; + child_stdout_fd_ = output_pipe[0]; + const int flags = ::fcntl(child_stdout_fd_, F_GETFL, 0); + if (flags >= 0) { + ::fcntl(child_stdout_fd_, F_SETFL, flags | O_NONBLOCK); + } + running_ = true; + if (!send_all_socket(child_stdin_fd_, payload.data(), payload.size())) { + launch_error_ = std::string("executor request write failed: ") + + std::strerror(errno); + terminate_executor(); + return; + } + std::fprintf(stderr, + "[tool-spec] launched request=%s tool=%s confidence=%.3f " + "resource=%d%%\n", + request_id_.c_str(), prediction_.call.name.c_str(), + prediction_.confidence, admission_.resource_percentage); +#endif +} + +json ToolSpeculationAttempt::base_metadata() const { + json metadata = { + {"protocol", "dflash.tool-speculation.v1"}, + {"confidence", prediction_.confidence}, + {"prediction", { + {"name", prediction_.call.name}, + {"arguments", prediction_.call.arguments}, + }}, + {"resource_percentage", + admission_.admitted + ? json(admission_.resource_percentage) + : json(nullptr)}, + {"expected_speedup", + admission_.admitted ? json(admission_.expected_speedup) : json(nullptr)}, + {"decode_interference_qualified", + admission_.admitted + ? json(admission_.decode_interference_qualified) + : json(nullptr)}, + {"accelerator_relation", + admission_.admitted + ? json(admission_.accelerator_relation) + : json(nullptr)}, + }; + return metadata; +} + +bool ToolSpeculationAttempt::send_control(const char * operation) { + if (in_process_execution_) { + return operation && *operation && + in_process_execution_->send_control(operation); + } +#if defined(_WIN32) + (void)operation; + return false; +#else + if (child_stdin_fd_ < 0 || !operation || !*operation) return false; + const std::string command = json({ + {"protocol", "dflash.tool-speculation.v1"}, + {"request_id", request_id_}, + {"op", operation}, + {"authoritative_resource_percentage", 100}, + }).dump() + "\n"; + return send_all_socket( + child_stdin_fd_, command.data(), command.size()); +#endif +} + +void ToolSpeculationAttempt::terminate_executor(bool allow_control_grace) { + if (in_process_execution_) { + in_process_execution_->terminate(allow_control_grace); + in_process_execution_.reset(); + running_ = false; + return; + } +#if !defined(_WIN32) + if (child_stdin_fd_ >= 0) { + ::close(child_stdin_fd_); + child_stdin_fd_ = -1; + } + if (child_stdout_fd_ >= 0) { + ::close(child_stdout_fd_); + child_stdout_fd_ = -1; + } + if (child_pid_ > 0) { + const pid_t pid = static_cast(child_pid_); + auto wait_until = [&](std::chrono::steady_clock::time_point deadline) { + int status = 0; + while (std::chrono::steady_clock::now() < deadline) { + const pid_t waited = ::waitpid(pid, &status, WNOHANG); + if (waited == pid || (waited < 0 && errno == ECHILD)) { + child_pid_ = -1; + running_ = false; + return true; + } + if (waited < 0 && errno != EINTR) break; + std::this_thread::sleep_for(std::chrono::milliseconds(2)); + } + return false; + }; + const int grace_ms = std::max(0, config_.cancel_grace_ms); + if (allow_control_grace && wait_until( + std::chrono::steady_clock::now() + + std::chrono::milliseconds(grace_ms))) { + return; + } + ::kill(pid, SIGTERM); + const int term_grace_ms = allow_control_grace + ? std::min(20, grace_ms) : grace_ms; + if (wait_until(std::chrono::steady_clock::now() + + std::chrono::milliseconds(term_grace_ms))) { + return; + } + int status = 0; + ::kill(pid, SIGKILL); + while (::waitpid(pid, &status, 0) < 0 && errno == EINTR) {} + child_pid_ = -1; + } +#endif + running_ = false; +} + +bool ToolSpeculationAttempt::collect_executor_result( + json & result, + double & wait_ms, + std::string & error) { + if (in_process_execution_) { + const bool ok = in_process_execution_->collect_result( + config_.timeout_ms, config_.max_result_bytes, + result, wait_ms, error); + in_process_execution_.reset(); + running_ = false; + return ok; + } +#if defined(_WIN32) + (void)result; + wait_ms = 0.0; + error = "tool speculation executors are not implemented on Windows"; + return false; +#else + const auto wait_started = std::chrono::steady_clock::now(); + const auto deadline = wait_started + + std::chrono::milliseconds(std::max(1, config_.timeout_ms)); + std::string output; + bool eof = false; + while (!eof) { + const auto now = std::chrono::steady_clock::now(); + if (now >= deadline) { + error = "executor_timeout"; + terminate_executor(); + wait_ms = std::chrono::duration( + std::chrono::steady_clock::now() - wait_started).count(); + return false; + } + const int remaining_ms = std::max(1, static_cast( + std::chrono::duration_cast( + deadline - now).count())); + pollfd descriptor{child_stdout_fd_, POLLIN | POLLHUP, 0}; + const int polled = ::poll(&descriptor, 1, remaining_ms); + if (polled < 0) { + if (errno == EINTR) continue; + error = std::string("executor_poll_failed: ") + std::strerror(errno); + terminate_executor(); + return false; + } + if (polled == 0) continue; + if (descriptor.revents & (POLLERR | POLLNVAL)) { + error = "executor_stdout_failed"; + terminate_executor(); + return false; + } + if (descriptor.revents & (POLLIN | POLLHUP)) { + char buffer[8192]; + while (true) { + const ssize_t count = ::read( + child_stdout_fd_, buffer, sizeof(buffer)); + if (count > 0) { + if (output.size() + static_cast(count) > + config_.max_result_bytes) { + error = "executor_result_too_large"; + terminate_executor(); + return false; + } + output.append(buffer, static_cast(count)); + continue; + } + if (count == 0) { + eof = true; + break; + } + if (errno == EINTR) continue; + if (errno == EAGAIN || errno == EWOULDBLOCK) break; + error = std::string("executor_read_failed: ") + + std::strerror(errno); + terminate_executor(); + return false; + } + } + } + ::close(child_stdout_fd_); + child_stdout_fd_ = -1; + + int child_status = 0; + while (true) { + const pid_t waited = ::waitpid( + static_cast(child_pid_), &child_status, WNOHANG); + if (waited == static_cast(child_pid_)) break; + if (waited < 0) { + if (errno == EINTR) continue; + error = std::string("executor_wait_failed: ") + + std::strerror(errno); + child_pid_ = -1; + running_ = false; + return false; + } + if (std::chrono::steady_clock::now() >= deadline) { + error = "executor_exit_timeout"; + terminate_executor(); + return false; + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + child_pid_ = -1; + running_ = false; + wait_ms = std::chrono::duration( + std::chrono::steady_clock::now() - wait_started).count(); + if (!WIFEXITED(child_status) || WEXITSTATUS(child_status) != 0) { + error = WIFEXITED(child_status) + ? "executor_exit_" + std::to_string(WEXITSTATUS(child_status)) + : "executor_terminated"; + return false; + } + + try { + const json envelope = json::parse(output); + if (!envelope.is_object() || !envelope.value("ok", false) || + !envelope.contains("result")) { + error = "executor_rejected_or_invalid_envelope"; + return false; + } + result = envelope["result"]; + error.clear(); + return true; + } catch (const std::exception & exception) { + error = std::string("executor_invalid_json: ") + exception.what(); + return false; + } +#endif +} + +json ToolSpeculationAttempt::resolve( + const std::vector & authoritative_calls) { + if (resolved_) { + json metadata = base_metadata(); + metadata["status"] = "failed"; + metadata["reason"] = "already_resolved"; + return metadata; + } + resolved_ = true; + json metadata = base_metadata(); + if (!admission_.admitted) { + metadata["status"] = "deferred"; + metadata["reason"] = admission_.reason; + return metadata; + } + if (!launch_error_.empty() || !running_) { + metadata["status"] = "failed"; + metadata["reason"] = "executor_launch_failed"; + metadata["detail"] = launch_error_.empty() + ? "executor did not start" : launch_error_; + terminate_executor(); + return metadata; + } + if (authoritative_calls.size() != 1) { + send_control("cancel"); + terminate_executor(true); + metadata["status"] = "miss"; + metadata["reason"] = "authoritative_call_count"; + return metadata; + } + + CanonicalToolInvocation authoritative; + std::string canonical_error; + if (!CanonicalToolInvocation::from_tool_call( + authoritative_calls[0], authoritative, canonical_error)) { + send_control("cancel"); + terminate_executor(true); + metadata["status"] = "miss"; + metadata["reason"] = "invalid_authoritative_call"; + return metadata; + } + if (!(authoritative == prediction_.call)) { + send_control("cancel"); + terminate_executor(true); + metadata["status"] = "miss"; + metadata["reason"] = "invocation_mismatch"; + return metadata; + } + + json result; + double wait_ms = 0.0; + std::string executor_error; + const bool commit_signal_sent = send_control("commit"); +#if !defined(_WIN32) + if (child_stdin_fd_ >= 0) { + ::close(child_stdin_fd_); + child_stdin_fd_ = -1; + } +#endif + if (!collect_executor_result(result, wait_ms, executor_error)) { + metadata["status"] = "failed"; + metadata["reason"] = "speculative_executor_failure"; + metadata["detail"] = executor_error; + metadata["commit_signal_sent"] = commit_signal_sent; + metadata["commit_wait_ms"] = wait_ms; + return metadata; + } + const double wall_ms = std::chrono::duration( + std::chrono::steady_clock::now() - started_at_).count(); + metadata["status"] = "hit"; + metadata["call_id"] = authoritative_calls[0].id; + metadata["result"] = std::move(result); + metadata["commit_signal_sent"] = commit_signal_sent; + metadata["executor_wall_ms"] = wall_ms; + metadata["commit_wait_ms"] = wait_ms; + std::fprintf(stderr, + "[tool-spec] hit request=%s tool=%s resource=%d%% " + "wall_ms=%.1f wait_ms=%.1f\n", + request_id_.c_str(), prediction_.call.name.c_str(), + admission_.resource_percentage, wall_ms, wait_ms); + return metadata; +} + +json ToolSpeculationAttempt::cancel(const std::string & reason) { + if (!resolved_) { + resolved_ = true; + send_control("cancel"); + terminate_executor(true); + } + json metadata = base_metadata(); + metadata["status"] = "cancelled"; + metadata["reason"] = reason; + return metadata; +} + +std::string render_tool_speculation_sse( + ApiFormat api_format, + const std::string & request_id, + const std::string & model, + const json & metadata) { + switch (api_format) { + case ApiFormat::OPENAI_CHAT: { + const json event = { + {"id", request_id}, + {"object", "chat.completion.chunk"}, + {"model", model}, + {"choices", json::array()}, + {"dflash_tool_speculation", metadata}, + }; + return "data: " + event.dump() + "\n\n"; + } + case ApiFormat::ANTHROPIC: { + const json event = { + {"type", "dflash_tool_speculation"}, + {"dflash_tool_speculation", metadata}, + }; + return "event: dflash_tool_speculation\ndata: " + + event.dump() + "\n\n"; + } + case ApiFormat::RESPONSES: { + const json event = { + {"type", "response.dflash_tool_speculation"}, + {"response_id", request_id}, + {"dflash_tool_speculation", metadata}, + }; + return "event: response.dflash_tool_speculation\ndata: " + + event.dump() + "\n\n"; + } + default: + return "data: " + json({{"dflash_tool_speculation", metadata}}).dump() + + "\n\n"; + } +} + +} // namespace dflash::common diff --git a/server/src/server/tool_speculation.h b/server/src/server/tool_speculation.h new file mode 100644 index 000000000..955c337d8 --- /dev/null +++ b/server/src/server/tool_speculation.h @@ -0,0 +1,246 @@ +// Lossless, confidence-gated speculative tool execution. +// +// The model remains authoritative. A predicted read-only invocation may run +// while inference is in flight, but its result is returned only when the +// emitted tool name and canonical JSON arguments match exactly. + +#pragma once + +#include "api_types.h" +#include "tool_parser.h" + +#include + +#include +#include +#include +#include +#include + +namespace dflash::common { + +using json = nlohmann::json; + +struct CanonicalToolInvocation { + std::string name; + json arguments = json::object(); + std::string arguments_json; + + static bool from_parts(const std::string & name, + const json & arguments, + CanonicalToolInvocation & out, + std::string & error); + static bool from_tool_call(const ToolCall & call, + CanonicalToolInvocation & out, + std::string & error); + + bool operator==(const CanonicalToolInvocation & other) const { + return name == other.name && arguments_json == other.arguments_json; + } +}; + +struct ToolSpeculationPrediction { + CanonicalToolInvocation call; + double confidence = 0.0; +}; + +// Parse the request extension: +// "tool_speculation": { +// "call": {"name": "...", "arguments": {...}}, +// "confidence": 0.0..1.0 +// } +// The predicted tool must also be present in the request's `tools` array. +bool parse_tool_speculation_prediction(const json & value, + const json & tools, + ToolSpeculationPrediction & out, + std::string & error); + +struct ToolSpeculationLane { + // Backend-neutral executor capacity. A CUDA adapter may map this to an + // MPS share; a ROCm, CPU, I/O, or remote adapter may interpret it using + // its own measured resource contract. + int resource_percentage = 0; + double control_task_ms = 0.0; + double hit_task_ms = 0.0; + double miss_task_ms = 0.0; + double model_slowdown_ratio = 1.0; + // True only when this exact executor/resource lane passed the model-output + // interference gate. Missing profile metadata defers tool speculation; + // token speculation is never disabled or replaced with AR decode. + bool decode_interference_qualified = false; + // Physical relationship between the tool accelerator and the model's + // primary accelerator. Same-GPU lanes have stricter runtime requirements + // because stream priority and CU masks do not isolate shared kernels. + std::string accelerator_relation = "unspecified"; + bool requires_static_model_routing = false; + bool requires_unique_expert_ownership = false; +}; + +struct ToolSpeculationAdmission { + bool admitted = false; + int resource_percentage = 0; + double expected_task_ms = 0.0; + double expected_speedup = 1.0; + bool decode_interference_qualified = false; + std::string accelerator_relation = "unspecified"; + std::string reason; +}; + +// Optional trusted in-process executor. This avoids a second accelerator +// process/context on runtimes where process-level time-slicing defeats CU or +// stream isolation. Implementations remain behind the same allowlist, +// empirical admission policy, exact-call commit, and private-result boundary +// as the child-process adapter. +class ToolSpeculationExecution { +public: + virtual ~ToolSpeculationExecution() = default; + virtual bool send_control(const std::string & operation) = 0; + virtual bool collect_result(int timeout_ms, + size_t max_result_bytes, + json & result, + double & wait_ms, + std::string & error) = 0; + virtual void terminate(bool allow_control_grace) = 0; +}; + +class ToolSpeculationExecutor { +public: + virtual ~ToolSpeculationExecutor() = default; + virtual std::unique_ptr start( + const json & request, + std::string & error) = 0; + virtual const char * mode_name() const = 0; +}; + +// Runtime policy loaded from a qualification report's `path_summary`. This +// keeps backend-specific interference measurements out of hard-coded engine +// heuristics. +class ToolSpeculationPolicy { +public: + bool load_file(const std::string & path, std::string & error); + bool load_json(const json & report, std::string & error); + + ToolSpeculationAdmission choose( + double confidence, + double max_model_slowdown_ratio) const; + + bool empty() const { return lanes_.empty(); } + double baseline_task_ms() const { return baseline_task_ms_; } + const std::vector & lanes() const { return lanes_; } + const std::string & profile_status() const { return profile_status_; } + const std::string & executor_contract() const { return executor_contract_; } + bool benchmark_only() const { + return profile_status_ == "provisional_benchmark_only"; + } + bool requires_static_model_routing() const; + bool requires_unique_expert_ownership() const; + +private: + std::vector lanes_; + double baseline_task_ms_ = 0.0; + std::string profile_status_ = "qualified"; + std::string executor_contract_; +}; + +struct ToolSpeculationConfig { + std::string executor_path; + std::shared_ptr in_process_executor; + std::string profile_path; + std::vector allowed_tools; + ToolSpeculationPolicy policy; + int timeout_ms = 60000; + int cancel_grace_ms = 100; + size_t max_result_bytes = 1024 * 1024; + double max_model_slowdown_ratio = 1.20; + // Snapshot of the model routing mode used to validate profile/runtime + // compatibility at startup and expose it through /props. + bool model_routing_static = true; + bool model_expert_ownership_unique = true; + // Runtime evidence that the model and an in-process HIP tool use + // complementary CU masks. Zero means no model-side CU reservation. + int hip_tool_device = -1; + int hip_reserved_tool_compute_units = 0; + bool enabled() const { + return (!executor_path.empty() || in_process_executor) && + !allowed_tools.empty() && + !policy.empty(); + } + const char * execution_mode() const { + return in_process_executor + ? in_process_executor->mode_name() + : executor_path.empty() ? "disabled" : "child_process"; + } + bool allows(const std::string & name) const; +}; + +// One request-scoped attempt. The configured executable is invoked without a +// shell and receives one JSON request on stdin. It must emit one JSON envelope +// on stdout: {"ok":true,"result":...}. Stdin remains open for a later +// `commit` (exact match; promote checkpointed remainder to the authoritative +// 100% lane) or `cancel` control record. A thin executable may forward this +// protocol to a persistent warm tool pool, keeping GPU initialization outside +// the request's critical path. +class ToolSpeculationAttempt { +public: + ToolSpeculationAttempt(const ToolSpeculationAttempt &) = delete; + ToolSpeculationAttempt & operator=(const ToolSpeculationAttempt &) = delete; + ~ToolSpeculationAttempt(); + + static std::unique_ptr create( + const ToolSpeculationConfig & config, + const ToolSpeculationPrediction & prediction, + const std::string & request_id); + + // Launch admitted work. Deferred and launch-failed attempts still return + // metadata through resolve(), so an opted-in client can see why it must + // execute the authoritative tool normally. + void start(); + + // Exact-match one authoritative call, expose a successful private result, + // or discard/cancel it. This method is single-use. + json resolve(const std::vector & authoritative_calls); + + // Cancel without exposing a result (disconnect, generation failure, etc.). + json cancel(const std::string & reason); + + bool admitted() const { return admission_.admitted; } + bool running() const { return running_; } +private: + ToolSpeculationAttempt(const ToolSpeculationConfig & config, + const ToolSpeculationPrediction & prediction, + const std::string & request_id); + + json base_metadata() const; + bool send_control(const char * operation); + void terminate_executor(bool allow_control_grace = false); + bool collect_executor_result(json & result, + double & wait_ms, + std::string & error); + + ToolSpeculationConfig config_; + ToolSpeculationPrediction prediction_; + std::string request_id_; + ToolSpeculationAdmission admission_; + std::chrono::steady_clock::time_point started_at_{}; + bool started_ = false; + bool running_ = false; + bool resolved_ = false; + std::string launch_error_; + std::unique_ptr in_process_execution_; + +#if !defined(_WIN32) + int child_stdin_fd_ = -1; + int child_stdout_fd_ = -1; + int child_pid_ = -1; +#endif +}; + +// Custom SSE extension emitted only for requests that supplied +// `tool_speculation`. Non-streaming responses use the same object under the +// top-level `dflash_tool_speculation` key. +std::string render_tool_speculation_sse(ApiFormat api_format, + const std::string & request_id, + const std::string & model, + const json & metadata); + +} // namespace dflash::common diff --git a/server/src/server/tool_speculation_hip_probe.cpp b/server/src/server/tool_speculation_hip_probe.cpp new file mode 100644 index 000000000..3c6155617 --- /dev/null +++ b/server/src/server/tool_speculation_hip_probe.cpp @@ -0,0 +1,456 @@ +#include "tool_speculation_hip_probe.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace dflash::common { +namespace { + +std::string hip_error(const char * operation, hipError_t status) { + return std::string(operation) + ": " + hipGetErrorString(status); +} + +std::string hipblas_error(const char * operation, hipblasStatus_t status) { + return std::string(operation) + " failed with status " + + std::to_string(static_cast(status)); +} + +// HIP's current device is thread-local process state. The HTTP worker that +// launches a tool may immediately continue into model inference, so a trusted +// executor must leave that state exactly as it found it. +class ScopedHipDevice final { +public: + explicit ScopedHipDevice(int device) { + status_ = hipGetDevice(&previous_device_); + if (status_ != hipSuccess) return; + if (previous_device_ == device) return; + status_ = hipSetDevice(device); + switched_ = status_ == hipSuccess; + } + + ~ScopedHipDevice() { + if (switched_) (void) hipSetDevice(previous_device_); + } + + bool ok() const { return status_ == hipSuccess; } + hipError_t status() const { return status_; } + +private: + int previous_device_ = 0; + hipError_t status_ = hipSuccess; + bool switched_ = false; +}; + +class HipSgemmState; + +class HipSgemmExecution final : public ToolSpeculationExecution { +public: + explicit HipSgemmExecution(std::shared_ptr state) + : state_(std::move(state)) {} + ~HipSgemmExecution() override; + + bool send_control(const std::string & operation) override; + bool collect_result(int timeout_ms, + size_t max_result_bytes, + json & result, + double & wait_ms, + std::string & error) override; + void terminate(bool allow_control_grace) override; + +private: + std::shared_ptr state_; + bool committed_ = false; + bool finished_ = false; +}; + +class HipSgemmState final { +public: + HipSgemmState(int device, int matrix_size, int total_cus) + : device_(device), matrix_size_(matrix_size), total_cus_(total_cus) {} + + ~HipSgemmState() { + std::lock_guard lock(mutex_); + ScopedHipDevice device(device_); + if (!device.ok()) return; + if (stream_) (void) hipStreamSynchronize(stream_); + if (finished_) (void) hipEventDestroy(finished_); + if (started_) (void) hipEventDestroy(started_); + if (handle_) (void) hipblasDestroy(handle_); + if (c_) (void) hipFree(c_); + if (b_) (void) hipFree(b_); + if (a_) (void) hipFree(a_); + if (stream_) (void) hipStreamDestroy(stream_); + } + + bool start(const json & request, std::string & error) { + ScopedHipDevice device(device_); + if (!device.ok()) { + error = hip_error("hipSetDevice(tool)", device.status()); + return false; + } + std::lock_guard lock(mutex_); + if (active_) { + error = "HIP probe already has active work"; + return false; + } + try { + const json & call = request.at("call"); + if (call.at("name").get() != "benchmark_hip_sgemm") { + error = "HIP probe only supports benchmark_hip_sgemm"; + return false; + } + const json & arguments = call.at("arguments"); + if (!arguments.is_object() || + !arguments.contains("iterations") || + !arguments["iterations"].is_number_integer()) { + error = "benchmark_hip_sgemm.iterations must be an integer"; + return false; + } + iterations_ = arguments["iterations"].get(); + if (iterations_ <= 0 || iterations_ > 1'000'000) { + error = "benchmark_hip_sgemm.iterations must be 1..1000000"; + return false; + } + const int resource_percentage = + request.at("resource_percentage").get(); + if (resource_percentage <= 0 || resource_percentage > 100) { + error = "resource_percentage must be 1..100"; + return false; + } + const int cu_count = std::clamp( + (total_cus_ * resource_percentage + 99) / 100, + 1, total_cus_); + if (!ensure_resources(cu_count, error)) return false; + + const float alpha = 1.0F; + const float beta = 0.0F; + hipError_t status = hipEventRecord(started_, stream_); + if (status != hipSuccess) { + error = hip_error("hipEventRecord(started)", status); + return false; + } + for (int iteration = 0; iteration < iterations_; ++iteration) { + const hipblasStatus_t blas_status = hipblasSgemm( + handle_, HIPBLAS_OP_N, HIPBLAS_OP_N, + matrix_size_, matrix_size_, matrix_size_, + &alpha, a_, matrix_size_, b_, matrix_size_, + &beta, c_, matrix_size_); + if (blas_status != HIPBLAS_STATUS_SUCCESS) { + error = hipblas_error("hipblasSgemm", blas_status); + (void) hipStreamSynchronize(stream_); + return false; + } + } + status = hipEventRecord(finished_, stream_); + if (status != hipSuccess) { + error = hip_error("hipEventRecord(finished)", status); + (void) hipStreamSynchronize(stream_); + return false; + } + active_ = true; + error.clear(); + return true; + } catch (const std::exception & exception) { + error = std::string("invalid HIP probe request: ") + exception.what(); + return false; + } + } + + bool collect(int timeout_ms, + size_t max_result_bytes, + json & result, + double & wait_ms, + std::string & error) { + ScopedHipDevice device(device_); + if (!device.ok()) { + error = hip_error("hipSetDevice(tool)", device.status()); + release_active(); + return false; + } + const auto wait_started = std::chrono::steady_clock::now(); + const auto deadline = wait_started + + std::chrono::milliseconds(std::max(1, timeout_ms)); + while (true) { + const hipError_t status = hipEventQuery(finished_); + if (status == hipSuccess) break; + if (status != hipErrorNotReady) { + error = hip_error("hipEventQuery", status); + release_active(); + return false; + } + if (std::chrono::steady_clock::now() >= deadline) { + error = "executor_timeout"; + synchronize_and_release(); + wait_ms = std::chrono::duration( + std::chrono::steady_clock::now() - wait_started).count(); + return false; + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + + float gpu_ms = 0.0F; + hipError_t status = hipEventElapsedTime(&gpu_ms, started_, finished_); + if (status != hipSuccess) { + error = hip_error("hipEventElapsedTime", status); + release_active(); + return false; + } + float sample = 0.0F; + status = hipMemcpyAsync( + &sample, c_, sizeof(sample), hipMemcpyDeviceToHost, stream_); + if (status == hipSuccess) status = hipStreamSynchronize(stream_); + if (status != hipSuccess || !std::isfinite(sample)) { + error = status == hipSuccess + ? "HIP probe produced a non-finite sample" + : hip_error("HIP probe result copy", status); + release_active(); + return false; + } + result = { + {"sample", sample}, + {"gpu_ms", gpu_ms}, + {"iterations", iterations_}, + {"matrix_size", matrix_size_}, + {"cu_count", current_cu_count_}, + }; + if (result.dump().size() > max_result_bytes) { + error = "executor_result_too_large"; + release_active(); + return false; + } + wait_ms = std::chrono::duration( + std::chrono::steady_clock::now() - wait_started).count(); + release_active(); + error.clear(); + return true; + } + + void synchronize_and_release() { + ScopedHipDevice device(device_); + std::lock_guard lock(mutex_); + if (device.ok() && active_ && stream_) { + (void) hipStreamSynchronize(stream_); + } + active_ = false; + } + +private: + bool ensure_resources(int cu_count, std::string & error) { + hipError_t status = hipSuccess; + if (!stream_ || !handle_ || current_cu_count_ != cu_count) { + if (stream_) { + (void) hipStreamSynchronize(stream_); + if (handle_) { + (void) hipblasDestroy(handle_); + handle_ = nullptr; + } + (void) hipStreamDestroy(stream_); + stream_ = nullptr; + } + const size_t mask_words = + static_cast((total_cus_ + 31) / 32); + std::vector mask(mask_words, 0); + for (int cu = 0; cu < cu_count; ++cu) { + mask[static_cast(cu / 32)] |= + uint32_t{1} << (cu % 32); + } + status = hipExtStreamCreateWithCUMask( + &stream_, static_cast(mask.size()), mask.data()); + if (status != hipSuccess) { + error = hip_error("hipExtStreamCreateWithCUMask", status); + return false; + } + const hipblasStatus_t create_status = hipblasCreate(&handle_); + if (create_status != HIPBLAS_STATUS_SUCCESS) { + error = hipblas_error("hipblasCreate", create_status); + return false; + } + const hipblasStatus_t stream_status = + hipblasSetStream(handle_, stream_); + if (stream_status != HIPBLAS_STATUS_SUCCESS) { + error = hipblas_error("hipblasSetStream", stream_status); + return false; + } + current_cu_count_ = cu_count; + } + if (!a_ || !b_ || !c_) { + if (c_) (void) hipFree(c_); + if (b_) (void) hipFree(b_); + if (a_) (void) hipFree(a_); + a_ = nullptr; + b_ = nullptr; + c_ = nullptr; + const size_t elements = + static_cast(matrix_size_) * matrix_size_; + const size_t bytes = elements * sizeof(float); + if ((status = hipMalloc(&a_, bytes)) != hipSuccess || + (status = hipMalloc(&b_, bytes)) != hipSuccess || + (status = hipMalloc(&c_, bytes)) != hipSuccess) { + error = hip_error("hipMalloc", status); + if (c_) (void) hipFree(c_); + if (b_) (void) hipFree(b_); + if (a_) (void) hipFree(a_); + a_ = nullptr; + b_ = nullptr; + c_ = nullptr; + return false; + } + if ((status = hipMemsetAsync(a_, 0x01, bytes, stream_)) != hipSuccess || + (status = hipMemsetAsync(b_, 0x02, bytes, stream_)) != hipSuccess || + (status = hipMemsetAsync(c_, 0, bytes, stream_)) != hipSuccess) { + error = hip_error("hipMemsetAsync", status); + return false; + } + const float alpha = 1.0F; + const float beta = 0.0F; + const hipblasStatus_t warm_status = hipblasSgemm( + handle_, HIPBLAS_OP_N, HIPBLAS_OP_N, + matrix_size_, matrix_size_, matrix_size_, + &alpha, a_, matrix_size_, b_, matrix_size_, + &beta, c_, matrix_size_); + if (warm_status != HIPBLAS_STATUS_SUCCESS) { + error = hipblas_error("hipblasSgemm(warmup)", warm_status); + return false; + } + if ((status = hipStreamSynchronize(stream_)) != hipSuccess) { + error = hip_error("hipStreamSynchronize(warmup)", status); + return false; + } + } + if (!started_ && + (status = hipEventCreate(&started_)) != hipSuccess) { + error = hip_error("hipEventCreate(started)", status); + return false; + } + if (!finished_ && + (status = hipEventCreate(&finished_)) != hipSuccess) { + error = hip_error("hipEventCreate(finished)", status); + return false; + } + return true; + } + + void release_active() { + std::lock_guard lock(mutex_); + active_ = false; + } + + std::mutex mutex_; + int device_ = 0; + int matrix_size_ = 0; + int total_cus_ = 0; + int current_cu_count_ = 0; + int iterations_ = 0; + bool active_ = false; + hipStream_t stream_ = nullptr; + hipblasHandle_t handle_ = nullptr; + hipEvent_t started_ = nullptr; + hipEvent_t finished_ = nullptr; + float * a_ = nullptr; + float * b_ = nullptr; + float * c_ = nullptr; +}; + +HipSgemmExecution::~HipSgemmExecution() { + if (!finished_) terminate(false); +} + +bool HipSgemmExecution::send_control(const std::string & operation) { + if (operation == "commit") { + committed_ = true; + return true; + } + if (operation == "cancel") { + committed_ = false; + return true; + } + return false; +} + +bool HipSgemmExecution::collect_result( + int timeout_ms, + size_t max_result_bytes, + json & result, + double & wait_ms, + std::string & error) { + if (finished_) { + error = "executor already collected"; + return false; + } + if (!committed_) { + error = "executor result requested before commit"; + terminate(false); + return false; + } + finished_ = true; + return state_->collect( + timeout_ms, max_result_bytes, result, wait_ms, error); +} + +void HipSgemmExecution::terminate(bool allow_control_grace) { + (void) allow_control_grace; + if (finished_) return; + finished_ = true; + state_->synchronize_and_release(); +} + +class HipSgemmExecutor final : public ToolSpeculationExecutor { +public: + explicit HipSgemmExecutor(std::shared_ptr state) + : state_(std::move(state)) {} + + std::unique_ptr start( + const json & request, std::string & error) override { + if (!state_->start(request, error)) return nullptr; + return std::make_unique(state_); + } + + const char * mode_name() const override { + return "in_process_hip_cu_mask"; + } + +private: + std::shared_ptr state_; +}; + +} // namespace + +std::shared_ptr +create_hip_sgemm_tool_speculation_executor( + int device, + int matrix_size, + int & total_compute_units, + std::string & error) { + total_compute_units = 0; + if (device < 0 || matrix_size <= 0 || matrix_size > 8192) { + error = "HIP probe needs DEVICE >= 0 and MATRIX_SIZE in 1..8192"; + return nullptr; + } + hipDeviceProp_t properties{}; + const hipError_t status = hipGetDeviceProperties(&properties, device); + if (status != hipSuccess) { + error = hip_error("hipGetDeviceProperties", status); + return nullptr; + } + if (properties.multiProcessorCount <= 0) { + error = "HIP device reports no compute units"; + return nullptr; + } + total_compute_units = properties.multiProcessorCount; + error.clear(); + auto state = std::make_shared( + device, matrix_size, properties.multiProcessorCount); + return std::make_shared(std::move(state)); +} + +} // namespace dflash::common diff --git a/server/src/server/tool_speculation_hip_probe.h b/server/src/server/tool_speculation_hip_probe.h new file mode 100644 index 000000000..673376521 --- /dev/null +++ b/server/src/server/tool_speculation_hip_probe.h @@ -0,0 +1,21 @@ +#pragma once + +#include "tool_speculation.h" + +#include +#include + +namespace dflash::common { + +// Benchmark-only trusted executor used to qualify same-process HIP sharing. +// It accepts the allowlisted `benchmark_hip_sgemm` tool with +// {"iterations": N}. The matrix size is fixed at startup so allocation and +// warmup stay outside measured requests. +std::shared_ptr +create_hip_sgemm_tool_speculation_executor( + int device, + int matrix_size, + int & total_compute_units, + std::string & error); + +} // namespace dflash::common diff --git a/server/test/test_server_unit.cpp b/server/test/test_server_unit.cpp index 55ae264fe..f10ec0ac7 100644 --- a/server/test/test_server_unit.cpp +++ b/server/test/test_server_unit.cpp @@ -4619,6 +4619,88 @@ TEST_CASE(ServerUnitFixture, test_props_budget_envelope_shape) { TEST_ASSERT(body["server"]["props_schema"].get() == 2); } +TEST_CASE(ServerUnitFixture, test_props_tool_speculation_shape) { + ServerConfig cfg; + Tokenizer tok; + PrefixCache pc(0, tok); + ToolMemory tm; + + json body = build_props_body(cfg, pc, tm); + TEST_ASSERT(body.contains("tool_speculation")); + const json & disabled = body["tool_speculation"]; + TEST_ASSERT(!disabled["enabled"].get()); + TEST_ASSERT(disabled["profile_status"].is_null()); + TEST_ASSERT(disabled["executor_contract"].is_null()); + TEST_ASSERT(disabled["protocol"].get() == + "dflash.tool-speculation.v1"); + TEST_ASSERT(disabled["requires_client_support"].get()); + TEST_ASSERT(disabled["preserves_token_speculation"].get()); + TEST_ASSERT(disabled["unqualified_lane_policy"].get() == + "defer"); + TEST_ASSERT(disabled["allowed_tools"].empty()); + TEST_ASSERT(disabled["model_routing_static"].get()); + TEST_ASSERT(disabled["model_expert_ownership_unique"].get()); + TEST_ASSERT(disabled["compute_isolation"].get() == "none"); + TEST_ASSERT(disabled["hip_tool_device"].is_null()); + TEST_ASSERT(disabled["hip_reserved_tool_compute_units"].get() == 0); + TEST_ASSERT(disabled["profile_lanes"].empty()); + + cfg.tool_speculation.executor_path = "/trusted/tool-adapter"; + cfg.tool_speculation.profile_path = "/measured/frontier.json"; + cfg.tool_speculation.allowed_tools = {"lookup"}; + std::string profile_error; + TEST_ASSERT(cfg.tool_speculation.policy.load_json(json{ + {"path_summary", { + {"25", { + {"decode_interference_qualified", true}, + {"hit", { + {"control_task_mean_ms", 100.0}, + {"speculative_task_mean_ms", 80.0}, + {"model_slowdown_percent", 2.0}, + }}, + {"miss", { + {"control_task_mean_ms", 100.0}, + {"speculative_task_mean_ms", 101.0}, + {"model_slowdown_percent", 2.0}, + }}, + }}, + }}, + }, profile_error)); + + body = build_props_body(cfg, pc, tm); + const json & enabled = body["tool_speculation"]; + TEST_ASSERT(enabled["enabled"].get()); + TEST_ASSERT(enabled["profile_status"].get() == "qualified"); + TEST_ASSERT(enabled["allowed_tools"] == json::array({"lookup"})); + TEST_ASSERT(enabled["preserves_token_speculation"].get()); + TEST_ASSERT(enabled["unqualified_lane_policy"].get() == + "defer"); + TEST_ASSERT(enabled["profile_lanes"].size() == 1); + TEST_ASSERT(enabled["profile_lanes"][0] + ["resource_percentage"].get() == 25); + TEST_ASSERT(std::fabs(enabled["profile_lanes"][0] + ["model_slowdown_ratio"].get() - 1.02) < + 1e-9); + TEST_ASSERT(enabled["profile_lanes"][0] + ["decode_interference_qualified"].get()); + TEST_ASSERT(enabled["profile_lanes"][0] + ["accelerator_relation"].get() == + "unspecified"); + TEST_ASSERT(!enabled["profile_lanes"][0] + ["requires_static_model_routing"].get()); + TEST_ASSERT(!enabled["profile_lanes"][0] + ["requires_unique_expert_ownership"].get()); + + cfg.tool_speculation.hip_tool_device = 1; + cfg.tool_speculation.hip_reserved_tool_compute_units = 1; + body = build_props_body(cfg, pc, tm); + const json & isolated = body["tool_speculation"]; + TEST_ASSERT(isolated["compute_isolation"].get() == + "disjoint_hip_cu_masks"); + TEST_ASSERT(isolated["hip_tool_device"].get() == 1); + TEST_ASSERT(isolated["hip_reserved_tool_compute_units"].get() == 1); +} + // ─── /props.runtime captures full config (§4.16) ────────────────────── // Snapshot/bench tooling reads /props.runtime wholesale into // result.json.server_info; this test pins the field set so additions @@ -4851,6 +4933,40 @@ TEST_CASE(ServerUnitFixture, test_model_backend_retries_empty_spec_restore_once_ TEST_ASSERT(backend.restore_saw_force_ar); } +TEST_CASE(ServerUnitFixture, test_model_backend_can_forbid_ar_retry) { + EmptySpecRetryBackend backend; + GenerateRequest req; + req.prompt = {1, 2, 3}; + req.n_gen = 4; + req.allow_decode_mode_retry = false; + DaemonIO io; + + GenerateResult result = backend.generate(req, io); + + TEST_ASSERT(result.ok()); + TEST_ASSERT(result.tokens.empty()); + TEST_ASSERT(result.spec_decode_ran); + TEST_ASSERT(backend.generate_calls == 1); + TEST_ASSERT(!backend.generate_saw_force_ar); +} + +TEST_CASE(ServerUnitFixture, test_model_backend_restore_can_forbid_ar_retry) { + EmptySpecRetryBackend backend; + GenerateRequest req; + req.prompt = {1, 2, 3}; + req.n_gen = 4; + req.allow_decode_mode_retry = false; + DaemonIO io; + + GenerateResult result = backend.restore_and_generate(7, req, io); + + TEST_ASSERT(result.ok()); + TEST_ASSERT(result.tokens.empty()); + TEST_ASSERT(result.spec_decode_ran); + TEST_ASSERT(backend.restore_calls == 1); + TEST_ASSERT(!backend.restore_saw_force_ar); +} + TEST_CASE(ServerUnitFixture, test_model_backend_retries_empty_visible_spec_generate_once_with_ar) { EmptySpecRetryBackend backend; backend.generate_first_empty_visible = true; diff --git a/server/test/test_tool_speculation.cpp b/server/test/test_tool_speculation.cpp new file mode 100644 index 000000000..baa1f1518 --- /dev/null +++ b/server/test/test_tool_speculation.cpp @@ -0,0 +1,530 @@ +#include "CppUnitTestFramework.hpp" +#include "server/tool_speculation.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +#if !defined(_WIN32) +# include +# include +#endif + +using dflash::common::ApiFormat; +using dflash::common::CanonicalToolInvocation; +using dflash::common::ToolCall; +using dflash::common::ToolSpeculationAttempt; +using dflash::common::ToolSpeculationConfig; +using dflash::common::ToolSpeculationExecution; +using dflash::common::ToolSpeculationExecutor; +using dflash::common::ToolSpeculationPolicy; +using dflash::common::ToolSpeculationPrediction; +using dflash::common::json; +using dflash::common::parse_tool_speculation_prediction; +using dflash::common::render_tool_speculation_sse; + +namespace { +struct ToolSpeculationFixture {}; + +struct FakeExecutionState { + json request; + std::vector controls; + bool terminated = false; + bool collected = false; +}; + +class FakeExecution final : public ToolSpeculationExecution { +public: + explicit FakeExecution(std::shared_ptr state) + : state_(std::move(state)) {} + + bool send_control(const std::string & operation) override { + state_->controls.push_back(operation); + return operation == "commit" || operation == "cancel"; + } + + bool collect_result(int timeout_ms, + size_t max_result_bytes, + json & result, + double & wait_ms, + std::string & error) override { + (void) timeout_ms; + state_->collected = true; + result = {{"value", 42}}; + wait_ms = 0.0; + if (result.dump().size() > max_result_bytes) { + error = "executor_result_too_large"; + return false; + } + error.clear(); + return true; + } + + void terminate(bool allow_control_grace) override { + (void) allow_control_grace; + state_->terminated = true; + } + +private: + std::shared_ptr state_; +}; + +class FakeExecutor final : public ToolSpeculationExecutor { +public: + explicit FakeExecutor(std::shared_ptr state) + : state_(std::move(state)) {} + + std::unique_ptr start( + const json & request, std::string & error) override { + state_->request = request; + error.clear(); + return std::make_unique(state_); + } + + const char * mode_name() const override { + return "fake_in_process"; + } + +private: + std::shared_ptr state_; +}; + +json policy_fixture(bool decode_interference_qualified = true) { + auto path = [decode_interference_qualified]( + double hit_task, double miss_task, + double slowdown_percent) { + return json{ + {"decode_interference_qualified", + decode_interference_qualified}, + {"hit", { + {"control_task_mean_ms", 100.0}, + {"speculative_task_mean_ms", hit_task}, + {"model_slowdown_percent", slowdown_percent}, + }}, + {"miss", { + {"control_task_mean_ms", 100.0}, + {"speculative_task_mean_ms", miss_task}, + {"model_slowdown_percent", slowdown_percent}, + }}, + }; + }; + json fixture = { + {"path_summary", { + {"25", path(80.0, 101.0, 2.0)}, + {"50", path(60.0, 110.0, 7.0)}, + {"100", path(50.0, 130.0, 16.0)}, + }}, + }; + return fixture; +} + +ToolSpeculationConfig test_config(const std::string & executor = {}) { + ToolSpeculationConfig config; + config.executor_path = executor.empty() ? "/unused/executor" : executor; + config.profile_path = "fixture.json"; + config.allowed_tools = {"lookup"}; + config.timeout_ms = 1000; + config.cancel_grace_ms = 20; + config.max_model_slowdown_ratio = 1.20; + std::string error; + if (!config.policy.load_json(policy_fixture(), error)) { + throw std::runtime_error(error); + } + return config; +} + +ToolSpeculationPrediction prediction(double confidence = 0.9) { + ToolSpeculationPrediction value; + std::string error; + if (!CanonicalToolInvocation::from_parts( + "lookup", json{{"a", 1}, {"b", 2}}, value.call, error)) { + throw std::runtime_error(error); + } + value.confidence = confidence; + return value; +} + +#if !defined(_WIN32) +std::string make_executor_script(const std::string & body) { + char path[] = "/tmp/dflash-tool-spec-test-XXXXXX"; + const int fd = ::mkstemp(path); + if (fd < 0) throw std::runtime_error("mkstemp failed"); + const std::string script = "#!/bin/sh\nIFS= read -r request\n" + body; + size_t offset = 0; + while (offset < script.size()) { + const ssize_t count = ::write( + fd, script.data() + offset, script.size() - offset); + if (count <= 0) { + ::close(fd); + ::unlink(path); + throw std::runtime_error("script write failed"); + } + offset += static_cast(count); + } + if (::fchmod(fd, 0700) != 0) { + ::close(fd); + ::unlink(path); + throw std::runtime_error("chmod failed"); + } + ::close(fd); + return path; +} + +std::string make_temp_path() { + char path[] = "/tmp/dflash-tool-spec-observed-XXXXXX"; + const int fd = ::mkstemp(path); + if (fd < 0) throw std::runtime_error("mkstemp failed"); + ::close(fd); + return path; +} + +std::string read_text_file(const std::string & path) { + FILE * file = std::fopen(path.c_str(), "rb"); + if (!file) return {}; + std::string value; + char buffer[256]; + while (const size_t count = std::fread(buffer, 1, sizeof(buffer), file)) { + value.append(buffer, count); + } + std::fclose(file); + return value; +} +#endif +} // namespace + +TEST_CASE(ToolSpeculationFixture, canonical_identity_ignores_argument_order) { + CanonicalToolInvocation first; + CanonicalToolInvocation second; + std::string error; + CHECK(CanonicalToolInvocation::from_parts( + "lookup", json{{"b", 2}, {"a", 1}}, first, error)); + CHECK(CanonicalToolInvocation::from_parts( + "lookup", json{{"a", 1}, {"b", 2}}, second, error)); + CHECK(first == second); + CHECK(first.arguments_json == R"({"a":1,"b":2})"); +} + +TEST_CASE(ToolSpeculationFixture, prediction_requires_declared_tool) { + const json tools = json::array({{ + {"type", "function"}, + {"function", {{"name", "lookup"}}}, + }}); + const json request = { + {"call", { + {"name", "lookup"}, + {"arguments", {{"key", "x"}}}, + }}, + {"confidence", 0.75}, + }; + ToolSpeculationPrediction parsed; + std::string error; + CHECK(parse_tool_speculation_prediction(request, tools, parsed, error)); + CHECK(parsed.call.name == "lookup"); + + json undeclared = request; + undeclared["call"]["name"] = "write_file"; + CHECK(!parse_tool_speculation_prediction( + undeclared, tools, parsed, error)); + CHECK(error.find("not declared") != std::string::npos); +} + +TEST_CASE(ToolSpeculationFixture, empirical_policy_selects_resource_by_confidence) { + ToolSpeculationPolicy policy; + std::string error; + CHECK(policy.load_json(policy_fixture(), error)); + + const auto deferred = policy.choose(0.0, 1.20); + CHECK(!deferred.admitted); + CHECK(deferred.reason == "below_profile_break_even"); + + const auto low = policy.choose(0.10, 1.20); + CHECK(low.admitted); + CHECK(low.resource_percentage == 25); + + const auto medium = policy.choose(0.50, 1.20); + CHECK(medium.admitted); + CHECK(medium.resource_percentage == 50); + + const auto high = policy.choose(0.90, 1.20); + CHECK(high.admitted); + CHECK(high.resource_percentage == 100); + + const auto guarded = policy.choose(0.90, 1.10); + CHECK(guarded.admitted); + CHECK(guarded.resource_percentage == 50); +} + +TEST_CASE(ToolSpeculationFixture, unqualified_resource_lanes_are_deferred) { + ToolSpeculationPolicy policy; + std::string error; + CHECK(policy.load_json(policy_fixture(false), error)); + + const auto decision = policy.choose(1.0, 1.20); + CHECK(!decision.admitted); + CHECK(decision.reason == "decode_interference_unqualified"); +} + +TEST_CASE(ToolSpeculationFixture, profile_metadata_is_fail_closed) { + ToolSpeculationPolicy policy; + std::string error; + json fixture = policy_fixture(); + fixture["profile_status"] = "provisional_benchmark_only"; + fixture["executor"] = "in_process_hip_cu_mask"; + CHECK(policy.load_json(fixture, error)); + CHECK(policy.benchmark_only()); + CHECK(policy.executor_contract() == "in_process_hip_cu_mask"); + + fixture["profile_status"] = "unknown"; + CHECK(!policy.load_json(fixture, error)); + CHECK(policy.empty()); +} + +TEST_CASE(ToolSpeculationFixture, same_gpu_profile_declares_routing_requirements) { + ToolSpeculationPolicy policy; + std::string error; + json fixture = policy_fixture(); + for (auto & lane : fixture["path_summary"]) { + lane["accelerator_relation"] = "same_physical_gpu"; + } + CHECK(policy.load_json(fixture, error)); + CHECK(!policy.requires_static_model_routing()); + CHECK(!policy.requires_unique_expert_ownership()); + + for (auto & lane : fixture["path_summary"]) { + lane["requires_static_model_routing"] = true; + } + CHECK(policy.load_json(fixture, error)); + CHECK(policy.requires_static_model_routing()); + CHECK(!policy.requires_unique_expert_ownership()); + + for (auto & lane : fixture["path_summary"]) { + lane["requires_unique_expert_ownership"] = true; + } + CHECK(policy.load_json(fixture, error)); + CHECK(policy.requires_static_model_routing()); + CHECK(policy.requires_unique_expert_ownership()); + const auto decision = policy.choose(1.0, 2.0); + CHECK(decision.admitted); + CHECK(decision.accelerator_relation == "same_physical_gpu"); +} + +TEST_CASE(ToolSpeculationFixture, same_gpu_profile_obeys_measured_break_even) { + ToolSpeculationPolicy policy; + std::string error; + CHECK(policy.load_json(json{ + {"path_summary", { + {"100", { + {"accelerator_relation", "same_physical_gpu"}, + {"requires_static_model_routing", true}, + {"requires_unique_expert_ownership", true}, + {"decode_interference_qualified", true}, + {"hit", { + {"control_task_mean_ms", 2670.178}, + {"speculative_task_mean_ms", 2389.530}, + {"model_slowdown_percent", 169.109}, + }}, + {"miss", { + {"control_task_mean_ms", 2670.178}, + {"speculative_task_mean_ms", 4171.884}, + {"model_slowdown_percent", 169.109}, + }}, + }}, + }}, + }, error)); + + const auto below = policy.choose(0.84, 3.0); + CHECK(!below.admitted); + CHECK(below.reason == "below_profile_break_even"); + + const auto above = policy.choose(0.85, 3.0); + CHECK(above.admitted); + CHECK(above.resource_percentage == 100); + + const auto guarded = policy.choose(1.0, 1.20); + CHECK(!guarded.admitted); + CHECK(guarded.reason == "model_slowdown_guardrail"); +} + +TEST_CASE(ToolSpeculationFixture, non_allowlisted_tool_is_deferred) { + ToolSpeculationConfig config = test_config(); + config.allowed_tools = {"other"}; + auto attempt = ToolSpeculationAttempt::create( + config, prediction(), "request_allowlist"); + attempt->start(); + const json metadata = attempt->resolve({ + ToolCall{"call_1", "lookup", R"({"a":1,"b":2})"}, + }); + CHECK(metadata["status"] == "deferred"); + CHECK(metadata["reason"] == "tool_not_allowlisted"); + CHECK(!metadata.contains("result")); +} + +TEST_CASE(ToolSpeculationFixture, in_process_exact_match_commits_private_result) { + auto state = std::make_shared(); + ToolSpeculationConfig config = test_config(); + config.executor_path.clear(); + config.in_process_executor = std::make_shared(state); + CHECK(config.enabled()); + CHECK(std::string(config.execution_mode()) == "fake_in_process"); + + auto attempt = ToolSpeculationAttempt::create( + config, prediction(), "request_in_process_hit"); + attempt->start(); + CHECK(attempt->running()); + CHECK(state->request["call"]["name"] == "lookup"); + CHECK(state->request["resource_percentage"] == 100); + + const json metadata = attempt->resolve({ + ToolCall{"call_1", "lookup", R"({"b":2,"a":1})"}, + }); + CHECK(metadata["status"] == "hit"); + CHECK(metadata["result"]["value"] == 42); + CHECK(state->collected); + CHECK(state->controls.size() == 1); + CHECK(state->controls[0] == "commit"); + CHECK(!state->terminated); +} + +TEST_CASE(ToolSpeculationFixture, in_process_mismatch_cancels_private_result) { + auto state = std::make_shared(); + ToolSpeculationConfig config = test_config(); + config.executor_path.clear(); + config.in_process_executor = std::make_shared(state); + + auto attempt = ToolSpeculationAttempt::create( + config, prediction(), "request_in_process_miss"); + attempt->start(); + const json metadata = attempt->resolve({ + ToolCall{"call_1", "lookup", R"({"a":999})"}, + }); + CHECK(metadata["status"] == "miss"); + CHECK(metadata["reason"] == "invocation_mismatch"); + CHECK(!metadata.contains("result")); + CHECK(!state->collected); + CHECK(state->terminated); + CHECK(state->controls.size() == 1); + CHECK(state->controls[0] == "cancel"); +} + +#if !defined(_WIN32) +TEST_CASE(ToolSpeculationFixture, exact_match_exposes_result_and_resource_share) { + const std::string control_path = make_temp_path(); + const std::string path = make_executor_script( + "IFS= read -r control\nprintf '%s' \"$control\" > " + control_path + "\n" + "printf '{\"ok\":true,\"result\":{\"resource\":\"%s\"," + "\"relation\":\"%s\",\"value\":42}}\\n' " + "\"$DFLASH_TOOL_SPECULATION_RESOURCE_PERCENTAGE\" " + "\"$DFLASH_TOOL_SPECULATION_ACCELERATOR_RELATION\"\n"); + ToolSpeculationConfig config = test_config(path); + auto attempt = ToolSpeculationAttempt::create( + config, prediction(), "request_hit"); + attempt->start(); + const json metadata = attempt->resolve({ + ToolCall{"call_1", "lookup", R"({"b":2,"a":1})"}, + }); + ::unlink(path.c_str()); + const std::string control = read_text_file(control_path); + ::unlink(control_path.c_str()); + + CHECK(metadata["status"] == "hit"); + CHECK(metadata["resource_percentage"] == 100); + CHECK(metadata["result"]["resource"] == "100"); + CHECK(metadata["result"]["relation"] == "unspecified"); + CHECK(metadata["result"]["value"] == 42); + CHECK(control.find("\"op\":\"commit\"") != std::string::npos); + CHECK(control.find("\"authoritative_resource_percentage\":100") != + std::string::npos); +} + +TEST_CASE(ToolSpeculationFixture, unqualified_tool_never_launches_or_changes_decode) { + const std::string path = make_executor_script( + "IFS= read -r control\n" + "exit 0\n"); + ToolSpeculationConfig config = test_config(path); + std::string error; + CHECK(config.policy.load_json(policy_fixture(false), error)); + auto deferred = ToolSpeculationAttempt::create( + config, prediction(), "request_guarded_decode"); + deferred->start(); + CHECK(!deferred->running()); + const json metadata = deferred->resolve({ + ToolCall{"call_1", "lookup", R"({"a":1,"b":2})"}, + }); + CHECK(metadata["status"] == "deferred"); + CHECK(metadata["reason"] == "decode_interference_unqualified"); + ::unlink(path.c_str()); +} + +TEST_CASE(ToolSpeculationFixture, executor_failure_is_private) { + ToolSpeculationConfig config = test_config( + "/definitely/missing/dflash-tool-spec-executor"); + auto attempt = ToolSpeculationAttempt::create( + config, prediction(), "request_executor_failure"); + attempt->start(); + const json metadata = attempt->resolve({ + ToolCall{"call_1", "lookup", R"({"a":1,"b":2})"}, + }); + CHECK(metadata["status"] == "failed"); + CHECK(metadata["reason"] == "executor_launch_failed"); + CHECK(!metadata.contains("result")); +} + +TEST_CASE(ToolSpeculationFixture, qualified_lane_keeps_speculative_decode) { + const std::string path = make_executor_script( + "IFS= read -r control\n" + "exit 0\n"); + ToolSpeculationConfig config = test_config(path); + std::string error; + CHECK(config.policy.load_json(policy_fixture(), error)); + auto attempt = ToolSpeculationAttempt::create( + config, prediction(), "request_qualified_lane"); + attempt->start(); + CHECK(attempt->running()); + const json metadata = attempt->cancel("test_complete"); + CHECK(metadata["decode_interference_qualified"].get()); + ::unlink(path.c_str()); +} + +TEST_CASE(ToolSpeculationFixture, mismatch_cancels_and_never_exposes_result) { + const std::string control_path = make_temp_path(); + const std::string path = make_executor_script( + "IFS= read -r control\nprintf '%s' \"$control\" > " + control_path + "\n" + "exit 3\n"); + ToolSpeculationConfig config = test_config(path); + auto attempt = ToolSpeculationAttempt::create( + config, prediction(), "request_miss"); + attempt->start(); + const auto started = std::chrono::steady_clock::now(); + const json metadata = attempt->resolve({ + ToolCall{"call_1", "lookup", R"({"a":999})"}, + }); + const double elapsed_ms = std::chrono::duration( + std::chrono::steady_clock::now() - started).count(); + ::unlink(path.c_str()); + const std::string control = read_text_file(control_path); + ::unlink(control_path.c_str()); + + CHECK(metadata["status"] == "miss"); + CHECK(metadata["reason"] == "invocation_mismatch"); + CHECK(!metadata.contains("result")); + CHECK(elapsed_ms < 1000.0); + CHECK(control.find("\"op\":\"cancel\"") != std::string::npos); +} +#endif + +TEST_CASE(ToolSpeculationFixture, streaming_extension_keeps_result_explicit) { + const json metadata = { + {"status", "hit"}, + {"result", {{"value", 42}}}, + }; + const std::string event = render_tool_speculation_sse( + ApiFormat::OPENAI_CHAT, "req_1", "model", metadata); + CHECK(event.find("dflash_tool_speculation") != std::string::npos); + CHECK(event.find("\"value\":42") != std::string::npos); +} From 48b3cb2be6a1e7b6646614c690ec82dbea599050 Mon Sep 17 00:00:00 2001 From: mrciffa <49000955+davide221@users.noreply.github.com> Date: Thu, 13 Aug 2026 19:10:12 +0200 Subject: [PATCH 02/11] feat(server): isolate speculative tools on CPU lanes --- optimizations/ooo_spec_lucebox5_cpu/README.md | 94 + .../benchmark_cpu_tool_speculation.py | 1135 ++++++++ .../build_cpu_sparse_executor.sh | 18 + .../cpu_sparse_tool_executor.cpp | 259 ++ .../profiles/lucebox5-cpu-lane-qualified.json | 69 + .../lucebox5-cpu-lane-qualification.json | 1839 +++++++++++++ .../results/lucebox5-cpu-native-20pairs.json | 2346 +++++++++++++++++ .../run_native_cpu_server_lucebox5.sh | 43 + .../test_benchmark_cpu_tool_speculation.py | 40 + server/src/server/http_server.cpp | 12 +- server/src/server/server_main.cpp | 29 +- server/src/server/tool_speculation.cpp | 239 +- server/src/server/tool_speculation.h | 24 +- server/test/test_server_unit.cpp | 17 + server/test/test_tool_speculation.cpp | 83 + 15 files changed, 6241 insertions(+), 6 deletions(-) create mode 100644 optimizations/ooo_spec_lucebox5_cpu/README.md create mode 100755 optimizations/ooo_spec_lucebox5_cpu/benchmark_cpu_tool_speculation.py create mode 100755 optimizations/ooo_spec_lucebox5_cpu/build_cpu_sparse_executor.sh create mode 100644 optimizations/ooo_spec_lucebox5_cpu/cpu_sparse_tool_executor.cpp create mode 100644 optimizations/ooo_spec_lucebox5_cpu/profiles/lucebox5-cpu-lane-qualified.json create mode 100644 optimizations/ooo_spec_lucebox5_cpu/results/lucebox5-cpu-lane-qualification.json create mode 100644 optimizations/ooo_spec_lucebox5_cpu/results/lucebox5-cpu-native-20pairs.json create mode 100755 optimizations/ooo_spec_lucebox5_cpu/run_native_cpu_server_lucebox5.sh create mode 100644 optimizations/ooo_spec_lucebox5_cpu/test_benchmark_cpu_tool_speculation.py diff --git a/optimizations/ooo_spec_lucebox5_cpu/README.md b/optimizations/ooo_spec_lucebox5_cpu/README.md new file mode 100644 index 000000000..76ecd5816 --- /dev/null +++ b/optimizations/ooo_spec_lucebox5_cpu/README.md @@ -0,0 +1,94 @@ +# Lucebox5 disjoint-CPU tool speculation + +This experiment runs a predicted read-only/idempotent tool concurrently with +DeepSeek generation. The model stays on the R9700 + Strix GPU path with DS4 +enabled, while the tool child process is pinned to CPU cores that the model +process cannot use. The result is released only when the generated canonical +tool call exactly matches the prediction. + +## Measured result + +The native engine benchmark on Lucebox5 passed its production gate: + +| Metric | Result | +| --- | ---: | +| Sequential model + tool, p50 | 5511.64 ms | +| Speculative full task, p50 | 2910.40 ms | +| Exact-hit speedup | **1.8938x** | +| Bootstrap 95% CI | 1.8859x - 1.8964x | +| Task-latency reduction | 47.20% | +| Model-compute slowdown | 0.46% | +| DS4 median acceptance rate | 0.4167 | +| Correct predictions | 20 / 20 | + +This workload's zero-interference ceiling is 1.9011x because model and tool +latencies are not perfectly equal. The implementation reaches 99.6% of that +ceiling; claiming 2x for this measured workload would be inaccurate. + +The control arm runs the model and then the identical CPU-pinned sparse tool. +The speculative arm starts that tool before generation. Arm order is randomized +inside each of 20 warm pairs, with two warmups. A 20,000-resample paired +bootstrap supplies the confidence interval. Model outputs, canonical calls, +and tool checksums are identical across arms. A deliberately wrong prediction +produces an `invocation_mismatch` and exposes no private tool result. + +Raw artifacts: + +- `results/lucebox5-cpu-native-20pairs.json` (`sha256:4a7f224f44cc7f2385e476f8227c6d51d4f4b90052e6a7b090bafcfc1f3b68a7`) +- `results/lucebox5-cpu-lane-qualification.json` (`sha256:6fc3f6d95c12db817b687b8c9509517d24230a508f520989483c1c6ed96c67df`) +- `profiles/lucebox5-cpu-lane-qualified.json` (`sha256:5cdf2550bb5a95c835daddac9f8d0126f470a5ac359e14008207e82c5dafb718`) + +## Isolation and compatibility + +Lucebox5 reserves logical CPUs `14-15,30-31` for the two-thread sparse tool and +launches the model with `0-13,16-29`. Startup fails closed if either mask +overlaps, if a listed CPU does not exist, or if an in-process executor is used. +Each child is pinned and its mask is read back before the request payload is +sent, so tool work cannot begin on model CPUs. + +The engine feature is Linux- and backend-neutral: a single-GPU system can use +the same child-process path when it has CPU cores to reserve. It does not +replace or disable autoregressive decoding or DS4 token speculation. The +benchmark requires a positive DS4 acceptance rate on every measured request; +the median was 0.4167. + +This improves the full latency of a correctly predicted tool-using request; it +does not double token generation throughput. On a miss, generation remains +authoritative, the speculative result stays private, and the caller executes +the generated tool call normally. + +## Reproduce + +Build the deterministic sparse-compute executor: + +```bash +JSON_INCLUDE=/path/to/server/deps/json/include \ + ./build_cpu_sparse_executor.sh ./cpu_sparse_tool_executor +``` + +Launch the qualified native server on an otherwise idle Lucebox5: + +```bash +./run_native_cpu_server_lucebox5.sh +``` + +Then run the measured gate: + +```bash +python3 benchmark_cpu_tool_speculation.py native \ + --url http://127.0.0.1:18145/v1/chat/completions \ + --binary ./cpu_sparse_tool_executor \ + --tool-cpus 14-15,30-31 \ + --iterations 172452 \ + --max-tokens 32 \ + --pairs 20 \ + --warmups 2 \ + --bootstrap-resamples 20000 \ + --min-speedup 1.8 \ + --min-speedup-ci-low 1.7 \ + --max-model-slowdown-percent 5 \ + --output results/lucebox5-cpu-native-20pairs.json +``` + +The harness exits nonzero if correctness, isolation, DS4 activity, slowdown, +or either speed threshold fails. diff --git a/optimizations/ooo_spec_lucebox5_cpu/benchmark_cpu_tool_speculation.py b/optimizations/ooo_spec_lucebox5_cpu/benchmark_cpu_tool_speculation.py new file mode 100755 index 000000000..7aae61cc2 --- /dev/null +++ b/optimizations/ooo_spec_lucebox5_cpu/benchmark_cpu_tool_speculation.py @@ -0,0 +1,1135 @@ +#!/usr/bin/env python3 +"""Qualify and benchmark a disjoint Strix CPU speculative-tool lane. + +The qualification phase runs the official model server without tool +speculation, pins the real sparse-compute tool to reserved physical cores, and +measures sequential versus overlapped execution. It emits a qualified engine +profile only if model output, DS4 activity, tool results, CPU isolation, and +the slowdown gate all pass. + +The native phase then measures the engine's exact-call commit path against the +same strong sequential CPU baseline. Wrong predictions are cancelled and +their private results must never be exposed. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import random +import re +import statistics +import subprocess +import time +import urllib.request +from pathlib import Path +from typing import Any, Iterable +from urllib.parse import urlsplit, urlunsplit + + +TOOL_NAME = "benchmark_cpu_sparse" +PROTOCOL = "dflash.tool-speculation.v1" +SPARSE_ROWS = 4096 +SPARSE_NONZEROS_PER_ROW = 16 +SPARSE_THREADS = 2 +SPARSE_SEED = 731 + + +def parse_cpu_list(value: str) -> list[int]: + cpus: set[int] = set() + for item in value.split(","): + if not item: + raise argparse.ArgumentTypeError("CPU list contains an empty item") + if "-" in item: + parts = item.split("-") + if len(parts) != 2 or not all(part.isdigit() for part in parts): + raise argparse.ArgumentTypeError(f"invalid CPU range: {item}") + first, last = map(int, parts) + if first > last: + raise argparse.ArgumentTypeError(f"invalid CPU range: {item}") + cpus.update(range(first, last + 1)) + elif item.isdigit(): + cpus.add(int(item)) + else: + raise argparse.ArgumentTypeError(f"invalid CPU id: {item}") + if not cpus: + raise argparse.ArgumentTypeError("CPU list must not be empty") + return sorted(cpus) + + +def compact_cpu_list(cpus: Iterable[int]) -> str: + return ",".join(str(cpu) for cpu in cpus) + + +def expected_arguments( + rows: int, + nonzeros_per_row: int, + iterations: int, + threads: int, + seed: int, +) -> dict[str, int]: + if ( + rows != SPARSE_ROWS + or nonzeros_per_row != SPARSE_NONZEROS_PER_ROW + or threads != SPARSE_THREADS + or seed != SPARSE_SEED + ): + raise ValueError("this qualification binary has a fixed sparse shape") + # Keep the generated tool call intentionally short. The deterministic + # benchmark binary owns the qualified sparse shape; only work duration is + # request-dependent, matching real tools with a compact identifier. + return {"iterations": iterations} + + +def tool_definition() -> dict[str, Any]: + properties = {"iterations": {"type": "integer"}} + return { + "name": TOOL_NAME, + "parameters": { + "type": "object", + "properties": properties, + "required": list(properties), + "additionalProperties": False, + }, + } + + +def request_body( + arguments: dict[str, int], + max_tokens: int, + *, + prediction: dict[str, int] | None, +) -> dict[str, Any]: + compact = json.dumps(arguments, separators=(",", ":")) + body: dict[str, Any] = { + "model": "dflash", + "stream": False, + "max_tokens": max_tokens, + "temperature": 0, + "messages": [ + { + "role": "user", + "content": f"Return only this JSON object and nothing else: {compact}", + } + ], + "tools": [tool_definition()], + } + if prediction is not None: + body["tool_speculation"] = { + "call": {"name": TOOL_NAME, "arguments": prediction}, + "confidence": 1.0, + } + return body + + +def normalize_tool_call(result: dict[str, Any]) -> dict[str, Any] | None: + message = result.get("choices", [{}])[0].get("message", {}) + tool_calls = message.get("tool_calls") or [] + if len(tool_calls) == 1: + function = tool_calls[0].get("function") or {} + arguments = function.get("arguments", "{}") + if isinstance(arguments, str): + try: + arguments = json.loads(arguments) + except json.JSONDecodeError: + return None + return {"name": function.get("name"), "arguments": arguments} + content = message.get("content") + if not isinstance(content, str) or not content: + return None + bracket_call = re.fullmatch(r'\["([^"]+)"\]\((\{.*\})\)', content) + if bracket_call: + try: + return { + "name": bracket_call.group(1), + "arguments": json.loads(bracket_call.group(2)), + } + except json.JSONDecodeError: + return None + try: + parsed = json.loads(content) + except json.JSONDecodeError: + return None + if not isinstance(parsed, dict): + return None + function = parsed.get("function", parsed.get("name")) + if isinstance(function, dict): + name = function.get("name") + arguments = function.get("arguments", function.get("parameters")) + else: + name = function + arguments = parsed.get( + "params", + parsed.get( + "parameters", + parsed.get("arguments", parsed.get("function_args")), + ), + ) + if arguments is None and isinstance(name, str): + arguments = { + key: value + for key, value in parsed.items() + if key not in {"function", "name", "type"} + } + if isinstance(arguments, str): + try: + arguments = json.loads(arguments) + except json.JSONDecodeError: + return None + if not isinstance(name, str) or not isinstance(arguments, dict): + return None + return {"name": name, "arguments": arguments} + + +def post_json( + url: str, body: dict[str, Any], timeout: float +) -> tuple[dict[str, Any], float]: + request = urllib.request.Request( + url, + data=json.dumps(body, separators=(",", ":")).encode(), + headers={"Content-Type": "application/json"}, + method="POST", + ) + started = time.perf_counter() + with urllib.request.urlopen(request, timeout=timeout) as response: + result = json.load(response) + if not isinstance(result, dict): + raise RuntimeError("model response is not a JSON object") + return result, (time.perf_counter() - started) * 1000.0 + + +def get_json(url: str, timeout: float) -> dict[str, Any]: + with urllib.request.urlopen(url, timeout=timeout) as response: + result = json.load(response) + if not isinstance(result, dict): + raise RuntimeError(f"expected a JSON object from {url}") + return result + + +def props_url(completion_url: str) -> str: + parsed = urlsplit(completion_url) + return urlunsplit((parsed.scheme, parsed.netloc, "/props", "", "")) + + +def observation(result: dict[str, Any], wall_ms: float) -> dict[str, Any]: + call = normalize_tool_call(result) + usage = result.get("usage") or {} + timings = usage.get("timings") or {} + message = result.get("choices", [{}])[0].get("message", {}) + content = message.get("content") or "" + canonical_call = json.dumps(call, sort_keys=True, separators=(",", ":")) + return { + "request_wall_ms": wall_ms, + "model_compute_ms": float(timings.get("prefill_ms", 0.0)) + + float(timings.get("decode_ms", 0.0)), + "prefill_ms": float(timings.get("prefill_ms", 0.0)), + "decode_ms": float(timings.get("decode_ms", 0.0)), + "decode_tokens_per_sec": float( + timings.get("decode_tokens_per_sec", 0.0) + ), + "completion_tokens": int(usage.get("completion_tokens", 0)), + "accept_rate": float(usage.get("accept_rate", 0.0)), + "tool_call": call, + "tool_call_sha256": hashlib.sha256(canonical_call.encode()).hexdigest(), + "assistant_content_sha256": hashlib.sha256(content.encode()).hexdigest(), + "speculation": result.get("dflash_tool_speculation"), + } + + +def post_model( + url: str, + arguments: dict[str, int], + max_tokens: int, + timeout: float, + *, + prediction: dict[str, int] | None = None, +) -> dict[str, Any]: + result, wall_ms = post_json( + url, + request_body(arguments, max_tokens, prediction=prediction), + timeout, + ) + return observation(result, wall_ms) + + +def executor_request( + arguments: dict[str, int], cpus: list[int], request_id: str +) -> dict[str, Any]: + return { + "protocol": PROTOCOL, + "request_id": request_id, + "mode": "authoritative-benchmark", + "resource_percentage": 100, + "accelerator_relation": "non_accelerator", + "cpu_affinity": cpus, + "cpu_affinity_isolated": True, + "call": {"name": TOOL_NAME, "arguments": arguments}, + } + + +def start_executor( + binary: Path, + arguments: dict[str, int], + cpus: list[int], + request_id: str, +) -> dict[str, Any]: + environment = os.environ.copy() + environment["DFLASH_TOOL_SPECULATION_CPU_AFFINITY"] = compact_cpu_list(cpus) + + def pin_child() -> None: + os.sched_setaffinity(0, set(cpus)) + + started = time.perf_counter() + process = subprocess.Popen( + [str(binary), "--dflash-tool-spec-v1"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=environment, + preexec_fn=pin_child, + ) + assert process.stdin is not None + process.stdin.write( + json.dumps( + executor_request(arguments, cpus, request_id), + separators=(",", ":"), + ) + + "\n" + ) + process.stdin.close() + process.stdin = None + return {"process": process, "started": started} + + +def finish_executor(handle: dict[str, Any], timeout: float) -> dict[str, Any]: + process: subprocess.Popen[str] = handle["process"] + try: + stdout, stderr = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + process.kill() + stdout, stderr = process.communicate(timeout=5) + raise RuntimeError("CPU executor timed out") + wall_ms = (time.perf_counter() - float(handle["started"])) * 1000.0 + if process.returncode != 0: + raise RuntimeError( + f"CPU executor exited {process.returncode}: {stderr.strip()}" + ) + try: + envelope = json.loads(stdout) + except json.JSONDecodeError as error: + raise RuntimeError(f"CPU executor returned invalid JSON: {stdout!r}") from error + if not isinstance(envelope, dict) or not envelope.get("ok"): + raise RuntimeError(f"CPU executor rejected request: {envelope!r}") + result = envelope.get("result") + if not isinstance(result, dict): + raise RuntimeError("CPU executor result is not an object") + return {"wall_ms": wall_ms, "result": result} + + +def stop_executor(handle: dict[str, Any]) -> None: + process: subprocess.Popen[str] = handle["process"] + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=1.0) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5.0) + + +def run_executor( + binary: Path, + arguments: dict[str, int], + cpus: list[int], + timeout: float, + request_id: str, +) -> dict[str, Any]: + return finish_executor( + start_executor(binary, arguments, cpus, request_id), timeout + ) + + +def expected_call(arguments: dict[str, int]) -> dict[str, Any]: + return {"name": TOOL_NAME, "arguments": arguments} + + +def validate_model_call(row: dict[str, Any], arguments: dict[str, int]) -> None: + expected = expected_call(arguments) + if row["tool_call"] != expected: + raise RuntimeError( + f"model emitted {row['tool_call']!r}, expected {expected!r}" + ) + + +def percentile(values: Iterable[float], quantile: float) -> float: + ordered = sorted(float(value) for value in values) + if not ordered: + raise ValueError("cannot take percentile of an empty sequence") + position = (len(ordered) - 1) * quantile + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return ordered[lower] + fraction = position - lower + return ordered[lower] * (1.0 - fraction) + ordered[upper] * fraction + + +def bootstrap_speedup_ci( + pairs: list[dict[str, Any]], resamples: int, seed: int +) -> list[float]: + generator = random.Random(seed) + ratios = [] + for _ in range(resamples): + sample = [pairs[generator.randrange(len(pairs))] for _ in pairs] + control = statistics.median( + float(pair["control"]["task_ms"]) for pair in sample + ) + speculative = statistics.median( + float(pair["speculative"]["task_ms"]) for pair in sample + ) + ratios.append(control / speculative) + return [percentile(ratios, 0.025), percentile(ratios, 0.975)] + + +def read_process_affinity(pid: int) -> list[int]: + return sorted(os.sched_getaffinity(pid)) + + +def calibrate( + args: argparse.Namespace, +) -> tuple[dict[str, int], dict[str, Any]]: + probe_arguments = expected_arguments( + args.rows, + args.nonzeros_per_row, + max(1, args.initial_iterations), + args.threads, + args.tool_seed, + ) + model_samples = [] + for _ in range(args.calibration_model_samples): + row = post_model( + args.url, probe_arguments, args.max_tokens, args.timeout + ) + validate_model_call(row, probe_arguments) + model_samples.append(row) + target_ms = statistics.median( + float(row["request_wall_ms"]) for row in model_samples + ) + + iterations = args.initial_iterations + calibration_steps = [] + for step in range(args.calibration_steps): + arguments = expected_arguments( + args.rows, + args.nonzeros_per_row, + iterations, + args.threads, + args.tool_seed, + ) + samples = [ + run_executor( + args.binary, + arguments, + args.tool_cpus, + args.timeout, + f"calibrate-{step}-{sample}", + ) + for sample in range(args.calibration_tool_samples) + ] + observed_ms = statistics.median( + float(sample["wall_ms"]) for sample in samples + ) + calibration_steps.append( + { + "iterations": iterations, + "tool_wall_p50_ms": observed_ms, + "samples": samples, + } + ) + if observed_ms <= 0: + raise RuntimeError("CPU executor calibration returned zero time") + ratio = target_ms / observed_ms + if 0.97 <= ratio <= 1.03: + break + iterations = max(1, round(iterations * ratio)) + + final_arguments = expected_arguments( + args.rows, + args.nonzeros_per_row, + iterations, + args.threads, + args.tool_seed, + ) + return final_arguments, { + "target_model_request_p50_ms": target_ms, + "model_samples": model_samples, + "steps": calibration_steps, + "selected_iterations": iterations, + } + + +def run_direct_control( + args: argparse.Namespace, + arguments: dict[str, int], + label: str, +) -> dict[str, Any]: + started = time.perf_counter() + model = post_model(args.url, arguments, args.max_tokens, args.timeout) + tool = run_executor( + args.binary, arguments, args.tool_cpus, args.timeout, f"{label}-tool" + ) + validate_model_call(model, arguments) + return { + "mode": "control", + "task_ms": (time.perf_counter() - started) * 1000.0, + "model": model, + "tool": tool, + } + + +def run_direct_overlap( + args: argparse.Namespace, + arguments: dict[str, int], + label: str, +) -> dict[str, Any]: + started = time.perf_counter() + handle = start_executor(args.binary, arguments, args.tool_cpus, label) + try: + model = post_model(args.url, arguments, args.max_tokens, args.timeout) + tool = finish_executor(handle, args.timeout) + except BaseException: + stop_executor(handle) + raise + validate_model_call(model, arguments) + return { + "mode": "speculative", + "task_ms": (time.perf_counter() - started) * 1000.0, + "model": model, + "tool": tool, + } + + +def run_direct_miss( + args: argparse.Namespace, + arguments: dict[str, int], + label: str, +) -> dict[str, Any]: + wrong = dict(arguments) + wrong["iterations"] = max(1, arguments["iterations"] - 1) + if wrong["iterations"] == arguments["iterations"]: + wrong["iterations"] += 1 + started = time.perf_counter() + private = start_executor(args.binary, wrong, args.tool_cpus, f"{label}-wrong") + model = post_model(args.url, arguments, args.max_tokens, args.timeout) + stop_executor(private) + authoritative = run_executor( + args.binary, + arguments, + args.tool_cpus, + args.timeout, + f"{label}-authoritative", + ) + validate_model_call(model, arguments) + return { + "mode": "miss", + "task_ms": (time.perf_counter() - started) * 1000.0, + "model": model, + "authoritative_tool": authoritative, + "private_result_exposed": False, + } + + +def qualify(args: argparse.Namespace) -> None: + model_affinity = read_process_affinity(args.model_pid) + overlap = sorted(set(model_affinity).intersection(args.tool_cpus)) + if overlap: + raise SystemExit(f"model/tool CPU affinity overlaps: {overlap}") + if args.threads > len(args.tool_cpus): + raise SystemExit("tool threads exceed reserved logical CPUs") + + arguments, calibration = calibrate(args) + for warmup in range(args.warmups): + run_direct_control(args, arguments, f"warmup-control-{warmup}") + run_direct_overlap(args, arguments, f"warmup-overlap-{warmup}") + + generator = random.Random(args.seed) + pairs = [] + for pair_index in range(args.pairs): + order = ["control", "speculative"] + generator.shuffle(order) + rows: dict[str, dict[str, Any]] = {} + for arm in order: + rows[arm] = ( + run_direct_control( + args, arguments, f"pair-{pair_index}-control" + ) + if arm == "control" + else run_direct_overlap( + args, arguments, f"pair-{pair_index}-speculative" + ) + ) + pairs.append({"pair_index": pair_index, "arm_order": order, **rows}) + print( + json.dumps( + { + "phase": "qualify", + "pair": pair_index + 1, + "control_ms": round(rows["control"]["task_ms"], 3), + "overlap_ms": round(rows["speculative"]["task_ms"], 3), + "speedup": round( + rows["control"]["task_ms"] + / rows["speculative"]["task_ms"], + 3, + ), + }, + sort_keys=True, + ), + flush=True, + ) + + misses = [ + run_direct_miss(args, arguments, f"miss-{index}") + for index in range(args.miss_samples) + ] + controls = [pair["control"] for pair in pairs] + speculative = [pair["speculative"] for pair in pairs] + control_task = statistics.median(row["task_ms"] for row in controls) + speculative_task = statistics.median( + row["task_ms"] for row in speculative + ) + control_model = statistics.median( + row["model"]["model_compute_ms"] for row in controls + ) + speculative_model = statistics.median( + row["model"]["model_compute_ms"] for row in speculative + ) + miss_model = statistics.median( + row["model"]["model_compute_ms"] for row in misses + ) + slowdown_percent = 100.0 * ( + max(speculative_model, miss_model) / control_model - 1.0 + ) + expected_checksum = controls[0]["tool"]["result"]["checksum"] + canonical_model_identity = { + ( + row["model"]["tool_call_sha256"], + row["model"]["assistant_content_sha256"], + row["model"]["completion_tokens"], + ) + for row in controls + speculative + misses + } + all_tools_equal = all( + row["tool"]["result"]["checksum"] == expected_checksum + and row["tool"]["result"]["cpu_affinity"] == args.tool_cpus + for row in controls + speculative + ) and all( + row["authoritative_tool"]["result"]["checksum"] == expected_checksum + for row in misses + ) + ds4_active = all( + row["model"]["accept_rate"] > 0 + for row in controls + speculative + misses + ) + speedup = control_task / speculative_task + checks = { + "disjoint_cpu_affinity": not overlap, + "identical_model_outputs": len(canonical_model_identity) == 1, + "identical_tool_outputs": all_tools_equal, + "ds4_active": ds4_active, + "model_slowdown": slowdown_percent <= args.max_model_slowdown_percent, + "direct_speedup": speedup >= args.min_qualification_speedup, + "private_miss_result_hidden": all( + not row["private_result_exposed"] for row in misses + ), + } + passed = all(checks.values()) + profile = { + "profile_status": "qualified" if passed else "rejected", + "executor": "child_process_cpu_affinity", + "profile_kind": "disjoint_strix_cpu_sparse_compute", + "qualification": { + "host": "lucebox5", + "model_cpu_affinity": model_affinity, + "tool_cpu_affinity": args.tool_cpus, + "checks": checks, + }, + "path_summary": { + "100": { + "accelerator_relation": "non_accelerator", + "decode_interference_qualified": passed, + "hit": { + "control_task_mean_ms": statistics.fmean( + row["task_ms"] for row in controls + ), + "speculative_task_mean_ms": statistics.fmean( + row["task_ms"] for row in speculative + ), + "model_slowdown_percent": slowdown_percent, + }, + "miss": { + "control_task_mean_ms": statistics.fmean( + row["task_ms"] for row in controls + ), + "speculative_task_mean_ms": statistics.fmean( + row["task_ms"] for row in misses + ), + "model_slowdown_percent": slowdown_percent, + }, + } + }, + } + summary = { + "pairs": len(pairs), + "control_task_p50_ms": control_task, + "overlap_task_p50_ms": speculative_task, + "direct_exact_hit_speedup": speedup, + "control_model_compute_p50_ms": control_model, + "overlap_model_compute_p50_ms": speculative_model, + "model_compute_slowdown_percent": slowdown_percent, + "control_tool_wall_p50_ms": statistics.median( + row["tool"]["wall_ms"] for row in controls + ), + "overlap_tool_wall_p50_ms": statistics.median( + row["tool"]["wall_ms"] for row in speculative + ), + "miss_task_p50_ms": statistics.median( + row["task_ms"] for row in misses + ), + "median_accept_rate": statistics.median( + row["model"]["accept_rate"] + for row in controls + speculative + misses + ), + "checks": checks, + "passed": passed, + } + report = { + "phase": "qualification", + "host": "lucebox5", + "config": report_config(args, arguments), + "model_pid": args.model_pid, + "model_cpu_affinity": model_affinity, + "tool_cpu_affinity": args.tool_cpus, + "calibration": calibration, + "summary": summary, + "profile": profile, + "pairs": pairs, + "misses": misses, + } + write_report(args.output, report) + if passed: + write_report(args.profile_output, profile) + print(json.dumps(summary, indent=2, sort_keys=True), flush=True) + if not passed: + raise SystemExit("CPU-lane qualification failed") + + +def native_control( + args: argparse.Namespace, + arguments: dict[str, int], + label: str, +) -> dict[str, Any]: + direct = run_direct_control(args, arguments, label) + tool_result = direct["tool"]["result"] + return { + "mode": "control", + "task_ms": direct["task_ms"], + **direct["model"], + "tool_wall_ms": direct["tool"]["wall_ms"], + "tool_compute_ms": float(tool_result["compute_ms"]), + "tool_checksum": tool_result["checksum"], + "tool_cpu_affinity": tool_result["cpu_affinity"], + } + + +def native_speculative( + args: argparse.Namespace, + arguments: dict[str, int], + *, + prediction: dict[str, int] | None = None, +) -> dict[str, Any]: + row = post_model( + args.url, + arguments, + args.max_tokens, + args.timeout, + prediction=prediction or arguments, + ) + validate_model_call(row, arguments) + metadata = row["speculation"] if isinstance(row["speculation"], dict) else {} + tool_result = metadata.get("result", {}) + return { + "mode": "speculative", + "task_ms": row["request_wall_ms"], + **row, + "tool_wall_ms": float(metadata.get("executor_wall_ms", math.nan)), + "tool_compute_ms": float(tool_result.get("compute_ms", math.nan)), + "tool_checksum": tool_result.get("checksum"), + "tool_cpu_affinity": tool_result.get("cpu_affinity"), + } + + +def summarize_native( + pairs: list[dict[str, Any]], resamples: int, seed: int +) -> dict[str, Any]: + controls = [pair["control"] for pair in pairs] + speculative = [pair["speculative"] for pair in pairs] + control_task = statistics.median(row["task_ms"] for row in controls) + speculative_task = statistics.median( + row["task_ms"] for row in speculative + ) + control_model = statistics.median( + row["model_compute_ms"] for row in controls + ) + speculative_model = statistics.median( + row["model_compute_ms"] for row in speculative + ) + control_tool = statistics.median( + row["tool_compute_ms"] for row in controls + ) + speculative_tool = statistics.median( + row["tool_compute_ms"] for row in speculative + ) + return { + "pairs": len(pairs), + "control_task_p50_ms": control_task, + "speculative_task_p50_ms": speculative_task, + "exact_hit_speedup": control_task / speculative_task, + "exact_hit_speedup_bootstrap_95ci": bootstrap_speedup_ci( + pairs, resamples, seed + ), + "task_latency_reduction_percent": 100.0 + * (control_task - speculative_task) + / control_task, + "control_model_compute_p50_ms": control_model, + "speculative_model_compute_p50_ms": speculative_model, + "model_compute_slowdown_percent": 100.0 + * (speculative_model / control_model - 1.0), + "control_tool_compute_p50_ms": control_tool, + "speculative_tool_compute_p50_ms": speculative_tool, + "tool_compute_slowdown_percent": 100.0 + * (speculative_tool / control_tool - 1.0), + "latency_match_ratio": min(control_model, control_tool) + / max(control_model, control_tool), + "ideal_zero_interference_speedup_ceiling": ( + control_model + control_tool + ) + / max(control_model, control_tool), + "median_decode_tokens_per_sec": statistics.median( + row["decode_tokens_per_sec"] for row in controls + speculative + ), + "median_accept_rate": statistics.median( + row["accept_rate"] for row in controls + speculative + ), + "native_hits": sum( + isinstance(row["speculation"], dict) + and row["speculation"].get("status") == "hit" + for row in speculative + ), + "all_calls_identical": all( + pair["control"]["tool_call_sha256"] + == pair["speculative"]["tool_call_sha256"] + for pair in pairs + ), + "all_model_outputs_identical": all( + pair["control"]["assistant_content_sha256"] + == pair["speculative"]["assistant_content_sha256"] + and pair["control"]["completion_tokens"] + == pair["speculative"]["completion_tokens"] + for pair in pairs + ), + "all_tool_outputs_equivalent": all( + pair["control"]["tool_checksum"] + == pair["speculative"]["tool_checksum"] + and pair["control"]["tool_cpu_affinity"] + == pair["speculative"]["tool_cpu_affinity"] + for pair in pairs + ), + } + + +def native(args: argparse.Namespace) -> None: + props = get_json(props_url(args.url), args.timeout) + tool_props = props.get("tool_speculation") + if not isinstance(tool_props, dict) or not tool_props.get("enabled"): + raise SystemExit("server tool speculation is not enabled") + expected_props = { + "execution_mode": "child_process_cpu_affinity", + "profile_status": "qualified", + "compute_isolation": "disjoint_cpu_affinity", + "cpu_affinity_isolated": True, + "preserves_token_speculation": True, + } + for key, expected in expected_props.items(): + if tool_props.get(key) != expected: + raise SystemExit( + f"server tool_speculation.{key}={tool_props.get(key)!r}, " + f"expected {expected!r}" + ) + if tool_props.get("tool_cpu_affinity") != args.tool_cpus: + raise SystemExit("server tool CPU affinity differs from benchmark") + model_affinity = tool_props.get("model_cpu_affinity") + if not isinstance(model_affinity, list) or set(model_affinity) & set( + args.tool_cpus + ): + raise SystemExit("server model/tool CPU affinity is not disjoint") + + arguments = expected_arguments( + args.rows, + args.nonzeros_per_row, + args.iterations, + args.threads, + args.tool_seed, + ) + for warmup in range(args.warmups): + native_control(args, arguments, f"native-warm-control-{warmup}") + row = native_speculative(args, arguments) + if (row["speculation"] or {}).get("status") != "hit": + raise RuntimeError("native speculative warmup did not commit") + + generator = random.Random(args.seed) + pairs = [] + for pair_index in range(args.pairs): + order = ["control", "speculative"] + generator.shuffle(order) + rows: dict[str, dict[str, Any]] = {} + for arm in order: + rows[arm] = ( + native_control( + args, arguments, f"native-pair-{pair_index}-control" + ) + if arm == "control" + else native_speculative(args, arguments) + ) + pairs.append({"pair_index": pair_index, "arm_order": order, **rows}) + print( + json.dumps( + { + "phase": "native", + "pair": pair_index + 1, + "control_ms": round(rows["control"]["task_ms"], 3), + "speculative_ms": round( + rows["speculative"]["task_ms"], 3 + ), + "speedup": round( + rows["control"]["task_ms"] + / rows["speculative"]["task_ms"], + 3, + ), + "status": (rows["speculative"]["speculation"] or {}).get( + "status" + ), + }, + sort_keys=True, + ), + flush=True, + ) + + wrong = dict(arguments) + wrong["iterations"] = max(1, arguments["iterations"] - 1) + if wrong["iterations"] == arguments["iterations"]: + wrong["iterations"] += 1 + miss = native_speculative(args, arguments, prediction=wrong) + miss_metadata = miss["speculation"] or {} + miss_check = { + "passed": miss_metadata.get("status") == "miss" + and miss_metadata.get("reason") == "invocation_mismatch" + and "result" not in miss_metadata + and all( + miss["assistant_content_sha256"] + == pair["control"]["assistant_content_sha256"] + and miss["completion_tokens"] + == pair["control"]["completion_tokens"] + for pair in pairs + ), + "status": miss_metadata.get("status"), + "reason": miss_metadata.get("reason"), + "private_result_exposed": "result" in miss_metadata, + } + summary = summarize_native(pairs, args.bootstrap_resamples, args.seed) + correctness_passed = ( + summary["native_hits"] == args.pairs + and summary["all_calls_identical"] + and summary["all_model_outputs_identical"] + and summary["all_tool_outputs_equivalent"] + and miss_check["passed"] + and all( + pair[arm]["accept_rate"] > 0 + for pair in pairs + for arm in ("control", "speculative") + ) + ) + ci_low = summary["exact_hit_speedup_bootstrap_95ci"][0] + checks = { + "correctness": correctness_passed, + "strong_sequential_baseline": True, + "exact_hit_speedup": summary["exact_hit_speedup"] >= args.min_speedup, + "speedup_ci_low": ci_low >= args.min_speedup_ci_low, + "model_slowdown": summary["model_compute_slowdown_percent"] + <= args.max_model_slowdown_percent, + } + production_gate = { + "passed": all(checks.values()), + "checks": checks, + "thresholds": { + "min_exact_hit_speedup": args.min_speedup, + "min_speedup_ci_low": args.min_speedup_ci_low, + "max_model_slowdown_percent": args.max_model_slowdown_percent, + }, + } + report = { + "phase": "native_engine", + "host": "lucebox5", + "config": report_config(args, arguments), + "server_snapshot": { + "runtime": props.get("runtime"), + "speculative": props.get("speculative"), + "tool_speculation": tool_props, + }, + "methodology": { + "control": "model request followed by the identical CPU-pinned sparse tool", + "speculative": "engine starts the identical CPU-pinned sparse tool before DS4 generation", + "commit": "result exposed only after exact canonical call match", + "pairing": "randomized arm order within every warm pair", + }, + "correctness_passed": correctness_passed, + "production_gate": production_gate, + "miss_check": miss_check, + "summary": summary, + "pairs": pairs, + } + write_report(args.output, report) + print( + json.dumps( + { + "correctness_passed": correctness_passed, + "production_gate": production_gate, + "miss_check": miss_check, + "summary": summary, + }, + indent=2, + sort_keys=True, + ), + flush=True, + ) + if not production_gate["passed"]: + raise SystemExit("native CPU tool-speculation production gate failed") + + +def report_config( + args: argparse.Namespace, arguments: dict[str, int] +) -> dict[str, Any]: + return { + "url": args.url, + "binary": str(args.binary.resolve()), + "tool_arguments": arguments, + "fixed_sparse_shape": { + "rows": args.rows, + "nonzeros_per_row": args.nonzeros_per_row, + "threads": args.threads, + "seed": args.tool_seed, + }, + "tool_cpus": args.tool_cpus, + "max_tokens": args.max_tokens, + "pairs": args.pairs, + "warmups": args.warmups, + "seed": args.seed, + } + + +def write_report(path: Path | None, value: dict[str, Any]) -> None: + if path is None: + return + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(value, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def add_common_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "--url", default="http://127.0.0.1:18145/v1/chat/completions" + ) + parser.add_argument("--binary", type=Path, required=True) + parser.add_argument("--tool-cpus", type=parse_cpu_list, required=True) + parser.add_argument("--rows", type=int, default=4096) + parser.add_argument("--nonzeros-per-row", type=int, default=16) + parser.add_argument("--threads", type=int, default=2) + parser.add_argument("--tool-seed", type=int, default=731) + parser.add_argument("--max-tokens", type=int, default=32) + parser.add_argument("--pairs", type=int, default=20) + parser.add_argument("--warmups", type=int, default=2) + parser.add_argument("--timeout", type=float, default=180.0) + parser.add_argument("--seed", type=int, default=814) + parser.add_argument("--max-model-slowdown-percent", type=float, default=5.0) + parser.add_argument("--output", type=Path) + + +def main() -> None: + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="phase", required=True) + + qualify_parser = subparsers.add_parser("qualify") + add_common_arguments(qualify_parser) + qualify_parser.add_argument("--model-pid", type=int, required=True) + qualify_parser.add_argument("--initial-iterations", type=int, default=20) + qualify_parser.add_argument("--calibration-steps", type=int, default=5) + qualify_parser.add_argument("--calibration-model-samples", type=int, default=3) + qualify_parser.add_argument("--calibration-tool-samples", type=int, default=2) + qualify_parser.add_argument("--miss-samples", type=int, default=5) + qualify_parser.add_argument("--min-qualification-speedup", type=float, default=1.70) + qualify_parser.add_argument("--profile-output", type=Path, required=True) + + native_parser = subparsers.add_parser("native") + add_common_arguments(native_parser) + native_parser.add_argument("--iterations", type=int, required=True) + native_parser.add_argument("--bootstrap-resamples", type=int, default=20_000) + native_parser.add_argument("--min-speedup", type=float, default=1.80) + native_parser.add_argument("--min-speedup-ci-low", type=float, default=1.70) + + args = parser.parse_args() + if not args.binary.is_file(): + parser.error(f"executor binary does not exist: {args.binary}") + positive = [ + args.rows, + args.nonzeros_per_row, + args.threads, + args.max_tokens, + args.pairs, + args.warmups, + args.timeout, + ] + if any(value <= 0 for value in positive): + parser.error("counts and timeouts must be positive") + if args.tool_seed < 0 or args.max_model_slowdown_percent < 0: + parser.error("seed and slowdown threshold must be non-negative") + if args.phase == "qualify": + if ( + args.model_pid <= 0 + or args.initial_iterations <= 0 + or args.calibration_steps <= 0 + or args.calibration_model_samples <= 0 + or args.calibration_tool_samples <= 0 + or args.miss_samples <= 0 + or args.min_qualification_speedup <= 1.0 + ): + parser.error("qualification settings must be positive") + qualify(args) + else: + if ( + args.iterations <= 0 + or args.bootstrap_resamples <= 0 + or args.min_speedup <= 1.0 + or args.min_speedup_ci_low <= 1.0 + or args.min_speedup_ci_low > args.min_speedup + ): + parser.error("native benchmark settings are invalid") + native(args) + + +if __name__ == "__main__": + main() diff --git a/optimizations/ooo_spec_lucebox5_cpu/build_cpu_sparse_executor.sh b/optimizations/ooo_spec_lucebox5_cpu/build_cpu_sparse_executor.sh new file mode 100755 index 000000000..4e48334e7 --- /dev/null +++ b/optimizations/ooo_spec_lucebox5_cpu/build_cpu_sparse_executor.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +source_file="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/cpu_sparse_tool_executor.cpp" +json_include="${JSON_INCLUDE:-}" +output="${1:-$(dirname "$source_file")/cpu_sparse_tool_executor}" + +if [[ -z "$json_include" ]]; then + printf 'JSON_INCLUDE must point to the directory containing nlohmann/json.hpp\n' >&2 + exit 2 +fi +if [[ ! -f "$json_include/nlohmann/json.hpp" ]]; then + printf 'missing JSON header: %s/nlohmann/json.hpp\n' "$json_include" >&2 + exit 2 +fi + +g++ -std=c++17 -O3 -DNDEBUG -pthread \ + -I"$json_include" "$source_file" -o "$output" diff --git a/optimizations/ooo_spec_lucebox5_cpu/cpu_sparse_tool_executor.cpp b/optimizations/ooo_spec_lucebox5_cpu/cpu_sparse_tool_executor.cpp new file mode 100644 index 000000000..95be7dd7b --- /dev/null +++ b/optimizations/ooo_spec_lucebox5_cpu/cpu_sparse_tool_executor.cpp @@ -0,0 +1,259 @@ +// Deterministic read-only sparse-compute adapter for +// dflash.tool-speculation.v1 qualification. +// +// This is a benchmark tool, not an application-specific tool. It provides a +// reproducible CPU-bound workload whose exact result can be compared between +// sequential and speculative execution. The engine pins this child before it +// releases the JSON request, and the adapter verifies the observed mask. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__linux__) +#include +#include +#endif + +using json = nlohmann::json; + +namespace { + +constexpr const char * kProtocol = "dflash.tool-speculation.v1"; +constexpr const char * kToolName = "benchmark_cpu_sparse"; +constexpr int kRows = 4096; +constexpr int kNonzerosPerRow = 16; +constexpr int kThreads = 2; +constexpr uint64_t kSeed = 731; + +uint64_t splitmix64(uint64_t & state) { + uint64_t value = (state += 0x9e3779b97f4a7c15ULL); + value = (value ^ (value >> 30U)) * 0xbf58476d1ce4e5b9ULL; + value = (value ^ (value >> 27U)) * 0x94d049bb133111ebULL; + return value ^ (value >> 31U); +} + +std::vector observed_affinity() { +#if defined(__linux__) + cpu_set_t mask; + CPU_ZERO(&mask); + if (::sched_getaffinity(0, sizeof(mask), &mask) != 0) { + throw std::runtime_error("sched_getaffinity failed"); + } + std::vector cpus; + for (int cpu = 0; cpu < CPU_SETSIZE; ++cpu) { + if (CPU_ISSET(cpu, &mask)) cpus.push_back(cpu); + } + return cpus; +#else + return {}; +#endif +} + +uint64_t sparse_worker(int worker, + int rows, + int nonzeros_per_row, + int iterations, + uint64_t seed) { + const size_t entries = + static_cast(rows) * static_cast(nonzeros_per_row); + std::vector columns(entries); + std::vector values(entries); + std::vector input(static_cast(rows)); + std::vector output(static_cast(rows)); + + uint64_t state = seed ^ + (0xd6e8feb86659fd93ULL * static_cast(worker + 1)); + for (size_t index = 0; index < entries; ++index) { + columns[index] = static_cast(splitmix64(state) % rows); + values[index] = static_cast(splitmix64(state) | 1ULL); + } + for (uint32_t & value : input) { + value = static_cast(splitmix64(state)); + } + + uint64_t rolling = 0xcbf29ce484222325ULL ^ + static_cast(worker); + for (int iteration = 0; iteration < iterations; ++iteration) { + for (int row = 0; row < rows; ++row) { + uint64_t accumulator = + static_cast(iteration + 1) * 0x9e3779b1U + + static_cast(row); + const size_t start = + static_cast(row) * nonzeros_per_row; + for (int offset = 0; offset < nonzeros_per_row; ++offset) { + const size_t index = start + static_cast(offset); + accumulator += static_cast(values[index]) * + input[columns[index]]; + } + const uint32_t folded = static_cast( + accumulator ^ (accumulator >> 32U)); + output[static_cast(row)] = + folded + static_cast(row * 2654435761U); + } + input.swap(output); + rolling ^= static_cast(input[ + static_cast(iteration) % input.size()]); + rolling *= 0x100000001b3ULL; + } + for (size_t index = 0; index < input.size(); index += 17) { + rolling ^= static_cast(input[index]) + index; + rolling *= 0x100000001b3ULL; + } + return rolling; +} + +int integer_argument(const json & arguments, + const char * name, + int minimum, + int maximum) { + if (!arguments.contains(name) || !arguments[name].is_number_integer()) { + throw std::runtime_error(std::string(name) + " must be an integer"); + } + const int value = arguments[name].get(); + if (value < minimum || value > maximum) { + throw std::runtime_error( + std::string(name) + " is outside the allowed range"); + } + return value; +} + +json execute(const json & request) { + if (!request.is_object() || request.value("protocol", "") != kProtocol) { + throw std::runtime_error("unsupported protocol"); + } + if (!request.contains("call") || !request["call"].is_object() || + request["call"].value("name", "") != kToolName) { + throw std::runtime_error("only benchmark_cpu_sparse is allowed"); + } + const json & arguments = request["call"].at("arguments"); + if (!arguments.is_object()) { + throw std::runtime_error("arguments must be an object"); + } + const int rows = arguments.contains("rows") + ? integer_argument(arguments, "rows", 64, 1 << 20) : kRows; + const int nonzeros = arguments.contains("nonzeros_per_row") + ? integer_argument(arguments, "nonzeros_per_row", 1, 256) + : kNonzerosPerRow; + const int iterations = integer_argument( + arguments, "iterations", 1, 1'000'000); + const int threads = arguments.contains("threads") + ? integer_argument(arguments, "threads", 1, 64) : kThreads; + if (static_cast(rows) * static_cast(nonzeros) > + 16ULL * 1024ULL * 1024ULL) { + throw std::runtime_error("sparse matrix exceeds the 16M-entry limit"); + } + uint64_t seed = kSeed; + if (arguments.contains("seed")) { + if (!arguments["seed"].is_number_integer()) { + throw std::runtime_error("seed must be an unsigned integer"); + } + if (arguments["seed"].is_number_unsigned()) { + seed = arguments["seed"].get(); + } else { + const int64_t signed_seed = arguments["seed"].get(); + if (signed_seed < 0) { + throw std::runtime_error("seed must be an unsigned integer"); + } + seed = static_cast(signed_seed); + } + } + + std::vector expected_affinity; + if (request.contains("cpu_affinity")) { + expected_affinity = request["cpu_affinity"].get>(); + std::sort(expected_affinity.begin(), expected_affinity.end()); + expected_affinity.erase( + std::unique(expected_affinity.begin(), expected_affinity.end()), + expected_affinity.end()); + } + const std::vector affinity = observed_affinity(); + if (!expected_affinity.empty() && affinity != expected_affinity) { + throw std::runtime_error("observed CPU affinity does not match request"); + } + if (!affinity.empty() && threads > static_cast(affinity.size())) { + throw std::runtime_error("threads exceed the pinned logical CPU count"); + } + + const auto started = std::chrono::steady_clock::now(); + std::vector partial(static_cast(threads)); + std::vector pin_errors(static_cast(threads), 0); + std::vector workers; + workers.reserve(static_cast(threads)); + for (int worker = 0; worker < threads; ++worker) { + workers.emplace_back([&, worker]() { +#if defined(__linux__) + if (!affinity.empty()) { + cpu_set_t worker_mask; + CPU_ZERO(&worker_mask); + CPU_SET(affinity[static_cast(worker)], &worker_mask); + pin_errors[static_cast(worker)] = + ::pthread_setaffinity_np( + ::pthread_self(), sizeof(worker_mask), &worker_mask); + if (pin_errors[static_cast(worker)] != 0) return; + } +#endif + partial[static_cast(worker)] = sparse_worker( + worker, rows, nonzeros, iterations, seed); + }); + } + for (std::thread & worker : workers) worker.join(); + if (std::any_of(pin_errors.begin(), pin_errors.end(), + [](int error) { return error != 0; })) { + throw std::runtime_error("worker CPU pinning failed"); + } + const double compute_ms = + std::chrono::duration( + std::chrono::steady_clock::now() - started).count(); + + uint64_t checksum = 0x6a09e667f3bcc909ULL; + for (const uint64_t value : partial) { + checksum ^= value + 0x9e3779b97f4a7c15ULL + + (checksum << 6U) + (checksum >> 2U); + } + return { + {"ok", true}, + {"result", { + {"checksum", std::to_string(checksum)}, + {"compute_ms", compute_ms}, + {"rows", rows}, + {"nonzeros_per_row", nonzeros}, + {"iterations", iterations}, + {"threads", threads}, + {"seed", seed}, + {"cpu_affinity", affinity}, + {"worker_cpus", std::vector( + affinity.begin(), affinity.begin() + + std::min(affinity.size(), static_cast(threads)))}, + }}, + }; +} + +} // namespace + +int main(int argc, char ** argv) { + if (argc != 2 || std::string(argv[1]) != "--dflash-tool-spec-v1") { + std::cerr << "expected --dflash-tool-spec-v1\n"; + return 2; + } + try { + std::string line; + if (!std::getline(std::cin, line) || line.empty()) { + throw std::runtime_error("missing request"); + } + std::cout << execute(json::parse(line)).dump() << '\n'; + std::cout.flush(); + return 0; + } catch (const std::exception & exception) { + std::cerr << exception.what() << '\n'; + return 2; + } +} diff --git a/optimizations/ooo_spec_lucebox5_cpu/profiles/lucebox5-cpu-lane-qualified.json b/optimizations/ooo_spec_lucebox5_cpu/profiles/lucebox5-cpu-lane-qualified.json new file mode 100644 index 000000000..84b0ad107 --- /dev/null +++ b/optimizations/ooo_spec_lucebox5_cpu/profiles/lucebox5-cpu-lane-qualified.json @@ -0,0 +1,69 @@ +{ + "executor": "child_process_cpu_affinity", + "path_summary": { + "100": { + "accelerator_relation": "non_accelerator", + "decode_interference_qualified": true, + "hit": { + "control_task_mean_ms": 5581.2723303339835, + "model_slowdown_percent": 0.11341375399527287, + "speculative_task_mean_ms": 2973.236727998786 + }, + "miss": { + "control_task_mean_ms": 5581.2723303339835, + "model_slowdown_percent": 0.11341375399527287, + "speculative_task_mean_ms": 5550.103543003206 + } + } + }, + "profile_kind": "disjoint_strix_cpu_sparse_compute", + "profile_status": "qualified", + "qualification": { + "checks": { + "direct_speedup": true, + "disjoint_cpu_affinity": true, + "ds4_active": true, + "identical_model_outputs": true, + "identical_tool_outputs": true, + "model_slowdown": true, + "private_miss_result_hidden": true + }, + "host": "lucebox5", + "model_cpu_affinity": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29 + ], + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ] + } +} diff --git a/optimizations/ooo_spec_lucebox5_cpu/results/lucebox5-cpu-lane-qualification.json b/optimizations/ooo_spec_lucebox5_cpu/results/lucebox5-cpu-lane-qualification.json new file mode 100644 index 000000000..70c6c50d5 --- /dev/null +++ b/optimizations/ooo_spec_lucebox5_cpu/results/lucebox5-cpu-lane-qualification.json @@ -0,0 +1,1839 @@ +{ + "calibration": { + "model_samples": [ + { + "accept_rate": 0.625, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 7, + "decode_ms": 728.4, + "decode_tokens_per_sec": 9.6, + "model_compute_ms": 3426.5, + "prefill_ms": 2698.1, + "request_wall_ms": 3444.775058000232, + "speculation": null, + "tool_call": { + "arguments": { + "iterations": 20 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "a97210b345373f933eeb50349895292971f6d1bfd676f63e999ab9f178cdeba2" + }, + { + "accept_rate": 0.625, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 7, + "decode_ms": 346.4, + "decode_tokens_per_sec": 20.2, + "model_compute_ms": 3728.5, + "prefill_ms": 3382.1, + "request_wall_ms": 3730.295108995051, + "speculation": null, + "tool_call": { + "arguments": { + "iterations": 20 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "a97210b345373f933eeb50349895292971f6d1bfd676f63e999ab9f178cdeba2" + }, + { + "accept_rate": 0.625, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 7, + "decode_ms": 342.1, + "decode_tokens_per_sec": 20.5, + "model_compute_ms": 2619.1, + "prefill_ms": 2277.0, + "request_wall_ms": 2620.606353986659, + "speculation": null, + "tool_call": { + "arguments": { + "iterations": 20 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "a97210b345373f933eeb50349895292971f6d1bfd676f63e999ab9f178cdeba2" + }, + { + "accept_rate": 0.625, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 7, + "decode_ms": 342.3, + "decode_tokens_per_sec": 20.4, + "model_compute_ms": 2619.2000000000003, + "prefill_ms": 2276.9, + "request_wall_ms": 2620.5322190071456, + "speculation": null, + "tool_call": { + "arguments": { + "iterations": 20 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "a97210b345373f933eeb50349895292971f6d1bfd676f63e999ab9f178cdeba2" + }, + { + "accept_rate": 0.625, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 7, + "decode_ms": 342.5, + "decode_tokens_per_sec": 20.4, + "model_compute_ms": 2620.6, + "prefill_ms": 2278.1, + "request_wall_ms": 2622.0629360032035, + "speculation": null, + "tool_call": { + "arguments": { + "iterations": 20 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "a97210b345373f933eeb50349895292971f6d1bfd676f63e999ab9f178cdeba2" + } + ], + "selected_iterations": 172452, + "steps": [ + { + "iterations": 20, + "samples": [ + { + "result": { + "checksum": "18180057682806790565", + "compute_ms": 1.4058, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 20, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 4.593224002746865 + }, + { + "result": { + "checksum": "18180057682806790565", + "compute_ms": 1.296406, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 20, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 3.755755999009125 + }, + { + "result": { + "checksum": "18180057682806790565", + "compute_ms": 0.823681, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 20, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 2.570759999798611 + } + ], + "tool_wall_p50_ms": 3.755755999009125 + }, + { + "iterations": 13963, + "samples": [ + { + "result": { + "checksum": "13677167202453426650", + "compute_ms": 249.549945, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 13963, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 251.17609799781349 + }, + { + "result": { + "checksum": "13677167202453426650", + "compute_ms": 210.351586, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 13963, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 212.30194400413893 + }, + { + "result": { + "checksum": "13677167202453426650", + "compute_ms": 210.386741, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 13963, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 212.24752299895044 + } + ], + "tool_wall_p50_ms": 212.30194400413893 + }, + { + "iterations": 172452, + "samples": [ + { + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2552.662603, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 2554.3570239969995 + }, + { + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2546.354615, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 2548.043864997453 + }, + { + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2546.93318, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 2548.6412460013526 + } + ], + "tool_wall_p50_ms": 2548.6412460013526 + } + ], + "target_model_request_p50_ms": 2622.0629360032035 + }, + "config": { + "binary": "/home/lucebox5/tool-spec-cpu-20260813/cpu_sparse_tool_executor", + "fixed_sparse_shape": { + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2 + }, + "max_tokens": 32, + "pairs": 12, + "seed": 814, + "tool_arguments": { + "iterations": 172452 + }, + "tool_cpus": [ + 14, + 15, + 30, + 31 + ], + "url": "http://127.0.0.1:18145/v1/chat/completions", + "warmups": 2 + }, + "host": "lucebox5", + "misses": [ + { + "authoritative_tool": { + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2607.356476, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 2610.2510019991314 + }, + "mode": "miss", + "model": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 475.8, + "decode_tokens_per_sec": 16.8, + "model_compute_ms": 2893.1000000000004, + "prefill_ms": 2417.3, + "request_wall_ms": 2895.4200049920473, + "speculation": null, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" + }, + "private_result_exposed": false, + "task_ms": 5507.685866992688 + }, + { + "authoritative_tool": { + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2634.827473, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 2637.8311429871246 + }, + "mode": "miss", + "model": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 475.1, + "decode_tokens_per_sec": 16.8, + "model_compute_ms": 2913.1, + "prefill_ms": 2438.0, + "request_wall_ms": 2915.902043998358, + "speculation": null, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" + }, + "private_result_exposed": false, + "task_ms": 5555.7190620020265 + }, + { + "authoritative_tool": { + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2596.870639, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 2599.092707008822 + }, + "mode": "miss", + "model": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 475.4, + "decode_tokens_per_sec": 16.8, + "model_compute_ms": 2914.5, + "prefill_ms": 2439.1, + "request_wall_ms": 2917.0129179983633, + "speculation": null, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" + }, + "private_result_exposed": false, + "task_ms": 5517.983499012189 + }, + { + "authoritative_tool": { + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2717.568347, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 2720.183129000361 + }, + "mode": "miss", + "model": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 475.4, + "decode_tokens_per_sec": 16.8, + "model_compute_ms": 2912.8, + "prefill_ms": 2437.4, + "request_wall_ms": 2915.4062980087474, + "speculation": null, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" + }, + "private_result_exposed": false, + "task_ms": 5637.134058008087 + }, + { + "authoritative_tool": { + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2611.866018, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 2614.5963160088286 + }, + "mode": "miss", + "model": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 475.3, + "decode_tokens_per_sec": 16.8, + "model_compute_ms": 2913.0, + "prefill_ms": 2437.7, + "request_wall_ms": 2915.487136997399, + "speculation": null, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" + }, + "private_result_exposed": false, + "task_ms": 5531.995229001041 + } + ], + "model_cpu_affinity": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29 + ], + "model_pid": 498874, + "pairs": [ + { + "arm_order": [ + "speculative", + "control" + ], + "control": { + "mode": "control", + "model": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 477.4, + "decode_tokens_per_sec": 16.8, + "model_compute_ms": 2891.3, + "prefill_ms": 2413.9, + "request_wall_ms": 2892.969884997001, + "speculation": null, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" + }, + "task_ms": 5524.237097008154, + "tool": { + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2627.687774, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 2631.0234859993216 + } + }, + "pair_index": 0, + "speculative": { + "mode": "speculative", + "model": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 474.6, + "decode_tokens_per_sec": 16.9, + "model_compute_ms": 2888.0, + "prefill_ms": 2413.4, + "request_wall_ms": 2889.660949993413, + "speculation": null, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" + }, + "task_ms": 2891.6836440039333, + "tool": { + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2734.337179, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 2891.6460639884463 + } + } + }, + { + "arm_order": [ + "speculative", + "control" + ], + "control": { + "mode": "control", + "model": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 477.1, + "decode_tokens_per_sec": 16.8, + "model_compute_ms": 2892.2999999999997, + "prefill_ms": 2415.2, + "request_wall_ms": 2893.8748500077054, + "speculation": null, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" + }, + "task_ms": 5493.723748004413, + "tool": { + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2597.238064, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 2599.693889002083 + } + }, + "pair_index": 1, + "speculative": { + "mode": "speculative", + "model": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 474.7, + "decode_tokens_per_sec": 16.9, + "model_compute_ms": 2909.1, + "prefill_ms": 2434.4, + "request_wall_ms": 2911.0499909875216, + "speculation": null, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" + }, + "task_ms": 2912.997834995622, + "tool": { + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2714.367807, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 2912.963351001963 + } + } + }, + { + "arm_order": [ + "speculative", + "control" + ], + "control": { + "mode": "control", + "model": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 477.2, + "decode_tokens_per_sec": 16.8, + "model_compute_ms": 2892.5, + "prefill_ms": 2415.3, + "request_wall_ms": 2893.6694570002146, + "speculation": null, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" + }, + "task_ms": 5522.214145996259, + "tool": { + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2625.622342, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 2628.3829860039987 + } + }, + "pair_index": 2, + "speculative": { + "mode": "speculative", + "model": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 474.8, + "decode_tokens_per_sec": 16.8, + "model_compute_ms": 2909.8, + "prefill_ms": 2435.0, + "request_wall_ms": 2912.0525879989145, + "speculation": null, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" + }, + "task_ms": 2913.6232059972826, + "tool": { + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2743.90931, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 2913.592108001467 + } + } + }, + { + "arm_order": [ + "speculative", + "control" + ], + "control": { + "mode": "control", + "model": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 477.1, + "decode_tokens_per_sec": 16.8, + "model_compute_ms": 2909.0, + "prefill_ms": 2431.9, + "request_wall_ms": 2911.317157006124, + "speculation": null, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" + }, + "task_ms": 5842.6958240015665, + "tool": { + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2927.840256, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 2931.1076189915184 + } + }, + "pair_index": 3, + "speculative": { + "mode": "speculative", + "model": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 475.2, + "decode_tokens_per_sec": 16.8, + "model_compute_ms": 2921.5, + "prefill_ms": 2446.3, + "request_wall_ms": 2923.7128459935775, + "speculation": null, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" + }, + "task_ms": 2990.422590999515, + "tool": { + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2988.352859, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 2990.3616270021303 + } + } + }, + { + "arm_order": [ + "control", + "speculative" + ], + "control": { + "mode": "control", + "model": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 477.4, + "decode_tokens_per_sec": 16.8, + "model_compute_ms": 2912.7000000000003, + "prefill_ms": 2435.3, + "request_wall_ms": 2915.1544740016107, + "speculation": null, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" + }, + "task_ms": 5531.82867099531, + "tool": { + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2613.478577, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 2616.4246090047527 + } + }, + "pair_index": 4, + "speculative": { + "mode": "speculative", + "model": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 475.0, + "decode_tokens_per_sec": 16.8, + "model_compute_ms": 2910.8, + "prefill_ms": 2435.8, + "request_wall_ms": 2913.0806419998407, + "speculation": null, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" + }, + "task_ms": 2914.699681001366, + "tool": { + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2726.278932, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 2914.6627919981256 + } + } + }, + { + "arm_order": [ + "speculative", + "control" + ], + "control": { + "mode": "control", + "model": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 477.2, + "decode_tokens_per_sec": 16.8, + "model_compute_ms": 2891.5, + "prefill_ms": 2414.3, + "request_wall_ms": 2892.994167006691, + "speculation": null, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" + }, + "task_ms": 5518.224094994366, + "tool": { + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2622.666621, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 2625.0104759965325 + } + }, + "pair_index": 5, + "speculative": { + "mode": "speculative", + "model": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 474.7, + "decode_tokens_per_sec": 16.9, + "model_compute_ms": 2890.1, + "prefill_ms": 2415.4, + "request_wall_ms": 2891.9894639984705, + "speculation": null, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" + }, + "task_ms": 2893.9604710030835, + "tool": { + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2770.130032, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 2893.926097007352 + } + } + }, + { + "arm_order": [ + "control", + "speculative" + ], + "control": { + "mode": "control", + "model": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 477.4, + "decode_tokens_per_sec": 16.8, + "model_compute_ms": 2914.5, + "prefill_ms": 2437.1, + "request_wall_ms": 2917.0066920050886, + "speculation": null, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" + }, + "task_ms": 5918.36711500946, + "tool": { + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2998.242149, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 3001.084837989765 + } + }, + "pair_index": 6, + "speculative": { + "mode": "speculative", + "model": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 475.0, + "decode_tokens_per_sec": 16.8, + "model_compute_ms": 2911.9, + "prefill_ms": 2436.9, + "request_wall_ms": 2914.4634409894934, + "speculation": null, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" + }, + "task_ms": 2916.5130859910278, + "tool": { + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2736.112772, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 2916.4474730059737 + } + } + }, + { + "arm_order": [ + "speculative", + "control" + ], + "control": { + "mode": "control", + "model": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 477.5, + "decode_tokens_per_sec": 16.8, + "model_compute_ms": 2910.4, + "prefill_ms": 2432.9, + "request_wall_ms": 2912.770160997752, + "speculation": null, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" + }, + "task_ms": 5521.725176004111, + "tool": { + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2606.126825, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 2608.781943010399 + } + }, + "pair_index": 7, + "speculative": { + "mode": "speculative", + "model": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 475.2, + "decode_tokens_per_sec": 16.8, + "model_compute_ms": 2892.2, + "prefill_ms": 2417.0, + "request_wall_ms": 2894.1632219939493, + "speculation": null, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" + }, + "task_ms": 3026.5094359929208, + "tool": { + "result": { + "checksum": "9705095564492366076", + "compute_ms": 3023.66315, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 3026.4584609976737 + } + } + }, + { + "arm_order": [ + "control", + "speculative" + ], + "control": { + "mode": "control", + "model": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 476.4, + "decode_tokens_per_sec": 16.8, + "model_compute_ms": 2911.8, + "prefill_ms": 2435.4, + "request_wall_ms": 2914.0317879937356, + "speculation": null, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" + }, + "task_ms": 5525.245836994145, + "tool": { + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2608.567377, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 2611.0557520005386 + } + }, + "pair_index": 8, + "speculative": { + "mode": "speculative", + "model": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 475.0, + "decode_tokens_per_sec": 16.8, + "model_compute_ms": 2911.4, + "prefill_ms": 2436.4, + "request_wall_ms": 2913.8368099957006, + "speculation": null, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" + }, + "task_ms": 2915.164353995351, + "tool": { + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2755.945269, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 2915.1280960068107 + } + } + }, + { + "arm_order": [ + "speculative", + "control" + ], + "control": { + "mode": "control", + "model": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 477.5, + "decode_tokens_per_sec": 16.8, + "model_compute_ms": 2892.5, + "prefill_ms": 2415.0, + "request_wall_ms": 2894.126478000544, + "speculation": null, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" + }, + "task_ms": 5507.012105998001, + "tool": { + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2610.119884, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 2612.722923993715 + } + }, + "pair_index": 9, + "speculative": { + "mode": "speculative", + "model": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 475.1, + "decode_tokens_per_sec": 16.8, + "model_compute_ms": 2890.9, + "prefill_ms": 2415.8, + "request_wall_ms": 2893.0320760118775, + "speculation": null, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" + }, + "task_ms": 2894.950055007939, + "tool": { + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2749.14522, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 2894.9089880043175 + } + } + }, + { + "arm_order": [ + "control", + "speculative" + ], + "control": { + "mode": "control", + "model": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 477.5, + "decode_tokens_per_sec": 16.8, + "model_compute_ms": 2914.7, + "prefill_ms": 2437.2, + "request_wall_ms": 2917.2719510097522, + "speculation": null, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" + }, + "task_ms": 5515.812735000509, + "tool": { + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2595.765081, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 2598.3751849998953 + } + }, + "pair_index": 10, + "speculative": { + "mode": "speculative", + "model": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 475.5, + "decode_tokens_per_sec": 16.8, + "model_compute_ms": 2913.4, + "prefill_ms": 2437.9, + "request_wall_ms": 2915.5481519992463, + "speculation": null, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" + }, + "task_ms": 3491.498399002012, + "tool": { + "result": { + "checksum": "9705095564492366076", + "compute_ms": 3489.411735, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 3491.450679008267 + } + } + }, + { + "arm_order": [ + "control", + "speculative" + ], + "control": { + "mode": "control", + "model": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 477.5, + "decode_tokens_per_sec": 16.8, + "model_compute_ms": 2914.5, + "prefill_ms": 2437.0, + "request_wall_ms": 2917.092024013982, + "speculation": null, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" + }, + "task_ms": 5554.181414001505, + "tool": { + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2634.129552, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 2636.9060069991974 + } + }, + "pair_index": 11, + "speculative": { + "mode": "speculative", + "model": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 475.4, + "decode_tokens_per_sec": 16.8, + "model_compute_ms": 2912.4, + "prefill_ms": 2437.0, + "request_wall_ms": 2914.609836996533, + "speculation": null, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" + }, + "task_ms": 2916.817977995379, + "tool": { + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2758.995074, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "wall_ms": 2916.75399900123 + } + } + } + ], + "phase": "qualification", + "profile": { + "executor": "child_process_cpu_affinity", + "path_summary": { + "100": { + "accelerator_relation": "non_accelerator", + "decode_interference_qualified": true, + "hit": { + "control_task_mean_ms": 5581.2723303339835, + "model_slowdown_percent": 0.11341375399527287, + "speculative_task_mean_ms": 2973.236727998786 + }, + "miss": { + "control_task_mean_ms": 5581.2723303339835, + "model_slowdown_percent": 0.11341375399527287, + "speculative_task_mean_ms": 5550.103543003206 + } + } + }, + "profile_kind": "disjoint_strix_cpu_sparse_compute", + "profile_status": "qualified", + "qualification": { + "checks": { + "direct_speedup": true, + "disjoint_cpu_affinity": true, + "ds4_active": true, + "identical_model_outputs": true, + "identical_tool_outputs": true, + "model_slowdown": true, + "private_miss_result_hidden": true + }, + "host": "lucebox5", + "model_cpu_affinity": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29 + ], + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ] + } + }, + "summary": { + "checks": { + "direct_speedup": true, + "disjoint_cpu_affinity": true, + "ds4_active": true, + "identical_model_outputs": true, + "identical_tool_outputs": true, + "model_slowdown": true, + "private_miss_result_hidden": true + }, + "control_model_compute_p50_ms": 2909.7, + "control_task_p50_ms": 5523.225621502206, + "control_tool_wall_p50_ms": 2620.7175425006426, + "direct_exact_hit_speedup": 1.8948042658786695, + "median_accept_rate": 0.4166666567325592, + "miss_task_p50_ms": 5531.995229001041, + "model_compute_slowdown_percent": 0.11341375399527287, + "overlap_model_compute_p50_ms": 2910.3, + "overlap_task_p50_ms": 2914.9320174983586, + "overlap_tool_wall_p50_ms": 2914.895444002468, + "pairs": 12, + "passed": true + }, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ] +} diff --git a/optimizations/ooo_spec_lucebox5_cpu/results/lucebox5-cpu-native-20pairs.json b/optimizations/ooo_spec_lucebox5_cpu/results/lucebox5-cpu-native-20pairs.json new file mode 100644 index 000000000..393f209cf --- /dev/null +++ b/optimizations/ooo_spec_lucebox5_cpu/results/lucebox5-cpu-native-20pairs.json @@ -0,0 +1,2346 @@ +{ + "config": { + "binary": "/home/lucebox5/tool-spec-cpu-20260813/cpu_sparse_tool_executor", + "fixed_sparse_shape": { + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2 + }, + "max_tokens": 32, + "pairs": 20, + "seed": 814, + "tool_arguments": { + "iterations": 172452 + }, + "tool_cpus": [ + 14, + 15, + 30, + 31 + ], + "url": "http://127.0.0.1:18145/v1/chat/completions", + "warmups": 2 + }, + "correctness_passed": true, + "host": "lucebox5", + "methodology": { + "commit": "result exposed only after exact canonical call match", + "control": "model request followed by the identical CPU-pinned sparse tool", + "pairing": "randomized arm order within every warm pair", + "speculative": "engine starts the identical CPU-pinned sparse tool before DS4 generation" + }, + "miss_check": { + "passed": true, + "private_result_exposed": false, + "reason": "invocation_mismatch", + "status": "miss" + }, + "pairs": [ + { + "arm_order": [ + "speculative", + "control" + ], + "control": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 476.8, + "decode_tokens_per_sec": 16.8, + "mode": "control", + "model_compute_ms": 2886.7000000000003, + "prefill_ms": 2409.9, + "request_wall_ms": 2887.7563690039096, + "speculation": null, + "task_ms": 5497.278031994938, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2606.721652, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2609.3294839956798 + }, + "pair_index": 0, + "speculative": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 473.5, + "decode_tokens_per_sec": 16.9, + "mode": "speculative", + "model_compute_ms": 2884.6, + "prefill_ms": 2411.1, + "request_wall_ms": 2886.079392003012, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_de0927ff226ce042cdf4d97b", + "commit_signal_sent": false, + "commit_wait_ms": 0.018996, + "confidence": 1.0, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 2885.142969, + "expected_speedup": 1.8771705185044594, + "prediction": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2716.64606, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "status": "hit" + }, + "task_ms": 2886.079392003012, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2716.64606, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2885.142969 + } + }, + { + "arm_order": [ + "speculative", + "control" + ], + "control": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 476.9, + "decode_tokens_per_sec": 16.8, + "mode": "control", + "model_compute_ms": 2888.3, + "prefill_ms": 2411.4, + "request_wall_ms": 2889.4118949974654, + "speculation": null, + "task_ms": 5495.428775990149, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2602.850033, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2605.80943300738 + }, + "pair_index": 1, + "speculative": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 473.8, + "decode_tokens_per_sec": 16.9, + "mode": "speculative", + "model_compute_ms": 2903.6000000000004, + "prefill_ms": 2429.8, + "request_wall_ms": 2936.8787970015546, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_a780efe5dceb275f785f0531", + "commit_signal_sent": false, + "commit_wait_ms": 0.015519, + "confidence": 1.0, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 2904.636247, + "expected_speedup": 1.8771705185044594, + "prediction": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2715.15693, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "status": "hit" + }, + "task_ms": 2936.8787970015546, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2715.15693, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2904.636247 + } + }, + { + "arm_order": [ + "speculative", + "control" + ], + "control": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 476.7, + "decode_tokens_per_sec": 16.8, + "mode": "control", + "model_compute_ms": 2902.5, + "prefill_ms": 2425.8, + "request_wall_ms": 2904.134411000996, + "speculation": null, + "task_ms": 5490.023413003655, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2582.769682, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2585.659131000284 + }, + "pair_index": 2, + "speculative": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 474.5, + "decode_tokens_per_sec": 16.9, + "mode": "speculative", + "model_compute_ms": 2905.9, + "prefill_ms": 2431.4, + "request_wall_ms": 2953.2264759909594, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_22cf7759b3f52086d81a1a87", + "commit_signal_sent": true, + "commit_wait_ms": 44.199584, + "confidence": 1.0, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 2951.125814, + "expected_speedup": 1.8771705185044594, + "prediction": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2949.894469, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "status": "hit" + }, + "task_ms": 2953.2264759909594, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2949.894469, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2951.125814 + } + }, + { + "arm_order": [ + "speculative", + "control" + ], + "control": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 476.8, + "decode_tokens_per_sec": 16.8, + "mode": "control", + "model_compute_ms": 2888.2000000000003, + "prefill_ms": 2411.4, + "request_wall_ms": 2889.3942540016724, + "speculation": null, + "task_ms": 5515.519638996921, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2623.90572, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2625.9387350000907 + }, + "pair_index": 3, + "speculative": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 474.5, + "decode_tokens_per_sec": 16.9, + "mode": "speculative", + "model_compute_ms": 2904.5, + "prefill_ms": 2430.0, + "request_wall_ms": 2907.0747140067397, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_2fefe53d3cad565ed33dcfc1", + "commit_signal_sent": false, + "commit_wait_ms": 0.014087, + "confidence": 1.0, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 2905.407734, + "expected_speedup": 1.8771705185044594, + "prediction": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2757.805857, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "status": "hit" + }, + "task_ms": 2907.0747140067397, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2757.805857, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2905.407734 + } + }, + { + "arm_order": [ + "control", + "speculative" + ], + "control": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 477.1, + "decode_tokens_per_sec": 16.8, + "mode": "control", + "model_compute_ms": 2908.9, + "prefill_ms": 2431.8, + "request_wall_ms": 2910.745012006373, + "speculation": null, + "task_ms": 5511.091899999883, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2597.192803, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2600.07576100179 + }, + "pair_index": 4, + "speculative": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 474.5, + "decode_tokens_per_sec": 16.9, + "mode": "speculative", + "model_compute_ms": 2906.6, + "prefill_ms": 2432.1, + "request_wall_ms": 2909.174690998043, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_badafb42708c7fd7b1cb8f88", + "commit_signal_sent": false, + "commit_wait_ms": 0.019236, + "confidence": 1.0, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 2907.57131, + "expected_speedup": 1.8771705185044594, + "prediction": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2727.222714, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "status": "hit" + }, + "task_ms": 2909.174690998043, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2727.222714, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2907.57131 + } + }, + { + "arm_order": [ + "speculative", + "control" + ], + "control": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 477.8, + "decode_tokens_per_sec": 16.7, + "mode": "control", + "model_compute_ms": 2934.2000000000003, + "prefill_ms": 2456.4, + "request_wall_ms": 2935.454065009253, + "speculation": null, + "task_ms": 5563.332241988974, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2624.312122, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2627.6193329977104 + }, + "pair_index": 5, + "speculative": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 474.7, + "decode_tokens_per_sec": 16.9, + "mode": "speculative", + "model_compute_ms": 2884.5, + "prefill_ms": 2409.8, + "request_wall_ms": 2886.354169007973, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_75f7f3e0c38ee6c0b3928dbc", + "commit_signal_sent": false, + "commit_wait_ms": 0.014156, + "confidence": 1.0, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 2885.126902, + "expected_speedup": 1.8771705185044594, + "prediction": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2751.568559, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "status": "hit" + }, + "task_ms": 2886.354169007973, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2751.568559, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2885.126902 + } + }, + { + "arm_order": [ + "control", + "speculative" + ], + "control": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 477.4, + "decode_tokens_per_sec": 16.8, + "mode": "control", + "model_compute_ms": 2910.0, + "prefill_ms": 2432.6, + "request_wall_ms": 2914.380813992466, + "speculation": null, + "task_ms": 5517.575254998519, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2599.810015, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2602.895701988018 + }, + "pair_index": 6, + "speculative": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 474.5, + "decode_tokens_per_sec": 16.9, + "mode": "speculative", + "model_compute_ms": 2904.8, + "prefill_ms": 2430.3, + "request_wall_ms": 2910.256342001958, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_787231685d2800c0b1d9a7e1", + "commit_signal_sent": false, + "commit_wait_ms": 0.018144, + "confidence": 1.0, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 2905.826018, + "expected_speedup": 1.8771705185044594, + "prediction": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2781.764245, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "status": "hit" + }, + "task_ms": 2910.256342001958, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2781.764245, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2905.826018 + } + }, + { + "arm_order": [ + "speculative", + "control" + ], + "control": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 476.8, + "decode_tokens_per_sec": 16.8, + "mode": "control", + "model_compute_ms": 2887.3, + "prefill_ms": 2410.5, + "request_wall_ms": 2888.609096989967, + "speculation": null, + "task_ms": 5489.209855993977, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2597.462915, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2600.3301620075945 + }, + "pair_index": 7, + "speculative": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 474.6, + "decode_tokens_per_sec": 16.9, + "mode": "speculative", + "model_compute_ms": 2885.2999999999997, + "prefill_ms": 2410.7, + "request_wall_ms": 2887.2296159970574, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_ef49973cfda3a48ef0e760b4", + "commit_signal_sent": false, + "commit_wait_ms": 0.029695, + "confidence": 1.0, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 2885.999673, + "expected_speedup": 1.8771705185044594, + "prediction": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2777.924106, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "status": "hit" + }, + "task_ms": 2887.2296159970574, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2777.924106, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2885.999673 + } + }, + { + "arm_order": [ + "control", + "speculative" + ], + "control": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 477.4, + "decode_tokens_per_sec": 16.8, + "mode": "control", + "model_compute_ms": 2909.2000000000003, + "prefill_ms": 2431.8, + "request_wall_ms": 2911.187883990351, + "speculation": null, + "task_ms": 5520.180744992103, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2605.88394, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2608.771926999907 + }, + "pair_index": 8, + "speculative": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 474.7, + "decode_tokens_per_sec": 16.9, + "mode": "speculative", + "model_compute_ms": 2906.6, + "prefill_ms": 2431.9, + "request_wall_ms": 2910.375022998778, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_49a6db2f32b7708066aad513", + "commit_signal_sent": false, + "commit_wait_ms": 0.017313, + "confidence": 1.0, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 2908.617383, + "expected_speedup": 1.8771705185044594, + "prediction": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2727.319056, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "status": "hit" + }, + "task_ms": 2910.375022998778, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2727.319056, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2908.617383 + } + }, + { + "arm_order": [ + "speculative", + "control" + ], + "control": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 477.4, + "decode_tokens_per_sec": 16.8, + "mode": "control", + "model_compute_ms": 2895.0, + "prefill_ms": 2417.6, + "request_wall_ms": 2896.2876409932505, + "speculation": null, + "task_ms": 5509.29852599802, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2610.033131, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2612.8059619950363 + }, + "pair_index": 9, + "speculative": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 475.6, + "decode_tokens_per_sec": 16.8, + "mode": "speculative", + "model_compute_ms": 2920.6, + "prefill_ms": 2445.0, + "request_wall_ms": 2922.3824299988337, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_8e769b586f88a80d9a2ecbbb", + "commit_signal_sent": false, + "commit_wait_ms": 0.020619, + "confidence": 1.0, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 2921.317176, + "expected_speedup": 1.8771705185044594, + "prediction": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2768.185407, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "status": "hit" + }, + "task_ms": 2922.3824299988337, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2768.185407, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2921.317176 + } + }, + { + "arm_order": [ + "control", + "speculative" + ], + "control": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 478.0, + "decode_tokens_per_sec": 16.7, + "mode": "control", + "model_compute_ms": 2910.0, + "prefill_ms": 2432.0, + "request_wall_ms": 2912.1384430036414, + "speculation": null, + "task_ms": 5512.187327010906, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2596.584479, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2599.779108000803 + }, + "pair_index": 10, + "speculative": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 475.1, + "decode_tokens_per_sec": 16.8, + "mode": "speculative", + "model_compute_ms": 2907.7, + "prefill_ms": 2432.6, + "request_wall_ms": 2910.429274998023, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_aeebec7e380f1356a52b1436", + "commit_signal_sent": false, + "commit_wait_ms": 0.017904, + "confidence": 1.0, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 2908.812999, + "expected_speedup": 1.8771705185044594, + "prediction": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2752.299532, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "status": "hit" + }, + "task_ms": 2910.429274998023, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2752.299532, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2908.812999 + } + }, + { + "arm_order": [ + "control", + "speculative" + ], + "control": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 478.0, + "decode_tokens_per_sec": 16.7, + "mode": "control", + "model_compute_ms": 2926.6, + "prefill_ms": 2448.6, + "request_wall_ms": 2927.8773260011803, + "speculation": null, + "task_ms": 5546.5061139984755, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2614.751961, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2618.455734991585 + }, + "pair_index": 11, + "speculative": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 475.1, + "decode_tokens_per_sec": 16.8, + "mode": "speculative", + "model_compute_ms": 2909.5, + "prefill_ms": 2434.4, + "request_wall_ms": 2912.3231999983545, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_41a313b6d60b1a5ca8220181", + "commit_signal_sent": false, + "commit_wait_ms": 0.014998, + "confidence": 1.0, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 2910.675286, + "expected_speedup": 1.8771705185044594, + "prediction": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2754.031487, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "status": "hit" + }, + "task_ms": 2912.3231999983545, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2754.031487, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2910.675286 + } + }, + { + "arm_order": [ + "control", + "speculative" + ], + "control": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 477.4, + "decode_tokens_per_sec": 16.8, + "mode": "control", + "model_compute_ms": 2888.7000000000003, + "prefill_ms": 2411.3, + "request_wall_ms": 2889.886129007209, + "speculation": null, + "task_ms": 5489.258956004051, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2596.611067, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2599.2073560046265 + }, + "pair_index": 12, + "speculative": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 475.1, + "decode_tokens_per_sec": 16.8, + "mode": "speculative", + "model_compute_ms": 2906.5, + "prefill_ms": 2431.4, + "request_wall_ms": 2909.0337640082, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_fb33f7142390b5ffa3f1df08", + "commit_signal_sent": false, + "commit_wait_ms": 0.020308, + "confidence": 1.0, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 2907.440281, + "expected_speedup": 1.8771705185044594, + "prediction": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2758.809735, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "status": "hit" + }, + "task_ms": 2909.0337640082, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2758.809735, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2907.440281 + } + }, + { + "arm_order": [ + "control", + "speculative" + ], + "control": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 477.0, + "decode_tokens_per_sec": 16.8, + "mode": "control", + "model_compute_ms": 2889.3, + "prefill_ms": 2412.3, + "request_wall_ms": 2890.6500519951805, + "speculation": null, + "task_ms": 5495.200345001649, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2601.708584, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2604.3903139943723 + }, + "pair_index": 13, + "speculative": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 475.3, + "decode_tokens_per_sec": 16.8, + "mode": "speculative", + "model_compute_ms": 2947.5, + "prefill_ms": 2472.2, + "request_wall_ms": 2950.2172590000555, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_411fd3326040ca4e30bc7f32", + "commit_signal_sent": false, + "commit_wait_ms": 0.019516, + "confidence": 1.0, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 2948.600252, + "expected_speedup": 1.8771705185044594, + "prediction": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2944.545582, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "status": "hit" + }, + "task_ms": 2950.2172590000555, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2944.545582, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2948.600252 + } + }, + { + "arm_order": [ + "speculative", + "control" + ], + "control": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 477.7, + "decode_tokens_per_sec": 16.7, + "mode": "control", + "model_compute_ms": 2891.5, + "prefill_ms": 2413.8, + "request_wall_ms": 2892.8247750009177, + "speculation": null, + "task_ms": 5513.9780350000365, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2618.248832, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2620.997508012806 + }, + "pair_index": 14, + "speculative": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 475.4, + "decode_tokens_per_sec": 16.8, + "mode": "speculative", + "model_compute_ms": 2889.8, + "prefill_ms": 2414.4, + "request_wall_ms": 2891.653121012496, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_f7e244b68e2cc9eb95428449", + "commit_signal_sent": false, + "commit_wait_ms": 0.018024, + "confidence": 1.0, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 2890.557961, + "expected_speedup": 1.8771705185044594, + "prediction": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2765.571476, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "status": "hit" + }, + "task_ms": 2891.653121012496, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2765.571476, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2890.557961 + } + }, + { + "arm_order": [ + "speculative", + "control" + ], + "control": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 477.6, + "decode_tokens_per_sec": 16.7, + "mode": "control", + "model_compute_ms": 2890.5, + "prefill_ms": 2412.9, + "request_wall_ms": 2891.906609002035, + "speculation": null, + "task_ms": 5502.877983992221, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2607.65675, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2610.64586599241 + }, + "pair_index": 15, + "speculative": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 475.2, + "decode_tokens_per_sec": 16.8, + "mode": "speculative", + "model_compute_ms": 2942.2999999999997, + "prefill_ms": 2467.1, + "request_wall_ms": 2945.138353999937, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_0e513da24ccdc47dfbb286f2", + "commit_signal_sent": false, + "commit_wait_ms": 0.009428, + "confidence": 1.0, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 2943.285245, + "expected_speedup": 1.8771705185044594, + "prediction": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2751.061218, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "status": "hit" + }, + "task_ms": 2945.138353999937, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2751.061218, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2943.285245 + } + }, + { + "arm_order": [ + "control", + "speculative" + ], + "control": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 478.1, + "decode_tokens_per_sec": 16.7, + "mode": "control", + "model_compute_ms": 2947.2999999999997, + "prefill_ms": 2469.2, + "request_wall_ms": 2949.831121004536, + "speculation": null, + "task_ms": 5568.309862996102, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2614.703825, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2618.221951997839 + }, + "pair_index": 16, + "speculative": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 475.0, + "decode_tokens_per_sec": 16.8, + "mode": "speculative", + "model_compute_ms": 2910.8, + "prefill_ms": 2435.8, + "request_wall_ms": 2914.3550349981524, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_e2fabbd7938f66d73229dede", + "commit_signal_sent": false, + "commit_wait_ms": 0.022062, + "confidence": 1.0, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 2911.896423, + "expected_speedup": 1.8771705185044594, + "prediction": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2735.067036, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "status": "hit" + }, + "task_ms": 2914.3550349981524, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2735.067036, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2911.896423 + } + }, + { + "arm_order": [ + "control", + "speculative" + ], + "control": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 477.5, + "decode_tokens_per_sec": 16.8, + "mode": "control", + "model_compute_ms": 2889.6, + "prefill_ms": 2412.1, + "request_wall_ms": 2891.2249889981467, + "speculation": null, + "task_ms": 6090.181550011039, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 3195.757001, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 3198.7133949878626 + }, + "pair_index": 17, + "speculative": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 475.0, + "decode_tokens_per_sec": 16.8, + "mode": "speculative", + "model_compute_ms": 2906.6, + "prefill_ms": 2431.6, + "request_wall_ms": 2909.964589009178, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_5eba859117cad5dfe23f4821", + "commit_signal_sent": false, + "commit_wait_ms": 0.014517, + "confidence": 1.0, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 2907.821637, + "expected_speedup": 1.8771705185044594, + "prediction": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2777.633709, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "status": "hit" + }, + "task_ms": 2909.964589009178, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2777.633709, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2907.821637 + } + }, + { + "arm_order": [ + "control", + "speculative" + ], + "control": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 477.5, + "decode_tokens_per_sec": 16.8, + "mode": "control", + "model_compute_ms": 2890.1, + "prefill_ms": 2412.6, + "request_wall_ms": 2891.499435005244, + "speculation": null, + "task_ms": 5504.3630370055325, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2610.124635, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2612.7046740002697 + }, + "pair_index": 18, + "speculative": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 475.1, + "decode_tokens_per_sec": 16.8, + "mode": "speculative", + "model_compute_ms": 2908.5, + "prefill_ms": 2433.4, + "request_wall_ms": 2911.1464049929054, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_d3fd156d25c15849551e318e", + "commit_signal_sent": false, + "commit_wait_ms": 0.019105, + "confidence": 1.0, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 2909.505163, + "expected_speedup": 1.8771705185044594, + "prediction": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2744.908547, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "status": "hit" + }, + "task_ms": 2911.1464049929054, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2744.908547, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2909.505163 + } + }, + { + "arm_order": [ + "control", + "speculative" + ], + "control": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 478.6, + "decode_tokens_per_sec": 16.7, + "mode": "control", + "model_compute_ms": 2924.9, + "prefill_ms": 2446.3, + "request_wall_ms": 2926.6352289996576, + "speculation": null, + "task_ms": 5560.942076990614, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2630.064817, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2634.031902998686 + }, + "pair_index": 19, + "speculative": { + "accept_rate": 0.4166666567325592, + "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "completion_tokens": 8, + "decode_ms": 475.5, + "decode_tokens_per_sec": 16.8, + "mode": "speculative", + "model_compute_ms": 2946.7, + "prefill_ms": 2471.2, + "request_wall_ms": 2950.1483669882873, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_69bbfcf178f13d17d02fc907", + "commit_signal_sent": false, + "commit_wait_ms": 0.017122, + "confidence": 1.0, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 2948.194829, + "expected_speedup": 1.8771705185044594, + "prediction": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "checksum": "9705095564492366076", + "compute_ms": 2766.327547, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "iterations": 172452, + "nonzeros_per_row": 16, + "rows": 4096, + "seed": 731, + "threads": 2, + "worker_cpus": [ + 14, + 15 + ] + }, + "status": "hit" + }, + "task_ms": 2950.1483669882873, + "tool_call": { + "arguments": { + "iterations": 172452 + }, + "name": "benchmark_cpu_sparse" + }, + "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", + "tool_checksum": "9705095564492366076", + "tool_compute_ms": 2766.327547, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "tool_wall_ms": 2948.194829 + } + } + ], + "phase": "native_engine", + "production_gate": { + "checks": { + "correctness": true, + "exact_hit_speedup": true, + "model_slowdown": true, + "speedup_ci_low": true, + "strong_sequential_baseline": true + }, + "passed": true, + "thresholds": { + "max_model_slowdown_percent": 5.0, + "min_exact_hit_speedup": 1.8, + "min_speedup_ci_low": 1.7 + } + }, + "server_snapshot": { + "runtime": { + "backend": "hip", + "chunk": 2048, + "draft_device": null, + "draft_residency": "auto", + "fa_window": 0, + "kv_cache_k": "q4_0", + "kv_cache_v": "q4_0", + "lazy_draft": false, + "target_device": "hip:0", + "target_sharding": false + }, + "speculative": { + "ddtree_budget": null, + "enabled": false + }, + "tool_speculation": { + "allowed_tools": [ + "benchmark_cpu_sparse" + ], + "compute_isolation": "disjoint_cpu_affinity", + "cpu_affinity_isolated": true, + "enabled": true, + "execution_mode": "child_process_cpu_affinity", + "executor_contract": "child_process_cpu_affinity", + "hip_reserved_tool_compute_units": 0, + "hip_tool_device": null, + "max_model_slowdown_ratio": 1.05, + "model_cpu_affinity": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29 + ], + "model_expert_ownership_unique": false, + "model_routing_static": false, + "preserves_token_speculation": true, + "profile_lanes": [ + { + "accelerator_relation": "non_accelerator", + "decode_interference_qualified": true, + "model_slowdown_ratio": 1.0011341375399527, + "requires_static_model_routing": false, + "requires_unique_expert_ownership": false, + "resource_percentage": 100 + } + ], + "profile_status": "qualified", + "protocol": "dflash.tool-speculation.v1", + "requires_client_support": true, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "unqualified_lane_policy": "defer" + } + }, + "summary": { + "all_calls_identical": true, + "all_model_outputs_identical": true, + "all_tool_outputs_equivalent": true, + "control_model_compute_p50_ms": 2893.25, + "control_task_p50_ms": 5511.639613505395, + "control_tool_compute_p50_ms": 2607.189201, + "exact_hit_speedup": 1.8937725205440066, + "exact_hit_speedup_bootstrap_95ci": [ + 1.8858550837704051, + 1.8964496590120083 + ], + "ideal_zero_interference_speedup_ceiling": 1.9011282125637259, + "latency_match_ratio": 0.901128212563726, + "median_accept_rate": 0.4166666567325592, + "median_decode_tokens_per_sec": 16.8, + "model_compute_slowdown_percent": 0.46141881966645926, + "native_hits": 20, + "pairs": 20, + "speculative_model_compute_p50_ms": 2906.6, + "speculative_task_p50_ms": 2910.4021489984007, + "speculative_tool_compute_p50_ms": 2755.918672, + "task_latency_reduction_percent": 47.19534742679975, + "tool_compute_slowdown_percent": 5.704590635115925 + } +} diff --git a/optimizations/ooo_spec_lucebox5_cpu/run_native_cpu_server_lucebox5.sh b/optimizations/ooo_spec_lucebox5_cpu/run_native_cpu_server_lucebox5.sh new file mode 100755 index 000000000..7fdae08ed --- /dev/null +++ b/optimizations/ooo_spec_lucebox5_cpu/run_native_cpu_server_lucebox5.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -euo pipefail + +root="/home/lucebox5" +experiment="$root/tool-spec-cpu-20260813" +launcher="$experiment/run-deepseek-0731-cpu-tool.sh" +executor="$experiment/cpu_sparse_tool_executor" +profile="$experiment/profiles/lucebox5-cpu-lane-qualified.json" + +for required in "$launcher" "$executor" "$profile"; do + [[ -e "$required" ]] || { + printf 'missing required path: %s\n' "$required" >&2 + exit 2 + } +done +if pgrep -x dflash_server >/dev/null; then + printf 'a dflash_server is already running; refusing to overlap it\n' >&2 + exit 75 +fi +if fuser -s /dev/kfd 2>/dev/null; then + printf '/dev/kfd already has an owner; refusing to overlap it\n' >&2 + fuser -v /dev/kfd >&2 || true + exit 75 +fi + +exec env \ + HOME="$root" \ + USER="lucebox5" \ + PATH="$root/.local/bin:/opt/rocm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \ + ENGINE_DIR="$root/lucebox-engine-0731" \ + BUILD_DIR="$root/codex-ds4-tool-spec-fix-20260812/build-tool-spec" \ + QUALIFIED_CONFIG_DIR="/opt/lucebox-manage/qualified/r9700_deepseek/runtime-config" \ + TARGET_MODEL="$root/lucebox-models/DeepSeek-V4-Flash-0731-ROCMFPX-MIX-STRIX.gguf" \ + DRAFT_MODEL="$root/lucebox-models/DeepSeek-V4-Flash-0731-DSpark-draft-Q4RMFP4-denseF16.gguf" \ + SERVER_PORT="18145" \ + MODEL_CPU_AFFINITY="0-13,16-29" \ + TOOL_SPEC_EXECUTOR="$executor" \ + TOOL_SPEC_PROFILE="$profile" \ + TOOL_SPEC_ALLOW="benchmark_cpu_sparse" \ + TOOL_SPEC_CPU_AFFINITY="14-15,30-31" \ + TOOL_SPEC_MAX_MODEL_SLOWDOWN="1.05" \ + LUCEBOX_INFERENCE_PROFILE="quality" \ + "$launcher" diff --git a/optimizations/ooo_spec_lucebox5_cpu/test_benchmark_cpu_tool_speculation.py b/optimizations/ooo_spec_lucebox5_cpu/test_benchmark_cpu_tool_speculation.py new file mode 100644 index 000000000..cf8df1a45 --- /dev/null +++ b/optimizations/ooo_spec_lucebox5_cpu/test_benchmark_cpu_tool_speculation.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import argparse +import unittest + +from benchmark_cpu_tool_speculation import ( + TOOL_NAME, + expected_arguments, + parse_cpu_list, + props_url, + request_body, +) + + +class CpuToolSpeculationBenchmarkTest(unittest.TestCase): + def test_cpu_list_parser_canonicalizes_ranges(self) -> None: + self.assertEqual(parse_cpu_list("30-31,15,14-15"), [14, 15, 30, 31]) + with self.assertRaises(argparse.ArgumentTypeError): + parse_cpu_list("14,,15") + + def test_request_has_exact_concrete_prediction(self) -> None: + arguments = expected_arguments(4096, 16, 77, 2, 731) + self.assertEqual(arguments, {"iterations": 77}) + body = request_body(arguments, 32, prediction=arguments) + self.assertEqual( + body["tool_speculation"]["call"], + {"name": TOOL_NAME, "arguments": arguments}, + ) + self.assertEqual(body["tool_speculation"]["confidence"], 1.0) + self.assertEqual(body["tools"][0]["name"], TOOL_NAME) + + def test_props_url_uses_server_origin(self) -> None: + self.assertEqual( + props_url("http://127.0.0.1:18145/v1/chat/completions?x=1"), + "http://127.0.0.1:18145/props", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/server/src/server/http_server.cpp b/server/src/server/http_server.cpp index bf8b0ad43..11218f8f1 100644 --- a/server/src/server/http_server.cpp +++ b/server/src/server/http_server.cpp @@ -817,8 +817,16 @@ json build_props_body(const ServerConfig & config, {"model_expert_ownership_unique", config.tool_speculation.model_expert_ownership_unique}, {"compute_isolation", - config.tool_speculation.hip_reserved_tool_compute_units > 0 - ? "disjoint_hip_cu_masks" : "none"}, + config.tool_speculation.cpu_affinity_isolated + ? "disjoint_cpu_affinity" + : config.tool_speculation.hip_reserved_tool_compute_units > 0 + ? "disjoint_hip_cu_masks" : "none"}, + {"cpu_affinity_isolated", + config.tool_speculation.cpu_affinity_isolated}, + {"tool_cpu_affinity", + config.tool_speculation.cpu_affinity}, + {"model_cpu_affinity", + config.tool_speculation.model_cpu_affinity}, {"hip_tool_device", config.tool_speculation.hip_tool_device >= 0 ? json(config.tool_speculation.hip_tool_device) diff --git a/server/src/server/server_main.cpp b/server/src/server/server_main.cpp index 936a7f41c..7dc9b28ed 100644 --- a/server/src/server/server_main.cpp +++ b/server/src/server/server_main.cpp @@ -189,6 +189,10 @@ static void print_usage(const char * prog) { #endif " --tool-spec-profile Measured resource-lane frontier JSON.\n" " --tool-spec-allow Allow one read-only/idempotent tool; repeatable.\n" + " --tool-spec-cpu-affinity \n" + " Pin child tools to Linux CPUs/ranges, e.g.\n" + " 14-15,30-31. The model process affinity\n" + " must exclude every listed CPU.\n" " --tool-spec-timeout-ms Executor result timeout (default: 60000).\n" " --tool-spec-max-model-slowdown \n" " Reject lanes slower than this inference\n" @@ -576,6 +580,15 @@ int main(int argc, char ** argv) { return 2; } sconfig.tool_speculation.allowed_tools.push_back(name); + } else if (std::strcmp(argv[i], "--tool-spec-cpu-affinity") == 0 && + i + 1 < argc) { + std::string affinity_error; + if (!parse_tool_speculation_cpu_affinity( + argv[++i], sconfig.tool_speculation.cpu_affinity, + affinity_error)) { + std::fprintf(stderr, "[server] %s\n", affinity_error.c_str()); + return 2; + } } else if (std::strcmp(argv[i], "--tool-spec-timeout-ms") == 0 && i + 1 < argc) { sconfig.tool_speculation.timeout_ms = std::atoi(argv[++i]); @@ -680,7 +693,8 @@ int main(int argc, char ** argv) { !sconfig.tool_speculation.executor_path.empty() || static_cast(sconfig.tool_speculation.in_process_executor) || !sconfig.tool_speculation.profile_path.empty() || - !sconfig.tool_speculation.allowed_tools.empty(); + !sconfig.tool_speculation.allowed_tools.empty() || + !sconfig.tool_speculation.cpu_affinity.empty(); if (tool_speculation_requested) { sconfig.tool_speculation.model_routing_static = !environment_flag_enabled( @@ -722,6 +736,19 @@ int main(int argc, char ** argv) { std::fprintf(stderr, "[server] %s\n", profile_error.c_str()); return 2; } + std::string cpu_affinity_error; + if (!qualify_tool_speculation_cpu_affinity( + sconfig.tool_speculation, cpu_affinity_error)) { + std::fprintf(stderr, "[server] %s\n", cpu_affinity_error.c_str()); + return 2; + } + if (sconfig.tool_speculation.cpu_affinity_isolated) { + std::fprintf(stderr, + "[server] disjoint CPU tool lane: %zu model logical CPUs, " + "%zu reserved tool logical CPUs\n", + sconfig.tool_speculation.model_cpu_affinity.size(), + sconfig.tool_speculation.cpu_affinity.size()); + } const std::string & executor_contract = sconfig.tool_speculation.policy.executor_contract(); if (!executor_contract.empty() && diff --git a/server/src/server/tool_speculation.cpp b/server/src/server/tool_speculation.cpp index f5fdd221f..f3a055eba 100644 --- a/server/src/server/tool_speculation.cpp +++ b/server/src/server/tool_speculation.cpp @@ -19,6 +19,9 @@ # include # include # include +# if defined(__linux__) +# include +# endif extern char ** environ; #endif @@ -52,6 +55,15 @@ bool request_declares_tool(const json & tools, const std::string & name) { return false; } +std::string format_cpu_affinity(const std::vector & cpus) { + std::string value; + for (const int cpu : cpus) { + if (!value.empty()) value.push_back(','); + value += std::to_string(cpu); + } + return value; +} + #if !defined(_WIN32) bool send_all_socket(int fd, const void * data, size_t bytes) { const char * cursor = static_cast(data); @@ -74,7 +86,8 @@ bool send_all_socket(int fd, const void * data, size_t bytes) { std::vector executor_environment( int resource_percentage, - const std::string & accelerator_relation) { + const std::string & accelerator_relation, + const std::vector & cpu_affinity) { std::vector values; static constexpr const char * kResourceKey = "DFLASH_TOOL_SPECULATION_RESOURCE_PERCENTAGE="; @@ -84,8 +97,13 @@ std::vector executor_environment( "DFLASH_TOOL_SPECULATION_ACCELERATOR_RELATION="; static constexpr size_t kRelationKeyLen = sizeof("DFLASH_TOOL_SPECULATION_ACCELERATOR_RELATION=") - 1; + static constexpr const char * kCpuAffinityKey = + "DFLASH_TOOL_SPECULATION_CPU_AFFINITY="; + static constexpr size_t kCpuAffinityKeyLen = + sizeof("DFLASH_TOOL_SPECULATION_CPU_AFFINITY=") - 1; bool resource_replaced = false; bool relation_replaced = false; + bool cpu_affinity_replaced = false; for (char ** item = environ; item && *item; ++item) { const std::string value(*item); if (value.compare(0, kResourceKeyLen, kResourceKey) == 0) { @@ -97,6 +115,16 @@ std::vector executor_environment( values.push_back( std::string(kRelationKey) + accelerator_relation); relation_replaced = true; + } else if (value.compare( + 0, kCpuAffinityKeyLen, kCpuAffinityKey) == 0) { + // Drop a stale inherited value when this executor has no CPU + // lane. Otherwise replace it with the verified canonical list. + if (!cpu_affinity.empty()) { + values.push_back( + std::string(kCpuAffinityKey) + + format_cpu_affinity(cpu_affinity)); + cpu_affinity_replaced = true; + } } else { values.push_back(value); } @@ -109,9 +137,56 @@ std::vector executor_environment( if (!relation_replaced) { values.push_back(std::string(kRelationKey) + accelerator_relation); } + if (!cpu_affinity.empty() && !cpu_affinity_replaced) { + values.push_back( + std::string(kCpuAffinityKey) + + format_cpu_affinity(cpu_affinity)); + } values.push_back("DFLASH_TOOL_SPECULATION=1"); return values; } + +# if defined(__linux__) +bool pin_and_verify_child_cpu_affinity( + pid_t child, + const std::vector & cpus, + std::string & error) { + if (cpus.empty()) { + error.clear(); + return true; + } + cpu_set_t requested; + CPU_ZERO(&requested); + for (const int cpu : cpus) { + if (cpu < 0 || cpu >= CPU_SETSIZE) { + error = "executor CPU is outside CPU_SETSIZE: " + + std::to_string(cpu); + return false; + } + CPU_SET(cpu, &requested); + } + if (::sched_setaffinity(child, sizeof(requested), &requested) != 0) { + error = std::string("executor sched_setaffinity failed: ") + + std::strerror(errno); + return false; + } + cpu_set_t observed; + CPU_ZERO(&observed); + if (::sched_getaffinity(child, sizeof(observed), &observed) != 0) { + error = std::string("executor sched_getaffinity failed: ") + + std::strerror(errno); + return false; + } + for (int cpu = 0; cpu < CPU_SETSIZE; ++cpu) { + if (CPU_ISSET(cpu, &requested) != CPU_ISSET(cpu, &observed)) { + error = "executor CPU affinity verification mismatch"; + return false; + } + } + error.clear(); + return true; +} +# endif #endif } // namespace @@ -202,6 +277,135 @@ bool parse_tool_speculation_prediction( return true; } +bool parse_tool_speculation_cpu_affinity( + const std::string & value, + std::vector & out, + std::string & error) { + out.clear(); + if (value.empty()) { + error = "tool CPU affinity must not be empty"; + return false; + } + size_t cursor = 0; + while (cursor < value.size()) { + const size_t comma = value.find(',', cursor); + const size_t end = comma == std::string::npos ? value.size() : comma; + const std::string token = value.substr(cursor, end - cursor); + if (token.empty()) { + error = "tool CPU affinity contains an empty item"; + out.clear(); + return false; + } + const size_t dash = token.find('-'); + auto parse_cpu = [&](const std::string & item, int & cpu) { + if (item.empty() || !std::all_of( + item.begin(), item.end(), [](unsigned char character) { + return character >= '0' && character <= '9'; + })) { + return false; + } + char * parsed_end = nullptr; + errno = 0; + const long parsed = std::strtol(item.c_str(), &parsed_end, 10); + if (errno != 0 || !parsed_end || *parsed_end != '\0' || + parsed < 0 || parsed > std::numeric_limits::max()) { + return false; + } + cpu = static_cast(parsed); + return true; + }; + int first = -1; + int last = -1; + if (dash == std::string::npos) { + if (!parse_cpu(token, first)) { + error = "invalid tool CPU affinity item: " + token; + out.clear(); + return false; + } + last = first; + } else if (token.find('-', dash + 1) != std::string::npos || + !parse_cpu(token.substr(0, dash), first) || + !parse_cpu(token.substr(dash + 1), last) || + first > last) { + error = "invalid tool CPU affinity range: " + token; + out.clear(); + return false; + } + if (static_cast(last) - + static_cast(first) > 65535ULL) { + error = "tool CPU affinity range is too large: " + token; + out.clear(); + return false; + } + for (int cpu = first; cpu <= last; ++cpu) { + out.push_back(cpu); + if (cpu == std::numeric_limits::max()) break; + } + if (comma == std::string::npos) break; + cursor = comma + 1; + } + std::sort(out.begin(), out.end()); + out.erase(std::unique(out.begin(), out.end()), out.end()); + error.clear(); + return true; +} + +bool qualify_tool_speculation_cpu_affinity( + ToolSpeculationConfig & config, + std::string & error) { + config.model_cpu_affinity.clear(); + config.cpu_affinity_isolated = false; + if (config.cpu_affinity.empty()) { + error.clear(); + return true; + } +#if defined(__linux__) + if (config.executor_path.empty() || config.in_process_executor) { + error = "tool CPU affinity requires a child-process executor"; + return false; + } + const long configured_cpus = ::sysconf(_SC_NPROCESSORS_CONF); + if (configured_cpus <= 0) { + error = "cannot determine configured CPU count"; + return false; + } + cpu_set_t model_set; + CPU_ZERO(&model_set); + if (::sched_getaffinity(0, sizeof(model_set), &model_set) != 0) { + error = std::string("model sched_getaffinity failed: ") + + std::strerror(errno); + return false; + } + for (int cpu = 0; cpu < CPU_SETSIZE; ++cpu) { + if (CPU_ISSET(cpu, &model_set)) { + config.model_cpu_affinity.push_back(cpu); + } + } + if (config.model_cpu_affinity.empty()) { + error = "model CPU affinity is empty"; + return false; + } + for (const int cpu : config.cpu_affinity) { + if (cpu < 0 || cpu >= CPU_SETSIZE || cpu >= configured_cpus) { + error = "tool CPU is not configured on this host: " + + std::to_string(cpu); + return false; + } + if (CPU_ISSET(cpu, &model_set)) { + error = "tool CPU affinity overlaps model CPU " + + std::to_string(cpu); + return false; + } + } + config.cpu_affinity_isolated = true; + error.clear(); + return true; +#else + error = "tool CPU affinity isolation is supported only on Linux"; + return false; +#endif +} + bool ToolSpeculationPolicy::load_file( const std::string & path, std::string & error) { std::ifstream input(path); @@ -490,6 +694,8 @@ void ToolSpeculationAttempt::start() { {"mode", "speculative"}, {"resource_percentage", admission_.resource_percentage}, {"accelerator_relation", admission_.accelerator_relation}, + {"cpu_affinity", config_.cpu_affinity}, + {"cpu_affinity_isolated", config_.cpu_affinity_isolated}, {"call", { {"name", prediction_.call.name}, {"arguments", prediction_.call.arguments}, @@ -565,7 +771,8 @@ void ToolSpeculationAttempt::start() { } std::vector env_storage = executor_environment( - admission_.resource_percentage, admission_.accelerator_relation); + admission_.resource_percentage, admission_.accelerator_relation, + config_.cpu_affinity); std::vector env; env.reserve(env_storage.size() + 1); for (std::string & value : env_storage) env.push_back(value.data()); @@ -596,6 +803,32 @@ void ToolSpeculationAttempt::start() { return; } +# if defined(__linux__) + if (!config_.cpu_affinity.empty()) { + std::string affinity_error; + if (!pin_and_verify_child_cpu_affinity( + child, config_.cpu_affinity, affinity_error)) { + ::kill(child, SIGKILL); + int child_status = 0; + while (::waitpid(child, &child_status, 0) < 0 && errno == EINTR) {} + ::close(input_socket[0]); + ::close(output_pipe[0]); + launch_error_ = std::move(affinity_error); + return; + } + } +# else + if (!config_.cpu_affinity.empty()) { + ::kill(child, SIGKILL); + int child_status = 0; + while (::waitpid(child, &child_status, 0) < 0 && errno == EINTR) {} + ::close(input_socket[0]); + ::close(output_pipe[0]); + launch_error_ = "tool CPU affinity isolation is supported only on Linux"; + return; + } +# endif + child_pid_ = static_cast(child); child_stdin_fd_ = input_socket[0]; child_stdout_fd_ = output_pipe[0]; @@ -640,6 +873,8 @@ json ToolSpeculationAttempt::base_metadata() const { admission_.admitted ? json(admission_.accelerator_relation) : json(nullptr)}, + {"cpu_affinity", config_.cpu_affinity}, + {"cpu_affinity_isolated", config_.cpu_affinity_isolated}, }; return metadata; } diff --git a/server/src/server/tool_speculation.h b/server/src/server/tool_speculation.h index 955c337d8..0e862a683 100644 --- a/server/src/server/tool_speculation.h +++ b/server/src/server/tool_speculation.h @@ -55,6 +55,12 @@ bool parse_tool_speculation_prediction(const json & value, ToolSpeculationPrediction & out, std::string & error); +// Parse a Linux CPU-list such as "14-15,30-31". The result is sorted and +// deduplicated so it can be compared directly with an observed affinity mask. +bool parse_tool_speculation_cpu_affinity(const std::string & value, + std::vector & out, + std::string & error); + struct ToolSpeculationLane { // Backend-neutral executor capacity. A CUDA adapter may map this to an // MPS share; a ROCm, CPU, I/O, or remote adapter may interpret it using @@ -160,6 +166,12 @@ struct ToolSpeculationConfig { // complementary CU masks. Zero means no model-side CU reservation. int hip_tool_device = -1; int hip_reserved_tool_compute_units = 0; + // Optional child-process CPU lane. Startup verifies that these logical + // CPUs are disjoint from the model process affinity; every child is pinned + // and re-read before its request payload is released. + std::vector cpu_affinity; + std::vector model_cpu_affinity; + bool cpu_affinity_isolated = false; bool enabled() const { return (!executor_path.empty() || in_process_executor) && !allowed_tools.empty() && @@ -168,11 +180,21 @@ struct ToolSpeculationConfig { const char * execution_mode() const { return in_process_executor ? in_process_executor->mode_name() - : executor_path.empty() ? "disabled" : "child_process"; + : executor_path.empty() + ? "disabled" + : cpu_affinity.empty() + ? "child_process" + : "child_process_cpu_affinity"; } bool allows(const std::string & name) const; }; +// Capture the model process affinity and fail closed unless it is physically +// disjoint from the configured child executor CPUs. No-op when no CPU lane is +// requested. +bool qualify_tool_speculation_cpu_affinity(ToolSpeculationConfig & config, + std::string & error); + // One request-scoped attempt. The configured executable is invoked without a // shell and receives one JSON request on stdin. It must emit one JSON envelope // on stdout: {"ok":true,"result":...}. Stdin remains open for a later diff --git a/server/test/test_server_unit.cpp b/server/test/test_server_unit.cpp index f10ec0ac7..83a7ddc25 100644 --- a/server/test/test_server_unit.cpp +++ b/server/test/test_server_unit.cpp @@ -4641,6 +4641,9 @@ TEST_CASE(ServerUnitFixture, test_props_tool_speculation_shape) { TEST_ASSERT(disabled["model_routing_static"].get()); TEST_ASSERT(disabled["model_expert_ownership_unique"].get()); TEST_ASSERT(disabled["compute_isolation"].get() == "none"); + TEST_ASSERT(!disabled["cpu_affinity_isolated"].get()); + TEST_ASSERT(disabled["tool_cpu_affinity"].empty()); + TEST_ASSERT(disabled["model_cpu_affinity"].empty()); TEST_ASSERT(disabled["hip_tool_device"].is_null()); TEST_ASSERT(disabled["hip_reserved_tool_compute_units"].get() == 0); TEST_ASSERT(disabled["profile_lanes"].empty()); @@ -4691,6 +4694,20 @@ TEST_CASE(ServerUnitFixture, test_props_tool_speculation_shape) { TEST_ASSERT(!enabled["profile_lanes"][0] ["requires_unique_expert_ownership"].get()); + cfg.tool_speculation.cpu_affinity = {14, 30}; + cfg.tool_speculation.model_cpu_affinity = {0, 1, 2, 3}; + cfg.tool_speculation.cpu_affinity_isolated = true; + body = build_props_body(cfg, pc, tm); + const json & cpu_isolated = body["tool_speculation"]; + TEST_ASSERT(cpu_isolated["compute_isolation"].get() == + "disjoint_cpu_affinity"); + TEST_ASSERT(cpu_isolated["cpu_affinity_isolated"].get()); + TEST_ASSERT(cpu_isolated["tool_cpu_affinity"] == + json::array({14, 30})); + TEST_ASSERT(cpu_isolated["model_cpu_affinity"] == + json::array({0, 1, 2, 3})); + + cfg.tool_speculation.cpu_affinity_isolated = false; cfg.tool_speculation.hip_tool_device = 1; cfg.tool_speculation.hip_reserved_tool_compute_units = 1; body = build_props_body(cfg, pc, tm); diff --git a/server/test/test_tool_speculation.cpp b/server/test/test_tool_speculation.cpp index baa1f1518..ed377f74c 100644 --- a/server/test/test_tool_speculation.cpp +++ b/server/test/test_tool_speculation.cpp @@ -14,6 +14,9 @@ #if !defined(_WIN32) # include # include +# if defined(__linux__) +# include +# endif #endif using dflash::common::ApiFormat; @@ -27,6 +30,8 @@ using dflash::common::ToolSpeculationPolicy; using dflash::common::ToolSpeculationPrediction; using dflash::common::json; using dflash::common::parse_tool_speculation_prediction; +using dflash::common::parse_tool_speculation_cpu_affinity; +using dflash::common::qualify_tool_speculation_cpu_affinity; using dflash::common::render_tool_speculation_sse; namespace { @@ -234,6 +239,19 @@ TEST_CASE(ToolSpeculationFixture, prediction_requires_declared_tool) { CHECK(error.find("not declared") != std::string::npos); } +TEST_CASE(ToolSpeculationFixture, cpu_affinity_parser_canonicalizes_ranges) { + std::vector cpus; + std::string error; + CHECK(parse_tool_speculation_cpu_affinity( + "30-31,15,14-15", cpus, error)); + CHECK(cpus == std::vector({14, 15, 30, 31})); + + CHECK(!parse_tool_speculation_cpu_affinity("14,,15", cpus, error)); + CHECK(cpus.empty()); + CHECK(!parse_tool_speculation_cpu_affinity("15-14", cpus, error)); + CHECK(cpus.empty()); +} + TEST_CASE(ToolSpeculationFixture, empirical_policy_selects_resource_by_confidence) { ToolSpeculationPolicy policy; std::string error; @@ -413,6 +431,71 @@ TEST_CASE(ToolSpeculationFixture, in_process_mismatch_cancels_private_result) { } #if !defined(_WIN32) +TEST_CASE(ToolSpeculationFixture, cpu_affinity_reaches_child_executor) { +#if defined(__linux__) + cpu_set_t allowed; + CPU_ZERO(&allowed); + CHECK(::sched_getaffinity(0, sizeof(allowed), &allowed) == 0); + int selected_cpu = -1; + for (int cpu = 0; cpu < CPU_SETSIZE; ++cpu) { + if (CPU_ISSET(cpu, &allowed)) { + selected_cpu = cpu; + break; + } + } + CHECK(selected_cpu >= 0); + const std::string selected = std::to_string(selected_cpu); + const std::string path = make_executor_script( + "IFS= read -r control\n" + "observed=$(awk '/Cpus_allowed_list/{print $2}' /proc/self/status)\n" + "printf '{\"ok\":true,\"result\":{\"configured\":\"%s\"," + "\"observed\":\"%s\"}}\\n' " + "\"$DFLASH_TOOL_SPECULATION_CPU_AFFINITY\" \"$observed\"\n"); + ToolSpeculationConfig config = test_config(path); + config.cpu_affinity = {selected_cpu}; + config.cpu_affinity_isolated = true; + CHECK(std::string(config.execution_mode()) == + "child_process_cpu_affinity"); + auto attempt = ToolSpeculationAttempt::create( + config, prediction(), "request_cpu_affinity"); + attempt->start(); + const json metadata = attempt->resolve({ + ToolCall{"call_1", "lookup", R"({"a":1,"b":2})"}, + }); + ::unlink(path.c_str()); + CHECK(metadata["status"] == "hit"); + CHECK(metadata["cpu_affinity_isolated"].get()); + CHECK(metadata["result"]["configured"] == selected); + CHECK(metadata["result"]["observed"] == selected); +#else + CHECK(true); +#endif +} + +TEST_CASE(ToolSpeculationFixture, cpu_affinity_qualification_rejects_overlap) { +#if defined(__linux__) + cpu_set_t allowed; + CPU_ZERO(&allowed); + CHECK(::sched_getaffinity(0, sizeof(allowed), &allowed) == 0); + int selected_cpu = -1; + for (int cpu = 0; cpu < CPU_SETSIZE; ++cpu) { + if (CPU_ISSET(cpu, &allowed)) { + selected_cpu = cpu; + break; + } + } + CHECK(selected_cpu >= 0); + ToolSpeculationConfig config = test_config(); + config.cpu_affinity = {selected_cpu}; + std::string error; + CHECK(!qualify_tool_speculation_cpu_affinity(config, error)); + CHECK(error.find("overlaps model CPU") != std::string::npos); + CHECK(!config.cpu_affinity_isolated); +#else + CHECK(true); +#endif +} + TEST_CASE(ToolSpeculationFixture, exact_match_exposes_result_and_resource_share) { const std::string control_path = make_temp_path(); const std::string path = make_executor_script( From a04869e4cfc04b730b323d3fb10e758f053ec1b6 Mon Sep 17 00:00:00 2001 From: mrciffa <49000955+davide221@users.noreply.github.com> Date: Sat, 15 Aug 2026 17:45:37 +0200 Subject: [PATCH 03/11] feat(server): add model-agnostic tool speculation --- docs/specs/tool-speculation.md | 100 ++++++ server/CMakeLists.txt | 15 +- server/src/server/http_server.cpp | 160 ++++++++- server/src/server/http_server.h | 8 + server/src/server/server_main.cpp | 81 +++++ server/src/server/tool_speculation.cpp | 453 +++++++++++++++++++++++++ server/src/server/tool_speculation.h | 117 +++++++ server/test/test_server_unit.cpp | 16 + server/test/test_tool_speculation.cpp | 209 ++++++++++++ 9 files changed, 1154 insertions(+), 5 deletions(-) create mode 100644 docs/specs/tool-speculation.md create mode 100644 server/src/server/tool_speculation.cpp create mode 100644 server/src/server/tool_speculation.h create mode 100644 server/test/test_tool_speculation.cpp diff --git a/docs/specs/tool-speculation.md b/docs/specs/tool-speculation.md new file mode 100644 index 000000000..015ba35e7 --- /dev/null +++ b/docs/specs/tool-speculation.md @@ -0,0 +1,100 @@ +# Model-agnostic tool speculation + +Tool speculation lets an external service predict and start one safe tool +before local model execution. The model remains authoritative: the engine +returns the private result only when the generated function name and canonical +JSON arguments match the prediction exactly. + +The engine does not load or identify the predictor model. Qwen, another model, +rules, or a cached predictor can implement the same protocol. Hardware and CPU +affinity also belong to the service, keeping DS4, DSpark, and autoregressive +decoding unchanged. + +## Configuration + +```text +--tool-spec-endpoint http://127.0.0.1:19090/v1/speculate +--tool-spec-allow get_weather +--tool-spec-allow search_documents +``` + +Only read-only or idempotent tools should be allowlisted. Optional controls: + +```text +--tool-spec-key +--tool-spec-min-confidence 0.75 +--tool-spec-start-timeout-ms 2000 +--tool-spec-finish-timeout-ms 60000 +``` + +The predictor runs before local prompt preparation and decoding. A successful +`start` response therefore means the tool is already running when model compute +begins. Predictor failure only disables speculation for that request. + +## Service protocol + +Every request and response is JSON. The current protocol identifier is +`dflash.tool-speculation.v1`. + +The engine starts an attempt with the normalized conversation and only the +allowlisted tool definitions: + +```json +{ + "protocol": "dflash.tool-speculation.v1", + "operation": "start", + "request_id": "chatcmpl_...", + "messages": [{"role": "user", "content": "Weather in Rome"}], + "tools": [{"name": "get_weather", "parameters": {"type": "object"}}], + "tool_choice": "auto", + "min_confidence": 0.75 +} +``` + +The service predicts a call, starts it privately, then responds: + +```json +{ + "ticket": "opaque-service-ticket", + "call": { + "name": "get_weather", + "arguments": {"city": "Rome", "unit": "celsius"} + }, + "confidence": 0.91 +} +``` + +The engine rejects unknown tools, malformed arguments, and predictions below +its threshold. Rejected tickets receive `cancel`. + +After generation, one exact call receives `commit`: + +```json +{ + "protocol": "dflash.tool-speculation.v1", + "operation": "commit", + "request_id": "chatcmpl_...", + "ticket": "opaque-service-ticket" +} +``` + +The service waits for the already-running tool if necessary and returns: + +```json +{"ok": true, "result": {"temperature": 24}} +``` + +A wrong prediction, generation failure, or disconnect receives `cancel` with a +reason. The service should acknowledge cancellation promptly and must never +expose a private result through that response. It must also expire abandoned +tickets, because a timeout can prevent the engine from delivering `commit` or +`cancel` reliably. + +Non-streaming responses expose the outcome under +`dflash_tool_speculation`. Streaming responses emit one API-shaped extension +immediately before the normal terminal event. On a miss, this metadata contains +no `result` field. A compatible client uses `result` as the completed output for +`call_id` only when `status` is `hit`, and must not invoke that call a second +time. For `miss`, `deferred`, `cancelled`, or `failed`, it follows the normal +tool-execution path. Enable the server feature only for clients that understand +this additive extension. diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index 0ed5743fb..c9a9b84dc 100644 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -468,6 +468,7 @@ add_library(dflash_common STATIC src/server/chat_template.cpp src/server/tool_parser.cpp src/server/tool_hint.cpp + src/server/tool_speculation.cpp src/server/reasoning.cpp src/server/tool_memory.cpp src/server/sse_emitter.cpp @@ -1557,6 +1558,16 @@ if(DFLASH27B_TESTS) list(APPEND _raw_unit_test_targets test_server_unit) endif() + add_executable(test_tool_speculation + test/test_unit_main.cpp + test/test_tool_speculation.cpp + src/server/tool_speculation.cpp) + target_include_directories(test_tool_speculation PRIVATE + ${DFLASH27B_SRC_INCLUDE_DIRS}) + target_link_libraries(test_tool_speculation PRIVATE + nlohmann_json::nlohmann_json) + list(APPEND _raw_unit_test_targets test_tool_speculation) + # Feature/architecture gate tests. check_feature_compatibility(), # collect_feature_warnings() and the capability table are pure functions, # so this target deliberately compiles only feature_gate.cpp and @@ -1741,7 +1752,9 @@ if(DFLASH27B_SERVER) if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/server/server_main.cpp") find_package(CURL QUIET) if(NOT CURL_FOUND) - message(WARNING "CURL not found — building dflash_server without passthrough proxy") + message(WARNING + "CURL not found — building dflash_server without passthrough " + "proxy or tool speculation") endif() add_executable(dflash_server src/server/server_main.cpp diff --git a/server/src/server/http_server.cpp b/server/src/server/http_server.cpp index 57b204246..04b3c4140 100644 --- a/server/src/server/http_server.cpp +++ b/server/src/server/http_server.cpp @@ -436,6 +436,86 @@ static bool curl_forward(const std::string & url, curl_easy_cleanup(curl); return res == CURLE_OK && response_sent; } + +struct CurlJsonResponse { + std::string body; + bool too_large = false; +}; + +static size_t curl_write_json( + char * ptr, size_t size, size_t nmemb, void * userdata) { + constexpr size_t kMaxResponseBytes = 4 * 1024 * 1024; + const size_t total = size * nmemb; + auto * response = static_cast(userdata); + if (total > kMaxResponseBytes - response->body.size()) { + response->too_large = true; + return 0; + } + response->body.append(ptr, total); + return total; +} + +static bool curl_post_json( + const std::string & url, + const std::string & api_key, + const json & request, + int timeout_ms, + json & response, + std::string & error) { + CURL * curl = curl_easy_init(); + if (!curl) { + error = "curl initialization failed"; + return false; + } + + const std::string body = request.dump(); + struct curl_slist * headers = nullptr; + headers = curl_slist_append(headers, "Content-Type: application/json"); + if (!api_key.empty()) { + const std::string auth = "Authorization: Bearer " + api_key; + headers = curl_slist_append(headers, auth.c_str()); + } + CurlJsonResponse output; + char curl_error[CURL_ERROR_SIZE] = {}; + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str()); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, (long)body.size()); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS, + (long)(std::min)(timeout_ms, 5000)); + curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, (long)timeout_ms); + curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write_json); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &output); + curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_error); + + const CURLcode result = curl_easy_perform(curl); + long status = 0; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status); + curl_slist_free_all(headers); + curl_easy_cleanup(curl); + + if (output.too_large) { + error = "response exceeds 4 MiB"; + return false; + } + if (result != CURLE_OK) { + error = curl_error[0] ? curl_error : curl_easy_strerror(result); + return false; + } + if (status < 200 || status >= 300) { + error = "HTTP status " + std::to_string(status); + return false; + } + try { + response = json::parse(output.body); + } catch (const std::exception & exception) { + error = std::string("invalid JSON response: ") + exception.what(); + return false; + } + error.clear(); + return true; +} #endif // DFLASH_HAS_CURL // ─── /props constants ─────────────────────────────────────────────────── @@ -809,6 +889,13 @@ json build_props_body(const ServerConfig & config, {"ddtree_budget", config.speculative_enabled ? json(config.ddtree_budget) : json(nullptr)}, }}, + {"tool_speculation", { + {"enabled", config.tool_speculation.enabled()}, + {"protocol", kToolSpeculationProtocol}, + {"schedule", "before_model"}, + {"provider", "external"}, + {"allowed_tools", config.tool_speculation.allowed_tools}, + }}, {"sampling", { {"capabilities", { {"supports_temperature", true}, @@ -1993,6 +2080,17 @@ bool HttpServer::route_request(SocketHandle fd, const HttpRequest & hr) { const std::vector chat_messages = normalize_chat_messages(req.messages, req.format, tool_memory_); + req.tool_speculation_messages = json::array(); + for (const ChatMessage & message : chat_messages) { + json normalized = { + {"role", message.role}, + {"content", message.content}, + }; + if (!message.tool_call_id.empty()) { + normalized["tool_call_id"] = message.tool_call_id; + } + req.tool_speculation_messages.push_back(std::move(normalized)); + } // Reasoning must be applied BEFORE rendering: the template injects // the empty \n\n\n\n block when thinking is disabled. apply_request_reasoning(body, req); @@ -3644,12 +3742,35 @@ void HttpServer::process_job(ServerJob * job) { } if (req.stream) start_job_stream(job); + // Predictor-first scheduling is intentional. The external service must + // return only after the private tool has started, so its model compute + // cannot contend with the target decoder. The target backend is untouched. + ToolSpeculationTransport tool_transport; +#ifdef DFLASH_HAS_CURL + if (config_.tool_speculation.enabled()) { + const std::string endpoint = config_.tool_speculation.endpoint; + const std::string api_key = config_.tool_speculation.api_key; + tool_transport = [endpoint, api_key]( + const json & request, int timeout_ms, + json & response, std::string & error) { + return curl_post_json(endpoint, api_key, request, timeout_ms, + response, error); + }; + } +#endif + auto tool_speculation = ToolSpeculationAttempt::begin( + config_.tool_speculation, req.response_id, + req.tool_speculation_messages, req.tools, req.tool_choice, + std::move(tool_transport)); + PreparedPrompt prepared = prepare_prompt(req); if (prepared.error_status != 0) { + if (tool_speculation) tool_speculation->cancel("prompt_rejected"); fail_request(prepared.error_status, prepared.error); return; } if (forward_upstream(job, req, prepared)) { + if (tool_speculation) tool_speculation->cancel("upstream_generation"); finish_job(); return; } @@ -3777,14 +3898,41 @@ void HttpServer::process_job(ServerJob * job) { status_.update_completion_tokens(completion_tokens); broadcast_status(); } - // Serialize final frames after disabling heartbeat comments so no comment - // can appear after the protocol's [DONE] marker. - stop_job_stream(job); if (job->client_disconnected.load(std::memory_order_acquire)) { client_disconnected = true; } + json tool_speculation_metadata; + bool has_tool_speculation_metadata = false; + if (tool_speculation) { + has_tool_speculation_metadata = true; + if (client_disconnected) { + tool_speculation_metadata = + tool_speculation->cancel("client_disconnected"); + } else if (!result.ok()) { + tool_speculation_metadata = + tool_speculation->cancel("generation_failed"); + } else { + tool_speculation_metadata = + tool_speculation->finish(emitter.tool_calls()); + } + } + + // Serialize final frames after disabling heartbeat comments so no comment + // can appear after the protocol's terminal event. Heartbeats stay active + // while a successful commit waits for the external tool to finish. + stop_job_stream(job); if (req.stream && !client_disconnected) { auto final_chunks = emitter.emit_finish(completion_tokens, &gen_timings); + if (has_tool_speculation_metadata) { + const std::string extension = render_tool_speculation_sse( + req.format, req.response_id, req.model, + tool_speculation_metadata); + if (final_chunks.empty()) { + final_chunks.push_back(extension); + } else { + final_chunks.insert(final_chunks.end() - 1, extension); + } + } for (const auto & chunk : final_chunks) { if (!send_job_bytes(job, chunk.data(), chunk.size())) { client_disconnected = true; @@ -3792,8 +3940,12 @@ void HttpServer::process_job(ServerJob * job) { } } } else if (!req.stream && !client_disconnected) { - const json response = build_non_streaming_response( + json response = build_non_streaming_response( req, result, n_gen_cap, gen_timings, tokenizer_, emitter); + if (has_tool_speculation_metadata) { + response["dflash_tool_speculation"] = + std::move(tool_speculation_metadata); + } // Streaming uses non-blocking sends; restore blocking mode before // writing a complete JSON response on this shared socket path. const int flags = sock_get_flags(fd); diff --git a/server/src/server/http_server.h b/server/src/server/http_server.h index 132c8ff55..a0b6889c4 100644 --- a/server/src/server/http_server.h +++ b/server/src/server/http_server.h @@ -18,6 +18,7 @@ #include "tokenizer.h" #include "chat_template.h" #include "tool_memory.h" +#include "tool_speculation.h" #include "prefix_cache.h" #include "disk_prefix_cache.h" #include "freeze_history.h" @@ -217,6 +218,11 @@ struct ServerConfig { // Routing data collection (--collect-routing ): write binary per-token // routing data (hidden states + expert selections) for predictor training. std::string collect_routing_path; + + // Optional external service for model-agnostic speculative tool execution. + // The service starts work before model compute; exact-call verification + // remains inside the engine. + ToolSpeculationConfig tool_speculation; }; namespace http_detail { @@ -251,6 +257,8 @@ struct ParsedRequest { json tool_choice; // Original messages (for response formatting) json messages; + // Provider-neutral role/content messages for tool speculation. + json tool_speculation_messages; // Original request body (for upstream proxy forwarding) json raw_body; // Response ID diff --git a/server/src/server/server_main.cpp b/server/src/server/server_main.cpp index 545c32830..d6b43ef07 100644 --- a/server/src/server/server_main.cpp +++ b/server/src/server/server_main.cpp @@ -26,6 +26,7 @@ #include "placement/draft_residency.h" #include +#include #include #include #include @@ -174,6 +175,14 @@ static void print_usage(const char * prog) { " --prefill-upstream-key Bearer token for the upstream.\n" " --prefill-upstream-model Model name on forwarded requests.\n" "\n" + "Speculative tools (optional external service):\n" + " --tool-spec-endpoint Model-agnostic speculation service.\n" + " --tool-spec-key Optional bearer token.\n" + " --tool-spec-allow Read-only/idempotent tool; repeat per tool.\n" + " --tool-spec-min-confidence

Admission threshold (default: 0.75).\n" + " --tool-spec-start-timeout-ms Predictor/start timeout (default: 2000).\n" + " --tool-spec-finish-timeout-ms Tool-result timeout (default: 60000).\n" + "\n" "Disk KV cache:\n" " --kv-cache-dir Directory for ondisk KV cache (enables feature)\n" " --kv-cache-budget Max disk usage in MB (default: 4096)\n" @@ -506,6 +515,42 @@ int main(int argc, char ** argv) { } else if (std::strcmp(argv[i], "--lazy-draft") == 0) { sconfig.lazy_draft = true; sconfig.draft_residency = DraftResidencyPolicy::RequestScoped; + } else if (std::strcmp(argv[i], "--tool-spec-endpoint") == 0 && + i + 1 < argc) { + sconfig.tool_speculation.endpoint = argv[++i]; + } else if (std::strcmp(argv[i], "--tool-spec-key") == 0 && + i + 1 < argc) { + sconfig.tool_speculation.api_key = argv[++i]; + } else if (std::strcmp(argv[i], "--tool-spec-allow") == 0 && + i + 1 < argc) { + sconfig.tool_speculation.allowed_tools.emplace_back(argv[++i]); + } else if (std::strcmp(argv[i], "--tool-spec-min-confidence") == 0 && + i + 1 < argc) { + char * end = nullptr; + const double value = std::strtod(argv[++i], &end); + if (!end || *end != '\0' || !std::isfinite(value) || + value < 0.0 || value > 1.0) { + std::fprintf(stderr, + "[server] --tool-spec-min-confidence must be between 0 and 1\n"); + return 2; + } + sconfig.tool_speculation.min_confidence = value; + } else if (std::strcmp(argv[i], "--tool-spec-start-timeout-ms") == 0 && + i + 1 < argc) { + sconfig.tool_speculation.start_timeout_ms = std::atoi(argv[++i]); + if (sconfig.tool_speculation.start_timeout_ms <= 0) { + std::fprintf(stderr, + "[server] --tool-spec-start-timeout-ms must be positive\n"); + return 2; + } + } else if (std::strcmp(argv[i], "--tool-spec-finish-timeout-ms") == 0 && + i + 1 < argc) { + sconfig.tool_speculation.finish_timeout_ms = std::atoi(argv[++i]); + if (sconfig.tool_speculation.finish_timeout_ms <= 0) { + std::fprintf(stderr, + "[server] --tool-spec-finish-timeout-ms must be positive\n"); + return 2; + } } else if (std::strcmp(argv[i], "--chat-template-file") == 0 && i + 1 < argc) { const char * path = argv[++i]; std::FILE * f = std::fopen(path, "rb"); @@ -570,6 +615,35 @@ int main(int argc, char ** argv) { return 2; } } + std::sort(sconfig.tool_speculation.allowed_tools.begin(), + sconfig.tool_speculation.allowed_tools.end()); + sconfig.tool_speculation.allowed_tools.erase( + std::unique(sconfig.tool_speculation.allowed_tools.begin(), + sconfig.tool_speculation.allowed_tools.end()), + sconfig.tool_speculation.allowed_tools.end()); + const bool tool_speculation_requested = + !sconfig.tool_speculation.endpoint.empty() || + !sconfig.tool_speculation.api_key.empty() || + !sconfig.tool_speculation.allowed_tools.empty(); + if (tool_speculation_requested && !sconfig.tool_speculation.enabled()) { + std::fprintf(stderr, + "[server] tool speculation requires --tool-spec-endpoint and at " + "least one --tool-spec-allow\n"); + return 2; + } +#ifndef DFLASH_HAS_CURL + if (sconfig.tool_speculation.enabled()) { + std::fprintf(stderr, + "[server] tool speculation requires a build with libcurl\n"); + return 2; + } +#endif + if (sconfig.tool_speculation.enabled() && + !sconfig.pflash_upstream_base.empty()) { + std::fprintf(stderr, + "[server] tool speculation currently requires local generation\n"); + return 2; + } if (fast_rollback_forced_off) { bargs.fast_rollback = false; target_split_fast_rollback_cli = false; @@ -1057,6 +1131,13 @@ int main(int argc, char ** argv) { std::fprintf(stderr, "[server] │ prefix_cache = %d slots\n", sconfig.prefix_cache_cap); std::fprintf(stderr, "[server] │ prefill_cache = %d slots\n", sconfig.prefill_cache_cap); std::fprintf(stderr, "[server] │ cors = %s\n", sconfig.enable_cors ? "ON" : "off"); + std::fprintf(stderr, "[server] │ tool_speculation= %s\n", + sconfig.tool_speculation.enabled() ? "ON" : "off"); + if (sconfig.tool_speculation.enabled()) { + std::fprintf(stderr, + "[server] │ tool_spec_service= external, predictor-first, tools=%zu\n", + sconfig.tool_speculation.allowed_tools.size()); + } std::fprintf(stderr, "[server] │ cache_type_k = %s\n", #ifdef GGML_USE_HIP cache_type_k.empty() ? "q4_0 (default, HIP)" : cache_type_k.c_str()); diff --git a/server/src/server/tool_speculation.cpp b/server/src/server/tool_speculation.cpp new file mode 100644 index 000000000..8f5421865 --- /dev/null +++ b/server/src/server/tool_speculation.cpp @@ -0,0 +1,453 @@ +#include "tool_speculation.h" + +#include +#include +#include +#include +#include + +namespace dflash::common { +namespace { + +constexpr int kCancelTimeoutMs = 250; + +bool declared_tool(const json & tools, const std::string & name) { + if (!tools.is_array() || name.empty()) return false; + for (const auto & tool : tools) { + if (!tool.is_object()) continue; + if (tool.contains("name") && tool["name"].is_string() && + tool["name"].get() == name) { + return true; + } + if (tool.contains("function") && tool["function"].is_object() && + tool["function"].contains("name") && + tool["function"]["name"].is_string() && + tool["function"]["name"].get() == name) { + return true; + } + } + return false; +} + +json allowed_tool_definitions(const json & tools, + const ToolSpeculationConfig & config) { + json allowed = json::array(); + if (!tools.is_array()) return allowed; + for (const auto & tool : tools) { + if (!tool.is_object()) continue; + std::string name; + std::string description; + json parameters = json::object(); + if (tool.contains("name") && tool["name"].is_string()) { + name = tool["name"].get(); + if (tool.contains("description") && + tool["description"].is_string()) { + description = tool["description"].get(); + } + if (tool.contains("parameters") && + tool["parameters"].is_object()) { + parameters = tool["parameters"]; + } else if (tool.contains("input_schema") && + tool["input_schema"].is_object()) { + parameters = tool["input_schema"]; + } + } + if (tool.contains("function") && tool["function"].is_object() && + tool["function"].contains("name") && + tool["function"]["name"].is_string()) { + const json & function = tool["function"]; + name = function["name"].get(); + if (function.contains("description") && + function["description"].is_string()) { + description = function["description"].get(); + } + if (function.contains("parameters") && + function["parameters"].is_object()) { + parameters = function["parameters"]; + } + } + if (config.allows(name)) { + json normalized = { + {"name", name}, + {"parameters", std::move(parameters)}, + }; + if (!description.empty()) { + normalized["description"] = std::move(description); + } + allowed.push_back(std::move(normalized)); + } + } + return allowed; +} + +json normalized_tool_choice(const json & choice) { + if (choice.is_null() || choice.is_string()) return choice; + if (!choice.is_object()) return nullptr; + if (choice.contains("name") && choice["name"].is_string()) { + return {{"name", choice["name"]}}; + } + if (choice.contains("function") && choice["function"].is_object() && + choice["function"].contains("name") && + choice["function"]["name"].is_string()) { + return {{"name", choice["function"]["name"]}}; + } + if (choice.contains("type") && choice["type"].is_string()) { + const std::string type = choice["type"].get(); + if (type == "any") return "required"; + if (type == "auto" || type == "none" || type == "required") { + return type; + } + } + return nullptr; +} + +std::string transport_failure(const std::string & prefix, + const std::string & detail) { + return detail.empty() ? prefix : prefix + ": " + detail; +} + +} // namespace + +bool ToolSpeculationConfig::allows(const std::string & name) const { + return std::find(allowed_tools.begin(), allowed_tools.end(), name) != + allowed_tools.end(); +} + +bool CanonicalToolInvocation::from_parts( + const std::string & name, + const json & arguments, + CanonicalToolInvocation & out, + std::string & error) { + if (name.empty()) { + error = "tool name must not be empty"; + return false; + } + if (!arguments.is_object()) { + error = "tool arguments must be a JSON object"; + return false; + } + out.name = name; + out.arguments = arguments; + // nlohmann::json objects are key ordered, making dump() a stable identity + // independent of the provider's input key order. + out.arguments_json = arguments.dump(); + error.clear(); + return true; +} + +bool CanonicalToolInvocation::from_tool_call( + const ToolCall & call, + CanonicalToolInvocation & out, + std::string & error) { + try { + const json arguments = call.arguments.empty() + ? json::object() + : json::parse(call.arguments); + return from_parts(call.name, arguments, out, error); + } catch (const std::exception & exception) { + error = std::string("authoritative arguments are invalid JSON: ") + + exception.what(); + return false; + } +} + +ToolSpeculationAttempt::ToolSpeculationAttempt( + const ToolSpeculationConfig & config, + std::string request_id, + ToolSpeculationTransport transport) + : config_(config), + request_id_(std::move(request_id)), + transport_(std::move(transport)) {} + +ToolSpeculationAttempt::~ToolSpeculationAttempt() { + if (started_ && !resolved_) { + json ignored; + cancel_service("attempt_destroyed", ignored); + } +} + +std::unique_ptr ToolSpeculationAttempt::begin( + const ToolSpeculationConfig & config, + const std::string & request_id, + const json & messages, + const json & tools, + const json & tool_choice, + ToolSpeculationTransport transport) { + if (!config.enabled() || !transport) return nullptr; + const json allowed = allowed_tool_definitions(tools, config); + if (allowed.empty()) return nullptr; + const json choice = normalized_tool_choice(tool_choice); + if (choice.is_string() && choice.get() == "none") { + return nullptr; + } + if (choice.is_object() && choice.contains("name") && + choice["name"].is_string() && + !declared_tool(allowed, choice["name"].get())) { + return nullptr; + } + + auto attempt = std::unique_ptr( + new ToolSpeculationAttempt(config, request_id, std::move(transport))); + attempt->start(messages, allowed, tool_choice); + return attempt; +} + +void ToolSpeculationAttempt::start( + const json & messages, + const json & tools, + const json & tool_choice) { + json request = { + {"protocol", kToolSpeculationProtocol}, + {"operation", "start"}, + {"request_id", request_id_}, + {"messages", messages}, + {"tools", tools}, + {"min_confidence", config_.min_confidence}, + }; + const json normalized_choice = normalized_tool_choice(tool_choice); + if (!normalized_choice.is_null()) { + request["tool_choice"] = normalized_choice; + } + + json response; + std::string transport_error; + const auto started_at = std::chrono::steady_clock::now(); + bool ok = false; + try { + ok = transport_(request, config_.start_timeout_ms, + response, transport_error); + } catch (const std::exception & exception) { + transport_error = exception.what(); + } catch (...) { + transport_error = "unknown transport exception"; + } + predictor_wall_ms_ = std::chrono::duration( + std::chrono::steady_clock::now() - started_at).count(); + if (!ok) { + start_error_ = transport_failure( + "speculation service unavailable", transport_error); + return; + } + if (!response.is_object()) { + start_error_ = "start response must be an object"; + return; + } + + ticket_ = response.contains("ticket") && response["ticket"].is_string() + ? response["ticket"].get() : std::string(); + if (!response.contains("call") || !response["call"].is_object()) { + start_error_ = "start response call must be an object"; + } else if (!response.contains("confidence") || + !response["confidence"].is_number()) { + start_error_ = "start response confidence must be a number"; + } else { + try { + const json & call = response["call"]; + const std::string name = + call.contains("name") && call["name"].is_string() + ? call["name"].get() : std::string(); + const json arguments = call.contains("arguments") + ? call["arguments"] : json(); + confidence_ = response["confidence"].get(); + std::string canonical_error; + if (ticket_.empty()) { + start_error_ = "start response ticket must not be empty"; + } else if (!std::isfinite(confidence_) || confidence_ < 0.0 || + confidence_ > 1.0) { + start_error_ = "prediction confidence must be between 0 and 1"; + } else if (confidence_ < config_.min_confidence) { + start_error_ = "prediction below minimum confidence"; + } else if (!config_.allows(name) || !declared_tool(tools, name)) { + start_error_ = + "predicted tool is not allowlisted for this request"; + } else if (!CanonicalToolInvocation::from_parts( + name, arguments, prediction_, canonical_error)) { + start_error_ = canonical_error; + } + } catch (const std::exception & exception) { + start_error_ = std::string("invalid start response: ") + + exception.what(); + } + } + + if (!start_error_.empty()) { + if (!ticket_.empty()) { + json ignored; + cancel_service("invalid_prediction", ignored); + } + return; + } + started_ = true; +} + +json ToolSpeculationAttempt::base_metadata() const { + json metadata = { + {"protocol", kToolSpeculationProtocol}, + {"prediction_source", "external"}, + {"predictor_wall_ms", predictor_wall_ms_}, + }; + if (!prediction_.name.empty()) { + metadata["predicted_call"] = { + {"name", prediction_.name}, + {"arguments", prediction_.arguments}, + }; + metadata["confidence"] = confidence_; + } + return metadata; +} + +void ToolSpeculationAttempt::cancel_service( + const std::string & reason, + json & metadata) { + if (ticket_.empty()) return; + const json request = { + {"protocol", kToolSpeculationProtocol}, + {"operation", "cancel"}, + {"request_id", request_id_}, + {"ticket", ticket_}, + {"reason", reason}, + }; + json response; + std::string error; + try { + if (!transport_(request, kCancelTimeoutMs, response, error)) { + metadata["cancel_error"] = transport_failure( + "cancel request failed", error); + } + } catch (const std::exception & exception) { + metadata["cancel_error"] = exception.what(); + } catch (...) { + metadata["cancel_error"] = "unknown transport exception"; + } + ticket_.clear(); + started_ = false; +} + +json ToolSpeculationAttempt::finish( + const std::vector & authoritative_calls) { + if (resolved_) { + json metadata = base_metadata(); + metadata["status"] = "failed"; + metadata["reason"] = "already_resolved"; + return metadata; + } + resolved_ = true; + json metadata = base_metadata(); + if (!started_) { + metadata["status"] = "deferred"; + metadata["reason"] = start_error_.empty() + ? "prediction_unavailable" : start_error_; + return metadata; + } + if (authoritative_calls.size() != 1) { + cancel_service("authoritative_call_count", metadata); + metadata["status"] = "miss"; + metadata["reason"] = "authoritative_call_count"; + return metadata; + } + + CanonicalToolInvocation authoritative; + std::string canonical_error; + if (!CanonicalToolInvocation::from_tool_call( + authoritative_calls[0], authoritative, canonical_error)) { + cancel_service("invalid_authoritative_call", metadata); + metadata["status"] = "miss"; + metadata["reason"] = "invalid_authoritative_call"; + return metadata; + } + if (!(authoritative == prediction_)) { + cancel_service("invocation_mismatch", metadata); + metadata["status"] = "miss"; + metadata["reason"] = "invocation_mismatch"; + return metadata; + } + + const json request = { + {"protocol", kToolSpeculationProtocol}, + {"operation", "commit"}, + {"request_id", request_id_}, + {"ticket", ticket_}, + }; + json response; + std::string error; + const auto wait_started = std::chrono::steady_clock::now(); + bool ok = false; + try { + ok = transport_(request, config_.finish_timeout_ms, response, error); + } catch (const std::exception & exception) { + error = exception.what(); + } catch (...) { + error = "unknown transport exception"; + } + const double wait_ms = std::chrono::duration( + std::chrono::steady_clock::now() - wait_started).count(); + ticket_.clear(); + started_ = false; + metadata["commit_wait_ms"] = wait_ms; + if (!ok) { + metadata["status"] = "failed"; + metadata["reason"] = "commit_request_failed"; + metadata["detail"] = error; + return metadata; + } + if (!response.is_object() || !response.contains("ok") || + !response["ok"].is_boolean() || !response["ok"].get() || + !response.contains("result")) { + metadata["status"] = "failed"; + metadata["reason"] = "invalid_commit_response"; + return metadata; + } + metadata["status"] = "hit"; + metadata["call_id"] = authoritative_calls[0].id; + metadata["result"] = response["result"]; + return metadata; +} + +json ToolSpeculationAttempt::cancel(const std::string & reason) { + if (resolved_) { + json metadata = base_metadata(); + metadata["status"] = "failed"; + metadata["reason"] = "already_resolved"; + return metadata; + } + resolved_ = true; + json metadata = base_metadata(); + cancel_service(reason, metadata); + metadata["status"] = "cancelled"; + metadata["reason"] = reason; + return metadata; +} + +std::string render_tool_speculation_sse( + ApiFormat api_format, + const std::string & request_id, + const std::string & model, + const json & metadata) { + switch (api_format) { + case ApiFormat::OPENAI_CHAT: + return "data: " + json({ + {"id", request_id}, + {"object", "chat.completion.chunk"}, + {"model", model}, + {"choices", json::array()}, + {"dflash_tool_speculation", metadata}, + }).dump() + "\n\n"; + case ApiFormat::ANTHROPIC: + return "event: dflash_tool_speculation\ndata: " + json({ + {"type", "dflash_tool_speculation"}, + {"dflash_tool_speculation", metadata}, + }).dump() + "\n\n"; + case ApiFormat::RESPONSES: + return "event: response.dflash_tool_speculation\ndata: " + json({ + {"type", "response.dflash_tool_speculation"}, + {"response_id", request_id}, + {"dflash_tool_speculation", metadata}, + }).dump() + "\n\n"; + default: + return "data: " + json({ + {"dflash_tool_speculation", metadata}, + }).dump() + "\n\n"; + } +} + +} // namespace dflash::common diff --git a/server/src/server/tool_speculation.h b/server/src/server/tool_speculation.h new file mode 100644 index 000000000..9fad526d3 --- /dev/null +++ b/server/src/server/tool_speculation.h @@ -0,0 +1,117 @@ +// Model-agnostic speculative tool execution. +// +// A trusted external service predicts and starts one allowlisted tool before +// model execution. The model remains authoritative: the private result is +// committed only when the emitted tool call matches exactly. + +#pragma once + +#include "api_types.h" +#include "tool_parser.h" + +#include + +#include +#include +#include +#include + +namespace dflash::common { + +using json = nlohmann::json; + +inline constexpr char kToolSpeculationProtocol[] = + "dflash.tool-speculation.v1"; + +struct ToolSpeculationConfig { + std::string endpoint; + std::string api_key; + std::vector allowed_tools; + double min_confidence = 0.75; + int start_timeout_ms = 2000; + int finish_timeout_ms = 60000; + + bool enabled() const { + return !endpoint.empty() && !allowed_tools.empty(); + } + bool allows(const std::string & name) const; +}; + +// The engine deliberately knows nothing about the predictor model, tool +// runtime, or hardware placement. A transport posts one protocol request to +// the configured service and returns its JSON response. +using ToolSpeculationTransport = std::function; + +struct CanonicalToolInvocation { + std::string name; + json arguments = json::object(); + std::string arguments_json; + + static bool from_parts(const std::string & name, + const json & arguments, + CanonicalToolInvocation & out, + std::string & error); + static bool from_tool_call(const ToolCall & call, + CanonicalToolInvocation & out, + std::string & error); + + bool operator==(const CanonicalToolInvocation & other) const { + return name == other.name && arguments_json == other.arguments_json; + } +}; + +class ToolSpeculationAttempt { +public: + ToolSpeculationAttempt(const ToolSpeculationAttempt &) = delete; + ToolSpeculationAttempt & operator=(const ToolSpeculationAttempt &) = delete; + ~ToolSpeculationAttempt(); + + // Returns null when the request has no allowlisted tools. Otherwise the + // service call is synchronous: a successful return means the tool has + // started before model compute begins. + static std::unique_ptr begin( + const ToolSpeculationConfig & config, + const std::string & request_id, + const json & messages, + const json & tools, + const json & tool_choice, + ToolSpeculationTransport transport); + + // Commit on one exact authoritative call; cancel on every other outcome. + // A speculative result is never included in metadata on a miss. + json finish(const std::vector & authoritative_calls); + json cancel(const std::string & reason); + +private: + ToolSpeculationAttempt(const ToolSpeculationConfig & config, + std::string request_id, + ToolSpeculationTransport transport); + + void start(const json & messages, + const json & tools, + const json & tool_choice); + void cancel_service(const std::string & reason, json & metadata); + json base_metadata() const; + + ToolSpeculationConfig config_; + std::string request_id_; + ToolSpeculationTransport transport_; + CanonicalToolInvocation prediction_; + std::string ticket_; + std::string start_error_; + double confidence_ = 0.0; + double predictor_wall_ms_ = 0.0; + bool started_ = false; + bool resolved_ = false; +}; + +std::string render_tool_speculation_sse(ApiFormat api_format, + const std::string & request_id, + const std::string & model, + const json & metadata); + +} // namespace dflash::common diff --git a/server/test/test_server_unit.cpp b/server/test/test_server_unit.cpp index b2d614fa8..a4d228abe 100644 --- a/server/test/test_server_unit.cpp +++ b/server/test/test_server_unit.cpp @@ -4694,6 +4694,22 @@ TEST_CASE(ServerUnitFixture, test_props_runtime_shape) { TEST_ASSERT(body["runtime"]["draft_device"].is_null()); } +TEST_CASE(ServerUnitFixture, test_props_tool_speculation_is_model_agnostic) { + ServerConfig cfg; + cfg.tool_speculation.endpoint = "http://127.0.0.1:19090/v1/speculate"; + cfg.tool_speculation.allowed_tools = {"get_weather"}; + Tokenizer tokenizer; + PrefixCache prefix_cache(0, tokenizer); + ToolMemory tool_memory; + + const json feature = build_props_body( + cfg, prefix_cache, tool_memory)["tool_speculation"]; + TEST_ASSERT(feature["enabled"].get()); + TEST_ASSERT(feature["protocol"] == kToolSpeculationProtocol); + TEST_ASSERT(feature["schedule"] == "before_model"); + TEST_ASSERT(feature["provider"] == "external"); +} + // ═══════════════════════════════════════════════════════════════════════ // usage.timings — per-request prefill / decode wall-clock breakdown // surfaced under usage.timings (spec §6.3). Tests cover all three diff --git a/server/test/test_tool_speculation.cpp b/server/test/test_tool_speculation.cpp new file mode 100644 index 000000000..ce0a20dbd --- /dev/null +++ b/server/test/test_tool_speculation.cpp @@ -0,0 +1,209 @@ +#include "CppUnitTestFramework.hpp" + +#include "server/tool_speculation.h" + +#include +#include + +using namespace dflash::common; + +namespace { + +struct ToolSpeculationFixture {}; + +json function_tool(const std::string & name) { + return { + {"type", "function"}, + {"function", { + {"name", name}, + {"description", "test tool"}, + {"parameters", { + {"type", "object"}, + {"properties", { + {"city", {{"type", "string"}}}, + {"unit", {{"type", "string"}}}, + }}, + }}, + }}, + }; +} + +ToolSpeculationConfig config() { + ToolSpeculationConfig value; + value.endpoint = "http://speculator.test/v1"; + value.allowed_tools = {"weather"}; + value.min_confidence = 0.75; + return value; +} + +struct FakeService { + json start_response = { + {"ticket", "ticket-1"}, + {"call", { + {"name", "weather"}, + {"arguments", {{"city", "Rome"}, {"unit", "celsius"}}}, + }}, + {"confidence", 0.9}, + }; + json commit_response = { + {"ok", true}, + {"result", {{"temperature", 24}}}, + }; + std::string failing_operation; + std::vector requests; + + ToolSpeculationTransport transport() { + return [this](const json & request, int, json & response, + std::string & error) { + requests.push_back(request); + const std::string operation = request.value("operation", ""); + if (operation == failing_operation) { + error = "injected failure"; + return false; + } + if (operation == "start") response = start_response; + else if (operation == "commit") response = commit_response; + else if (operation == "cancel") response = {{"ok", true}}; + else { + error = "unexpected operation"; + return false; + } + error.clear(); + return true; + }; + } +}; + +std::unique_ptr begin(FakeService & service) { + return ToolSpeculationAttempt::begin( + config(), "request-1", + json::array({{{"role", "user"}, {"content", "Weather in Rome"}}}), + json::array({function_tool("weather"), function_tool("delete_file")}), + "auto", service.transport()); +} + +ToolCall weather_call( + const std::string & arguments = + R"({"unit":"celsius","city":"Rome"})") { + return {"call-1", "weather", arguments}; +} + +} // namespace + +TEST_CASE(ToolSpeculationFixture, provider_contract_is_model_agnostic) { + FakeService service; + auto attempt = begin(service); + CHECK_NOT_NULL(attempt.get()); + CHECK_EQUAL(size_t{1}, service.requests.size()); + + const json & request = service.requests.front(); + CHECK(request["protocol"] == kToolSpeculationProtocol); + CHECK(request["operation"] == "start"); + CHECK(!request.contains("model")); + CHECK(!request.contains("token_ids")); + CHECK_EQUAL(size_t{1}, request["tools"].size()); + CHECK(request["tools"][0]["name"] == "weather"); + CHECK(!request["tools"][0].contains("function")); + + const json metadata = attempt->finish({weather_call()}); + CHECK(metadata["status"] == "hit"); + CHECK(metadata["result"]["temperature"] == 24); + CHECK(metadata["prediction_source"] == "external"); + CHECK_EQUAL(size_t{2}, service.requests.size()); + CHECK(service.requests.back()["operation"] == "commit"); +} + +TEST_CASE(ToolSpeculationFixture, mismatch_cancels_and_hides_private_result) { + FakeService service; + auto attempt = begin(service); + const ToolCall actual{ + "call-1", "weather", R"({"city":"Milan","unit":"celsius"})"}; + + const json metadata = attempt->finish({actual}); + CHECK(metadata["status"] == "miss"); + CHECK(metadata["reason"] == "invocation_mismatch"); + CHECK(!metadata.contains("result")); + CHECK_EQUAL(size_t{2}, service.requests.size()); + CHECK(service.requests.back()["operation"] == "cancel"); +} + +TEST_CASE(ToolSpeculationFixture, undeclared_prediction_is_cancelled) { + FakeService service; + service.start_response["call"]["name"] = "delete_file"; + auto attempt = begin(service); + + CHECK_EQUAL(size_t{2}, service.requests.size()); + CHECK(service.requests.back()["operation"] == "cancel"); + const json metadata = attempt->finish({weather_call()}); + CHECK(metadata["status"] == "deferred"); + CHECK(metadata["reason"].get().find("allowlisted") != + std::string::npos); + CHECK(!metadata.contains("result")); +} + +TEST_CASE(ToolSpeculationFixture, low_confidence_prediction_is_cancelled) { + FakeService service; + service.start_response["confidence"] = 0.74; + auto attempt = begin(service); + + CHECK_EQUAL(size_t{2}, service.requests.size()); + CHECK(service.requests.back()["operation"] == "cancel"); + const json metadata = attempt->finish({weather_call()}); + CHECK(metadata["status"] == "deferred"); + CHECK(metadata["reason"] == "prediction below minimum confidence"); +} + +TEST_CASE(ToolSpeculationFixture, provider_failure_does_not_touch_generation) { + FakeService service; + service.failing_operation = "start"; + auto attempt = begin(service); + + const json metadata = attempt->finish({weather_call()}); + CHECK(metadata["status"] == "deferred"); + CHECK(metadata["reason"].get().find("unavailable") != + std::string::npos); + CHECK(!metadata.contains("result")); +} + +TEST_CASE(ToolSpeculationFixture, requests_without_allowlisted_tools_are_skipped) { + FakeService service; + auto attempt = ToolSpeculationAttempt::begin( + config(), "request-1", json::array(), + json::array({function_tool("delete_file")}), "auto", + service.transport()); + + CHECK_NULL(attempt.get()); + CHECK(service.requests.empty()); + + attempt = ToolSpeculationAttempt::begin( + config(), "request-2", json::array(), + json::array({function_tool("weather")}), "none", + service.transport()); + CHECK_NULL(attempt.get()); + CHECK(service.requests.empty()); +} + +TEST_CASE(ToolSpeculationFixture, multiple_authoritative_calls_are_a_miss) { + FakeService service; + auto attempt = begin(service); + const json metadata = attempt->finish({weather_call(), weather_call()}); + + CHECK(metadata["status"] == "miss"); + CHECK(metadata["reason"] == "authoritative_call_count"); + CHECK(!metadata.contains("result")); + CHECK(service.requests.back()["operation"] == "cancel"); +} + +TEST_CASE(ToolSpeculationFixture, streaming_metadata_preserves_api_shape) { + const json metadata = {{"status", "hit"}}; + const std::string openai = render_tool_speculation_sse( + ApiFormat::OPENAI_CHAT, "request-1", "target", metadata); + const std::string anthropic = render_tool_speculation_sse( + ApiFormat::ANTHROPIC, "request-1", "target", metadata); + const std::string responses = render_tool_speculation_sse( + ApiFormat::RESPONSES, "request-1", "target", metadata); + + CHECK(openai.find("chat.completion.chunk") != std::string::npos); + CHECK(anthropic.find("event: dflash_tool_speculation") == 0); + CHECK(responses.find("event: response.dflash_tool_speculation") == 0); +} From c09e64c2caec2d187214ef2c1507d4554ff3b3f8 Mon Sep 17 00:00:00 2001 From: mrciffa <49000955+davide221@users.noreply.github.com> Date: Mon, 17 Aug 2026 22:00:25 +0200 Subject: [PATCH 04/11] feat(server): predict and prelaunch tool workflows --- optimizations/ooo_spec_lucebox5_cpu/README.md | 163 +- .../benchmark_cpu_tool_speculation.py | 280 ++- .../benchmark_trace_compiled_workflows.py | 1639 +++++++++++++++++ .../bfcl_replay_tool_executor.py | 103 ++ ...sh_server_native_tool_predictor_wrapper.sh | 61 + ...engine-qwen-production-6pairs-compact.json | 872 +++++++++ .../trace-compiled-training-traces.json | 46 + .../run_native_cpu_server_lucebox5.sh | 31 +- .../test_benchmark_cpu_tool_speculation.py | 42 + ...test_benchmark_trace_compiled_workflows.py | 400 ++++ .../test_trace_compiled_tool_executor.py | 75 + .../trace_compiled_tool_executor.py | 177 ++ server/CMakeLists.txt | 13 + server/src/common/backend_ipc.cpp | 5 + server/src/common/backend_ipc.h | 1 + .../src/common/qwen3_tool_predictor_ipc.cpp | 119 ++ server/src/common/qwen3_tool_predictor_ipc.h | 57 + .../qwen3_tool_predictor_ipc_daemon.cpp | 120 ++ server/src/ipc/backend_ipc_main.cpp | 8 + server/src/qwen3/qwen3_loader.cpp | 16 +- server/src/server/chat_template.cpp | 28 +- server/src/server/chat_template.h | 9 +- server/src/server/http_server.cpp | 533 +++++- server/src/server/http_server.h | 25 + .../server/native_semantic_tool_predictor.cpp | 85 + .../server/native_semantic_tool_predictor.h | 40 + server/src/server/semantic_tool_hint.cpp | 500 +++++ server/src/server/semantic_tool_hint.h | 105 ++ server/src/server/server_main.cpp | 155 ++ server/src/server/tool_speculation.cpp | 34 +- server/src/server/tool_speculation.h | 9 + .../test/smoke_qwen3_tool_predictor_ipc.cpp | 191 ++ server/test/test_moe_hybrid_storage.cpp | 6 +- server/test/test_semantic_tool_hint.cpp | 248 +++ server/test/test_server_unit.cpp | 85 + server/test/test_tool_speculation.cpp | 17 + 36 files changed, 6190 insertions(+), 108 deletions(-) create mode 100644 optimizations/ooo_spec_lucebox5_cpu/benchmark_trace_compiled_workflows.py create mode 100755 optimizations/ooo_spec_lucebox5_cpu/bfcl_replay_tool_executor.py create mode 100755 optimizations/ooo_spec_lucebox5_cpu/dflash_server_native_tool_predictor_wrapper.sh create mode 100644 optimizations/ooo_spec_lucebox5_cpu/results/trace-compiled-engine-qwen-production-6pairs-compact.json create mode 100644 optimizations/ooo_spec_lucebox5_cpu/results/trace-compiled-training-traces.json create mode 100644 optimizations/ooo_spec_lucebox5_cpu/test_benchmark_trace_compiled_workflows.py create mode 100644 optimizations/ooo_spec_lucebox5_cpu/test_trace_compiled_tool_executor.py create mode 100755 optimizations/ooo_spec_lucebox5_cpu/trace_compiled_tool_executor.py create mode 100644 server/src/common/qwen3_tool_predictor_ipc.cpp create mode 100644 server/src/common/qwen3_tool_predictor_ipc.h create mode 100644 server/src/common/qwen3_tool_predictor_ipc_daemon.cpp create mode 100644 server/src/server/native_semantic_tool_predictor.cpp create mode 100644 server/src/server/native_semantic_tool_predictor.h create mode 100644 server/src/server/semantic_tool_hint.cpp create mode 100644 server/src/server/semantic_tool_hint.h create mode 100644 server/test/smoke_qwen3_tool_predictor_ipc.cpp create mode 100644 server/test/test_semantic_tool_hint.cpp diff --git a/optimizations/ooo_spec_lucebox5_cpu/README.md b/optimizations/ooo_spec_lucebox5_cpu/README.md index 76ecd5816..1f18a0e8e 100644 --- a/optimizations/ooo_spec_lucebox5_cpu/README.md +++ b/optimizations/ooo_spec_lucebox5_cpu/README.md @@ -1,94 +1,133 @@ -# Lucebox5 disjoint-CPU tool speculation +# CPU-isolated tool speculation on Lucebox5 -This experiment runs a predicted read-only/idempotent tool concurrently with -DeepSeek generation. The model stays on the R9700 + Strix GPU path with DS4 -enabled, while the tool child process is pinned to CPU cores that the model -process cannot use. The result is released only when the generated canonical -tool call exactly matches the prediction. +The engine asks a small predictor for one concrete tool call before the target +model runs. On Lucebox5, Qwen3-0.6B Q8_0 predicts on Strix, then exits its GPU +compute window; the predicted read-only tool runs on reserved CPU cores while +DeepSeek-V4-0731 decodes with DS4/DSpark on R9700 + Strix. The result stays +private unless DeepSeek emits the exact same canonical function and arguments. -## Measured result +This path does not inject tokens, replace DSpark, or retry speculative decoding +with autoregressive decoding. A wrong prediction is discarded and the caller +executes the target model's authoritative call normally. -The native engine benchmark on Lucebox5 passed its production gate: +## Production result + +The 2026-08-17 paired run compiled a recurring, side-effect-free five-step +trace into one typed workflow tool. Independent branches ran concurrently on +the isolated CPU lane. The six measured tasks covered 10, 15, and 20 leaf calls +twice each, with randomized arm order and one warmup task. | Metric | Result | | --- | ---: | -| Sequential model + tool, p50 | 5511.64 ms | -| Speculative full task, p50 | 2910.40 ms | -| Exact-hit speedup | **1.8938x** | -| Bootstrap 95% CI | 1.8859x - 1.8964x | -| Task-latency reduction | 47.20% | -| Model-compute slowdown | 0.46% | -| DS4 median acceptance rate | 0.4167 | -| Correct predictions | 20 / 20 | - -This workload's zero-interference ceiling is 1.9011x because model and tool -latencies are not perfectly equal. The implementation reaches 99.6% of that -ceiling; claiming 2x for this measured workload would be inaccurate. - -The control arm runs the model and then the identical CPU-pinned sparse tool. -The speculative arm starts that tool before generation. Arm order is randomized -inside each of 20 warm pairs, with two warmups. A 20,000-resample paired -bootstrap supplies the confidence interval. Model outputs, canonical calls, -and tool checksums are identical across arms. A deliberately wrong prediction -produces an `invocation_mismatch` and exposes no private tool result. - -Raw artifacts: - -- `results/lucebox5-cpu-native-20pairs.json` (`sha256:4a7f224f44cc7f2385e476f8227c6d51d4f4b90052e6a7b090bafcfc1f3b68a7`) -- `results/lucebox5-cpu-lane-qualification.json` (`sha256:6fc3f6d95c12db817b687b8c9509517d24230a508f520989483c1c6ed96c67df`) -- `profiles/lucebox5-cpu-lane-qualified.json` (`sha256:5cdf2550bb5a95c835daddac9f8d0126f470a5ac359e14008207e82c5dafb718`) - -## Isolation and compatibility - -Lucebox5 reserves logical CPUs `14-15,30-31` for the two-thread sparse tool and -launches the model with `0-13,16-29`. Startup fails closed if either mask -overlaps, if a listed CPU does not exist, or if an in-process executor is used. -Each child is pinned and its mask is read back before the request payload is -sent, so tool work cannot begin on model CPUs. - -The engine feature is Linux- and backend-neutral: a single-GPU system can use -the same child-process path when it has CPU cores to reserve. It does not -replace or disable autoregressive decoding or DS4 token speculation. The -benchmark requires a positive DS4 acceptance rate on every measured request; -the median was 0.4167. - -This improves the full latency of a correctly predicted tool-using request; it -does not double token generation throughput. On a miss, generation remains -authoritative, the speculative result stays private, and the caller executes -the generated tool call normally. +| Normal stage-batched workflow, p50 | 81.030 s | +| Trace-compiled + speculative workflow, p50 | **14.597 s** | +| End-to-end speedup, paired p50 | **5.5961x** | +| End-to-end bootstrap 95% CI | **5.4577x–5.6599x** | +| Trace compilation alone | **3.2806x** | +| Early launch on top of compilation | **1.6954x** | +| Early-launch bootstrap 95% CI | **1.6760x–1.7210x** | +| Exposed tool wait, compiled / speculative p50 | 10.143 s / **0.027 ms** | +| Qwen prediction latency, p50 | 203.5 ms | +| Target model-compute slowdown, p50 / p95 | -0.458% / -0.332% | +| Target decode slowdown, p50 / p95 | -0.101% / 0.219% | +| Exact predictor-to-target hits | 6 / 6 | + +All 20 production gates passed: identical leaf calls, tool-result hashes, +macro calls, and final outputs; positive DS4 acceptance on every call turn; +correct CPU isolation; and no measurable target slowdown. The 5.60x result +combines two independent gains: four fewer model/tool synchronization barriers +from trace compilation, plus the 1.70x gained by starting the compiled graph +before target authorization completes. + +Artifact: + +- `results/trace-compiled-engine-qwen-production-6pairs-compact.json` + (`sha256:0807cca1d22453728b069a0150800fcfa9a513db6a9f25815663fc03b99285b9`) + +The compact training fixture below reproduces the artifact's compiled pattern +fingerprint (`06d95882…0645`); the artifact retains the original full-report +hash for provenance. + +## Safety and portability + +- Only explicitly allowlisted, read-only/idempotent tools are eligible. +- The external result is committed only on an exact canonical call match. +- The executor is launched directly without a shell and has a hard timeout. +- Lucebox5 reserves CPUs `14-15,30-31`; the model uses `0-13,16-29`. +- Startup fails closed if CPU masks overlap or the measured lane profile fails. +- `before-model` is the native predictor default, so shared-GPU prediction + cannot reduce target prefill/decode throughput. +- The same API works on a single GPU: run the predictor before the target and + overlap only the CPU tool. An HTTP predictor can use the same verification + and executor path on other model families. + +The speedup applies to tool-using request latency, not token throughput. Its +real-world value depends on exact predictor hit rate and on how much tool work +can overlap target generation. ## Reproduce -Build the deterministic sparse-compute executor: +Build the deterministic sparse tool used by the single-call qualification: ```bash JSON_INCLUDE=/path/to/server/deps/json/include \ ./build_cpu_sparse_executor.sh ./cpu_sparse_tool_executor ``` -Launch the qualified native server on an otherwise idle Lucebox5: +Launch the qualified single-call configuration on an otherwise idle Lucebox5: ```bash ./run_native_cpu_server_lucebox5.sh ``` -Then run the measured gate: +The launcher defaults to Qwen3-0.6B Q8_0 on predictor GPU 1. Override placement +with `PREDICTOR_MODEL`, `PREDICTOR_GPU`, `PREDICTOR_MAX_CTX`, and +`PREDICTOR_MAX_TOKENS`. The adjacent `candidate-build` symlink in the wrapper +selects a build even though the qualified launcher clears ambient variables. + +Run the single-call paired gate: ```bash -python3 benchmark_cpu_tool_speculation.py native \ +python3 benchmark_cpu_tool_speculation.py native-qwen \ --url http://127.0.0.1:18145/v1/chat/completions \ --binary ./cpu_sparse_tool_executor \ --tool-cpus 14-15,30-31 \ --iterations 172452 \ --max-tokens 32 \ --pairs 20 \ - --warmups 2 \ + --warmups 5 \ --bootstrap-resamples 20000 \ - --min-speedup 1.8 \ - --min-speedup-ci-low 1.7 \ + --min-speedup 1.6 \ + --min-speedup-ci-low 1.5 \ + --min-speedup-p05 1.5 \ + --min-prediction-hit-rate 1.0 \ --max-model-slowdown-percent 5 \ - --output results/lucebox5-cpu-native-20pairs.json + --output results/qwen-auto-production-20pairs.json +``` + +For the 10–20-call workflow gate, launch with the trace executor and macro +allowlist: + +```bash +TOOL_SPEC_EXECUTOR=./trace_compiled_tool_executor.py \ +TOOL_SPEC_ALLOW=resolve_customer,list_open_orders,get_order_details,calculate_shipping,prepare_customer_summary,execute_customer_workflows \ + ./run_native_cpu_server_lucebox5.sh +``` + +Then run: + +```bash +python3 benchmark_trace_compiled_workflows.py \ + --binary ./bfcl_replay_tool_executor.py \ + --training-report results/trace-compiled-training-traces.json \ + --pairs 6 \ + --warmup-tasks 1 \ + --min-branches 2 \ + --max-branches 4 \ + --seed 814 \ + --bootstrap-resamples 20000 \ + --output results/trace-compiled-engine-qwen-production-6pairs-compact.json ``` -The harness exits nonzero if correctness, isolation, DS4 activity, slowdown, -or either speed threshold fails. +The harness exits nonzero on any correctness, isolation, DS4-activity, +slowdown, hit-rate, or speed threshold failure. diff --git a/optimizations/ooo_spec_lucebox5_cpu/benchmark_cpu_tool_speculation.py b/optimizations/ooo_spec_lucebox5_cpu/benchmark_cpu_tool_speculation.py index 7aae61cc2..b2238c97d 100755 --- a/optimizations/ooo_spec_lucebox5_cpu/benchmark_cpu_tool_speculation.py +++ b/optimizations/ooo_spec_lucebox5_cpu/benchmark_cpu_tool_speculation.py @@ -32,6 +32,7 @@ TOOL_NAME = "benchmark_cpu_sparse" PROTOCOL = "dflash.tool-speculation.v1" +NATIVE_PREDICTION_SOURCE = "native-qwen3" SPARSE_ROWS = 4096 SPARSE_NONZEROS_PER_ROW = 16 SPARSE_THREADS = 2 @@ -102,8 +103,11 @@ def request_body( max_tokens: int, *, prediction: dict[str, int] | None, + automatic_prediction: bool = False, + tool_choice: str | None = None, ) -> dict[str, Any]: compact = json.dumps(arguments, separators=(",", ":")) + prompt = f"Return only this JSON object and nothing else: {compact}" body: dict[str, Any] = { "model": "dflash", "stream": False, @@ -112,11 +116,14 @@ def request_body( "messages": [ { "role": "user", - "content": f"Return only this JSON object and nothing else: {compact}", + "content": prompt, } ], "tools": [tool_definition()], + "automatic_tool_speculation": automatic_prediction, } + if tool_choice is not None: + body["tool_choice"] = tool_choice if prediction is not None: body["tool_speculation"] = { "call": {"name": TOOL_NAME, "arguments": prediction}, @@ -168,6 +175,14 @@ def normalize_tool_call(result: dict[str, Any]) -> dict[str, Any] | None: parsed.get("arguments", parsed.get("function_args")), ), ) + if ( + arguments is None + and isinstance(parsed.get("parameter"), str) + and "parameter_value" in parsed + ): + # DeepSeek may serialize a one-argument native call as a compact + # name/value envelope. It is semantically the same function call. + arguments = {parsed["parameter"]: parsed["parameter_value"]} if arguments is None and isinstance(name, str): arguments = { key: value @@ -246,10 +261,18 @@ def post_model( timeout: float, *, prediction: dict[str, int] | None = None, + automatic_prediction: bool = False, + tool_choice: str | None = None, ) -> dict[str, Any]: result, wall_ms = post_json( url, - request_body(arguments, max_tokens, prediction=prediction), + request_body( + arguments, + max_tokens, + prediction=prediction, + automatic_prediction=automatic_prediction, + tool_choice=tool_choice, + ), timeout, ) return observation(result, wall_ms) @@ -772,6 +795,94 @@ def native_speculative( } +def native_qwen_control( + args: argparse.Namespace, + arguments: dict[str, int], + label: str, +) -> dict[str, Any]: + started = time.perf_counter() + model = post_model( + args.url, + arguments, + args.max_tokens, + args.timeout, + automatic_prediction=False, + tool_choice="required", + ) + tool = run_executor( + args.binary, + arguments, + args.tool_cpus, + args.timeout, + f"{label}-tool", + ) + validate_model_call(model, arguments) + result = tool["result"] + return { + "mode": "control", + "task_ms": (time.perf_counter() - started) * 1000.0, + **model, + "tool_wall_ms": float(tool["wall_ms"]), + "tool_compute_ms": float(result["compute_ms"]), + "tool_checksum": result["checksum"], + "tool_cpu_affinity": result["cpu_affinity"], + "prediction_hit": False, + "predictor_wall_ms": 0.0, + } + + +def native_qwen_speculative( + args: argparse.Namespace, + arguments: dict[str, int], + label: str, +) -> dict[str, Any]: + started = time.perf_counter() + model = post_model( + args.url, + arguments, + args.max_tokens, + args.timeout, + automatic_prediction=True, + tool_choice="required", + ) + validate_model_call(model, arguments) + metadata = model["speculation"] if isinstance( + model["speculation"], dict + ) else {} + prediction_hit = metadata.get("status") == "hit" + if prediction_hit: + tool_result = metadata.get("result") + if not isinstance(tool_result, dict): + raise RuntimeError("automatic hit did not expose a tool result") + tool_wall_ms = float(metadata.get("executor_wall_ms", math.nan)) + else: + # This is the real miss path: discard the private speculative result, + # then execute the authoritative model call normally. + fallback = run_executor( + args.binary, + arguments, + args.tool_cpus, + args.timeout, + f"{label}-fallback", + ) + tool_result = fallback["result"] + tool_wall_ms = float(fallback["wall_ms"]) + return { + "mode": "qwen_speculative", + "task_ms": (time.perf_counter() - started) * 1000.0, + **model, + "tool_wall_ms": tool_wall_ms, + "tool_compute_ms": float(tool_result["compute_ms"]), + "tool_checksum": tool_result["checksum"], + "tool_cpu_affinity": tool_result["cpu_affinity"], + "prediction_hit": prediction_hit, + "predictor_wall_ms": float(metadata.get("predictor_wall_ms", 0.0)), + "prediction_source": metadata.get("prediction_source"), + "prediction_status": metadata.get("status"), + "prediction_reason": metadata.get("reason"), + } + + def summarize_native( pairs: list[dict[str, Any]], resamples: int, seed: int ) -> dict[str, Any]: @@ -793,11 +904,28 @@ def summarize_native( speculative_tool = statistics.median( row["tool_compute_ms"] for row in speculative ) + paired_speedups = [ + float(pair["control"]["task_ms"]) + / float(pair["speculative"]["task_ms"]) + for pair in pairs + ] return { "pairs": len(pairs), "control_task_p50_ms": control_task, + "control_task_p95_ms": percentile( + (row["task_ms"] for row in controls), 0.95 + ), + "control_task_max_ms": max(row["task_ms"] for row in controls), "speculative_task_p50_ms": speculative_task, + "speculative_task_p95_ms": percentile( + (row["task_ms"] for row in speculative), 0.95 + ), + "speculative_task_max_ms": max( + row["task_ms"] for row in speculative + ), "exact_hit_speedup": control_task / speculative_task, + "paired_speedup_p05": percentile(paired_speedups, 0.05), + "paired_speedup_min": min(paired_speedups), "exact_hit_speedup_bootstrap_95ci": bootstrap_speedup_ci( pairs, resamples, seed ), @@ -1019,6 +1147,134 @@ def native(args: argparse.Namespace) -> None: raise SystemExit("native CPU tool-speculation production gate failed") +def native_qwen(args: argparse.Namespace) -> None: + props = get_json(props_url(args.url), args.timeout) + tool_props = props.get("tool_speculation") + if not isinstance(tool_props, dict) or not tool_props.get("enabled"): + raise SystemExit("server tool speculation is not enabled") + if not tool_props.get("automatic_prediction_enabled"): + raise SystemExit("server automatic Qwen prediction is not enabled") + if tool_props.get("execution_mode") != "child_process_cpu_affinity": + raise SystemExit("automatic benchmark requires the isolated CPU executor") + if tool_props.get("tool_cpu_affinity") != args.tool_cpus: + raise SystemExit("server tool CPU affinity differs from benchmark") + + arguments = expected_arguments( + args.rows, + args.nonzeros_per_row, + args.iterations, + args.threads, + args.tool_seed, + ) + for warmup in range(args.warmups): + native_qwen_control( + args, arguments, f"qwen-warm-{warmup}-control" + ) + native_qwen_speculative( + args, arguments, f"qwen-warm-{warmup}-speculative" + ) + + generator = random.Random(args.seed) + pairs: list[dict[str, Any]] = [] + for pair_index in range(args.pairs): + order = ["control", "speculative"] + generator.shuffle(order) + rows: dict[str, dict[str, Any]] = {} + for arm in order: + rows[arm] = ( + native_qwen_control( + args, arguments, f"qwen-{pair_index}-control" + ) + if arm == "control" + else native_qwen_speculative( + args, arguments, f"qwen-{pair_index}-speculative" + ) + ) + pairs.append({"pair_index": pair_index, "arm_order": order, **rows}) + print(json.dumps({ + "phase": "native-qwen", + "pair": pair_index + 1, + "control_ms": round(rows["control"]["task_ms"], 3), + "speculative_ms": round(rows["speculative"]["task_ms"], 3), + "speedup": round( + rows["control"]["task_ms"] / + rows["speculative"]["task_ms"], 3), + "prediction_status": rows["speculative"]["prediction_status"], + "predictor_ms": round( + rows["speculative"]["predictor_wall_ms"], 3), + }, sort_keys=True), flush=True) + + summary = summarize_native(pairs, args.bootstrap_resamples, args.seed) + speculative = [pair["speculative"] for pair in pairs] + hits = sum(row["prediction_hit"] for row in speculative) + summary.update({ + "qwen_prediction_hits": hits, + "qwen_prediction_hit_rate": hits / len(speculative), + "qwen_predictor_p50_ms": statistics.median( + row["predictor_wall_ms"] for row in speculative + ), + "qwen_predictor_p95_ms": percentile( + (row["predictor_wall_ms"] for row in speculative), 0.95 + ), + "qwen_prediction_source_valid": all( + row["prediction_source"] == NATIVE_PREDICTION_SOURCE + for row in speculative + ), + }) + checks = { + "all_model_outputs_identical": summary["all_model_outputs_identical"], + "all_calls_identical": summary["all_calls_identical"], + "all_tool_outputs_equivalent": summary["all_tool_outputs_equivalent"], + "ds4_active": summary["median_accept_rate"] > 0, + "prediction_source": summary["qwen_prediction_source_valid"], + "prediction_hit_rate": + summary["qwen_prediction_hit_rate"] >= args.min_prediction_hit_rate, + "speedup": summary["exact_hit_speedup"] >= args.min_speedup, + "speedup_ci_low": + summary["exact_hit_speedup_bootstrap_95ci"][0] >= + args.min_speedup_ci_low, + "speedup_p05": + summary["paired_speedup_p05"] >= args.min_speedup_p05, + "model_slowdown": summary["model_compute_slowdown_percent"] <= + args.max_model_slowdown_percent, + } + report = { + "phase": "native_qwen_engine", + "host": "lucebox5", + "config": report_config(args, arguments), + "methodology": { + "control": "DS4 generation, then the authoritative CPU-pinned tool", + "speculative": "Qwen3-0.6B predicts the call; the engine launches the private CPU-pinned tool before DS4 finishes", + "prediction_cost": "included in speculative request wall time", + "miss_cost": "included by running the authoritative tool after every miss", + "commit": "exact canonical function name and arguments", + "semantic_token_injection": False, + }, + "server_snapshot": {"tool_speculation": tool_props}, + "production_gate": { + "passed": all(checks.values()), + "checks": checks, + "thresholds": { + "min_speedup": args.min_speedup, + "min_speedup_ci_low": args.min_speedup_ci_low, + "min_speedup_p05": args.min_speedup_p05, + "min_prediction_hit_rate": args.min_prediction_hit_rate, + "max_model_slowdown_percent": + args.max_model_slowdown_percent, + }, + }, + "summary": summary, + "pairs": pairs, + } + write_report(args.output, report) + print(json.dumps({ + "production_gate": report["production_gate"], + "summary": summary, + }, indent=2, sort_keys=True), flush=True) + if not report["production_gate"]["passed"]: + raise SystemExit("automatic Qwen tool-speculation gate failed") + + def report_config( args: argparse.Namespace, arguments: dict[str, int] ) -> dict[str, Any]: @@ -1091,6 +1347,15 @@ def main() -> None: native_parser.add_argument("--min-speedup", type=float, default=1.80) native_parser.add_argument("--min-speedup-ci-low", type=float, default=1.70) + qwen_parser = subparsers.add_parser("native-qwen") + add_common_arguments(qwen_parser) + qwen_parser.add_argument("--iterations", type=int, required=True) + qwen_parser.add_argument("--bootstrap-resamples", type=int, default=20_000) + qwen_parser.add_argument("--min-speedup", type=float, default=1.60) + qwen_parser.add_argument("--min-speedup-ci-low", type=float, default=1.50) + qwen_parser.add_argument("--min-speedup-p05", type=float, default=1.25) + qwen_parser.add_argument("--min-prediction-hit-rate", type=float, default=0.90) + args = parser.parse_args() if not args.binary.is_file(): parser.error(f"executor binary does not exist: {args.binary}") @@ -1128,7 +1393,16 @@ def main() -> None: or args.min_speedup_ci_low > args.min_speedup ): parser.error("native benchmark settings are invalid") - native(args) + if args.phase == "native-qwen": + if ( + not 0.0 <= args.min_prediction_hit_rate <= 1.0 + or args.min_speedup_p05 <= 1.0 + or args.min_speedup_p05 > args.min_speedup + ): + parser.error("native Qwen benchmark settings are invalid") + native_qwen(args) + else: + native(args) if __name__ == "__main__": diff --git a/optimizations/ooo_spec_lucebox5_cpu/benchmark_trace_compiled_workflows.py b/optimizations/ooo_spec_lucebox5_cpu/benchmark_trace_compiled_workflows.py new file mode 100644 index 000000000..208bb7861 --- /dev/null +++ b/optimizations/ooo_spec_lucebox5_cpu/benchmark_trace_compiled_workflows.py @@ -0,0 +1,1639 @@ +#!/usr/bin/env python3 +"""Benchmark no-training trace-compiled tool workflows on Lucebox5. + +The benchmark compares three end-to-end agent paths over 10--20 real model +tool calls: + +* ``stage_batched``: DS4 authorizes one batch per dependency stage and every + call in that stage executes concurrently. This gives the control parallel + tools without speculating or compiling the complete workflow. +* ``compiled``: a recurring, side-effect-free trace is exposed as one generated + macro tool; DS4 authorizes it once and independent branches run in parallel. +* ``speculative``: the PR's Qwen predictor proposes the macro call and the + engine starts its compiled graph before DS4 finishes. The private result is + committed only after an exact name-and-arguments match. + +The workflow compiler is learned from prior successful traces. It performs no +model training and refuses literals, side effects, ambiguous dataflow, or +inconsistent control flow. Every arm uses the production DS4+DSpark endpoint, +includes all model turns and tools in wall time, and must produce the same +underlying calls, tool results, and exact final answer. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import random +import re +import statistics +import subprocess +import time +from concurrent.futures import Future, ThreadPoolExecutor +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from benchmark_cpu_tool_speculation import ( + NATIVE_PREDICTION_SOURCE, + get_json, + normalize_tool_call, + parse_cpu_list, + percentile, + post_json, + props_url, +) +from bfcl_replay_tool_executor import ( + PROTOCOL, + call_ref, + call_sha256, + canonical_call, +) + + +@dataclass(frozen=True) +class ArgumentBinding: + source: str + key: str + + +@dataclass(frozen=True) +class PatternStep: + tool: str + arguments: tuple[tuple[str, ArgumentBinding], ...] + + +@dataclass(frozen=True) +class CompiledPattern: + steps: tuple[PatternStep, ...] + root_fields: tuple[str, ...] + training_traces: int + + @property + def macro_name(self) -> str: + # Semantic names are materially more reliable for tool-call generation + # than opaque hashes. The registry remains keyed by the full pattern + # fingerprint, so this user-facing alias does not define identity. + subject = self.steps[0].tool.rsplit("_", 1)[-1] + return f"execute_{subject}_workflows" + + @property + def fingerprint(self) -> str: + payload = json.dumps( + [ + [ + step.tool, + [[name, binding.source, binding.key] for name, binding in step.arguments], + ] + for step in self.steps + ], + separators=(",", ":"), + ) + return hashlib.sha256(payload.encode()).hexdigest() + + def instantiate( + self, root: dict[str, str], previous_result: dict[str, Any] | None, index: int + ) -> dict[str, Any]: + step = self.steps[index] + arguments: dict[str, Any] = {} + for name, binding in step.arguments: + source: dict[str, Any] + if binding.source == "root": + source = root + elif binding.source == "previous_result" and previous_result is not None: + source = previous_result + else: + raise RuntimeError( + f"cannot resolve {binding.source}.{binding.key} for {step.tool}" + ) + if binding.key not in source: + raise RuntimeError( + f"missing {binding.source}.{binding.key} for {step.tool}" + ) + arguments[name] = source[binding.key] + return {"name": step.tool, "arguments": arguments} + + def simulate(self, root: dict[str, str]) -> list[dict[str, Any]]: + calls = [] + previous: dict[str, Any] | None = None + for index in range(len(self.steps)): + call = self.instantiate(root, previous, index) + calls.append(call) + previous = simulated_tool_result(call) + return calls + + def macro_tool( + self, max_items: int, workflow_ref: str | None = None + ) -> dict[str, Any]: + if workflow_ref is not None: + parameters = { + "type": "object", + "properties": { + "workflow_ref": { + "type": "string", + "enum": [workflow_ref], + "description": "Bound workflow instance for this request.", + } + }, + "required": ["workflow_ref"], + "additionalProperties": False, + } + else: + properties = {field: {"type": "string"} for field in self.root_fields} + parameters = { + "type": "object", + "properties": { + "customers": { + "type": "array", + "items": { + "type": "object", + "properties": properties, + "required": list(self.root_fields), + "additionalProperties": False, + }, + "minItems": 1, + "maxItems": max_items, + } + }, + "required": ["customers"], + "additionalProperties": False, + } + return { + "type": "function", + "function": { + "name": self.macro_name, + "description": ( + "Execute the validated five-step customer workflow independently " + "for every requested customer." + ), + "parameters": parameters, + }, + } + + +def simulated_tool_result(call: dict[str, Any]) -> dict[str, Any]: + return { + "call_ref": call_ref(call), + "call_sha256": call_sha256(call), + "tool_name": call["name"], + "side_effects": False, + } + + +def _infer_binding( + value: Any, + root: dict[str, str], + previous_result: dict[str, Any] | None, +) -> ArgumentBinding: + previous_matches = [] + if previous_result is not None: + previous_matches = [key for key, candidate in previous_result.items() if candidate == value] + root_matches = [key for key, candidate in root.items() if candidate == value] + if len(previous_matches) == 1: + return ArgumentBinding("previous_result", previous_matches[0]) + if len(root_matches) == 1: + return ArgumentBinding("root", root_matches[0]) + raise ValueError("argument is literal or has ambiguous trace dataflow") + + +def mine_pattern(traces: list[dict[str, Any]]) -> CompiledPattern: + if len(traces) < 2: + raise ValueError("at least two successful traces are required") + signatures = [] + root_fields: set[str] = set() + for trace in traces: + root = trace.get("root") + calls = trace.get("calls") + results = trace.get("results") + if not isinstance(root, dict) or not isinstance(calls, list) or not isinstance(results, list): + raise ValueError("trace must contain root, calls, and results") + if not calls or len(calls) != len(results): + raise ValueError("trace calls and results must be non-empty and aligned") + signature = [] + previous: dict[str, Any] | None = None + for call, result in zip(calls, results, strict=True): + if not isinstance(call, dict) or not isinstance(call.get("arguments"), dict): + raise ValueError("trace call is malformed") + if not isinstance(result, dict) or result.get("side_effects") is not False: + raise ValueError("only explicitly side-effect-free traces can be compiled") + if result.get("call_sha256") != call_sha256(call): + raise ValueError("trace result does not match its call") + bindings = [] + for name, value in sorted(call["arguments"].items()): + binding = _infer_binding(value, root, previous) + if binding.source == "root": + root_fields.add(binding.key) + bindings.append((name, binding)) + signature.append(PatternStep(str(call.get("name", "")), tuple(bindings))) + previous = result + signatures.append(tuple(signature)) + if any(signature != signatures[0] for signature in signatures[1:]): + raise ValueError("training traces do not share one control/data-flow pattern") + if not root_fields: + raise ValueError("compiled workflow exposes no request-bound arguments") + return CompiledPattern( + steps=signatures[0], + root_fields=tuple(sorted(root_fields)), + training_traces=len(traces), + ) + + +def load_training_traces(path: Path, required_steps: int) -> list[dict[str, Any]]: + report = json.loads(path.read_text(encoding="utf-8")) + compact_traces = report.get("traces") if isinstance(report, dict) else None + if isinstance(compact_traces, list): + traces = [ + trace + for trace in compact_traces + if isinstance(trace, dict) + and isinstance(trace.get("calls"), list) + and len(trace["calls"]) == required_steps + ] + if len(traces) < 2: + raise ValueError( + "training trace file contains fewer than two complete traces" + ) + return traces + pairs = report.get("pairs") if isinstance(report, dict) else None + if not isinstance(pairs, list): + raise ValueError("training report has no pairs") + traces = [] + for pair in pairs: + task = pair.get("task") if isinstance(pair, dict) else None + control = pair.get("control") if isinstance(pair, dict) else None + steps = control.get("steps") if isinstance(control, dict) else None + if ( + not isinstance(task, dict) + or not isinstance(steps, list) + or len(steps) != required_steps + or not control.get("all_calls_correct") + ): + continue + traces.append( + { + "root": { + "customer_email": task["customer_email"], + "destination": task["destination"], + }, + "calls": [step["call"] for step in steps], + "results": [step["tool_result"] for step in steps], + } + ) + if len(traces) < 2: + raise ValueError("training report contains fewer than two complete correct traces") + return traces + + +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def alphabetic_identifier(value: int) -> str: + """Encode an integer with letters so copy accuracy is not biased by 0/o.""" + prefix = "warm" if value < 0 else "task" + number = abs(value) + encoded = "" + while True: + encoded = chr(ord("a") + number % 26) + encoded + number = number // 26 - 1 + if number < 0: + break + return prefix + encoded + + +def make_task( + index: int, branch_count: int, pattern: CompiledPattern +) -> dict[str, Any]: + destinations = ("Rome", "Milan", "Turin", "Bologna", "Florence", "Naples") + items = [] + used_refs = [set() for _ in pattern.steps] + candidate = max(index, 0) * 100 + while len(items) < branch_count and candidate < max(index, 0) * 100 + 20_000: + root = { + "customer_email": ( + f"agent-{alphabetic_identifier(index)}-" + f"{alphabetic_identifier(candidate)}@example.test" + ), + "destination": destinations[(index + candidate) % len(destinations)], + } + refs = [call_ref(call) for call in pattern.simulate(root)] + if all(reference not in used_refs[step] for step, reference in enumerate(refs)): + items.append(root) + for step, reference in enumerate(refs): + used_refs[step].add(reference) + candidate += 1 + if len(items) != branch_count: + raise RuntimeError("could not construct collision-free workflow branches") + return { + "id": f"trace_compiled_{index:03d}", + "workflow_ref": f"workflow_{alphabetic_identifier(index)}", + "items": items, + "branch_count": branch_count, + "call_count": branch_count * len(pattern.steps), + } + + +def request_content(task: dict[str, Any]) -> str: + rendered = "; ".join( + f"{item['customer_email']} to {item['destination']}" for item in task["items"] + ) + return f"Customers: {rendered}." + + +def workflow_reference(task: dict[str, Any], pattern: CompiledPattern) -> str: + del pattern + workflow_ref = task.get("workflow_ref") + if not isinstance(workflow_ref, str) or re.fullmatch( + r"workflow_[a-z]+", workflow_ref + ) is None: + raise ValueError("task has no valid request-scoped workflow_ref") + return workflow_ref + + +def write_workflow_registry( + path: Path, pattern: CompiledPattern, tasks: list[dict[str, Any]] +) -> None: + workflows = { + workflow_reference(task, pattern): { + "pattern_fingerprint": pattern.fingerprint, + "items": task["items"], + } + for task in tasks + } + if len(workflows) != len(tasks): + raise ValueError("workflow_ref collision in request-scoped registry") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps( + { + "schema_version": 1, + "pattern_fingerprint": pattern.fingerprint, + "workflows": workflows, + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + + +def parse_request_customers(content: str) -> list[dict[str, str]]: + prefix = "Customers: " + if not content.startswith(prefix) or not content.endswith("."): + raise ValueError("request does not use the validated customer-list format") + customers = [] + for entry in content[len(prefix) : -1].split("; "): + match = re.fullmatch(r"([^\s;]+) to ([A-Za-z][A-Za-z -]*)", entry) + if match is None: + raise ValueError("request customer entry is malformed") + customers.append( + {"customer_email": match.group(1), "destination": match.group(2)} + ) + if not customers: + raise ValueError("request contains no customers") + return customers + + +def stage_batch_tool( + pattern: CompiledPattern, + step_index: int, + max_items: int, + stage_ref: str | None = None, +) -> dict[str, Any]: + step = pattern.steps[step_index] + if stage_ref is not None: + parameters = { + "type": "object", + "properties": { + "stage_ref": { + "type": "string", + "enum": [stage_ref], + "description": "Bound batch for this workflow stage.", + } + }, + "required": ["stage_ref"], + "additionalProperties": False, + } + else: + properties = {name: {"type": "string"} for name, _ in step.arguments} + parameters = { + "type": "object", + "properties": { + "calls": { + "type": "array", + "items": { + "type": "object", + "properties": properties, + "required": list(properties), + "additionalProperties": False, + }, + "minItems": 1, + "maxItems": max_items, + } + }, + "required": ["calls"], + "additionalProperties": False, + } + return { + "type": "function", + "function": { + "name": f"batch_{step.tool}", + "description": ( + f"Run {step.tool} once for every item. Calls execute concurrently " + "and results preserve input order." + ), + "parameters": parameters, + }, + } + + +def stage_batched_messages( + task: dict[str, Any], pattern: CompiledPattern +) -> list[dict[str, Any]]: + del pattern + system = ( + "The scheduler exposes exactly one currently-ready batch tool at a time. " + "Call only that declared tool once and copy its bound stage_ref exactly. " + "The bound batch contains every customer in input order. Do not invent or " + "name future tools, and emit no prose." + ) + return [ + {"role": "system", "content": system}, + {"role": "user", "content": request_content(task)}, + ] + + +def macro_messages(task: dict[str, Any], pattern: CompiledPattern) -> list[dict[str, Any]]: + return [ + { + "role": "system", + "content": ( + "Use the workflow tool exactly once for every requested customer. " + "Copy its bound workflow_ref exactly. No prose." + ), + }, + {"role": "user", "content": request_content(task)}, + ] + + +def stage_reference(task: dict[str, Any], pattern: CompiledPattern, index: int) -> str: + stage_names = ("one", "two", "three", "four", "five") + if not 0 <= index < len(stage_names): + raise ValueError("stage index is outside the compiled workflow") + return f"{workflow_reference(task, pattern)}_stage_{stage_names[index]}" + + +def model_observation(result: dict[str, Any], wall_ms: float) -> dict[str, Any]: + choices = result.get("choices") + message = choices[0].get("message") if isinstance(choices, list) and choices else None + if not isinstance(message, dict): + raise RuntimeError("model response has no assistant message") + raw_calls = message.get("tool_calls") + if not isinstance(raw_calls, list): + raw_calls = [] + calls = [] + for raw in raw_calls: + function = raw.get("function") if isinstance(raw, dict) else None + if not isinstance(function, dict): + raise RuntimeError("model emitted a malformed tool call") + arguments = function.get("arguments") + if isinstance(arguments, str): + arguments = json.loads(arguments) + if not isinstance(function.get("name"), str) or not isinstance(arguments, dict): + raise RuntimeError("model emitted invalid tool name or arguments") + call_id = raw.get("id") + if not isinstance(call_id, str) or not call_id: + raise RuntimeError("model tool call has no id") + calls.append( + { + "id": call_id, + "call": {"name": function["name"], "arguments": arguments}, + } + ) + content = message.get("content") + if not isinstance(content, str): + content = "" + usage = result.get("usage") if isinstance(result.get("usage"), dict) else {} + timings = usage.get("timings") if isinstance(usage.get("timings"), dict) else {} + assistant_message = dict(message) + assistant_message["role"] = "assistant" + assistant_message["content"] = content + content_format_call = False + if not calls and content: + parsed_call = normalize_tool_call(result) + if parsed_call is not None: + content_format_call = True + call_id = "call_content_" + hashlib.sha256( + canonical_call(parsed_call).encode() + ).hexdigest()[:16] + calls.append({"id": call_id, "call": parsed_call}) + assistant_message = { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": call_id, + "type": "function", + "function": { + "name": parsed_call["name"], + "arguments": json.dumps( + parsed_call["arguments"], + sort_keys=True, + separators=(",", ":"), + ), + }, + } + ], + } + return { + "request_wall_ms": wall_ms, + "model_compute_ms": float(timings.get("prefill_ms", 0.0)) + + float(timings.get("decode_ms", 0.0)), + "prefill_ms": float(timings.get("prefill_ms", 0.0)), + "decode_ms": float(timings.get("decode_ms", 0.0)), + "decode_tokens_per_sec": float(timings.get("decode_tokens_per_sec", 0.0)), + "cache_hit": bool(timings.get("cache_hit", False)), + "cached_prefix_tokens": int(timings.get("cached_prefix_tokens", 0)), + "completion_tokens": int(usage.get("completion_tokens", 0)), + "accept_rate": float(usage.get("accept_rate", 0.0)), + "content": content, + "content_sha256": hashlib.sha256(content.encode()).hexdigest(), + "assistant_message": assistant_message, + "calls": calls, + "content_format_call": content_format_call, + } + + +def post_turn( + args: argparse.Namespace, + messages: list[dict[str, Any]], + tools: list[dict[str, Any]], + tool_choice: Any, + max_tokens: int, + *, + automatic_tool_speculation: bool = False, +) -> dict[str, Any]: + result, wall_ms = post_json( + args.url, + { + "model": "dflash", + "messages": messages, + "tools": tools, + "tool_choice": tool_choice, + "temperature": 0, + "seed": args.seed, + "max_tokens": max_tokens, + "stream": False, + "automatic_tool_speculation": automatic_tool_speculation, + }, + args.timeout, + ) + observation = model_observation(result, wall_ms) + observation["speculation"] = result.get("dflash_tool_speculation") + return observation + + +def execute_tool_safe( + binary: Path, + call: dict[str, Any], + cpus: list[int], + timeout: float, + request_id: str, +) -> dict[str, Any]: + request = { + "protocol": PROTOCOL, + "request_id": request_id, + "resource_percentage": 100, + "accelerator_relation": "non_accelerator", + "cpu_affinity": cpus, + "cpu_affinity_isolated": True, + "call": call, + } + command = [str(binary), "--dflash-tool-spec-v1"] + if os.name == "posix" and Path("/usr/bin/taskset").is_file(): + command = ["/usr/bin/taskset", "-c", ",".join(map(str, cpus)), *command] + started = time.perf_counter() + process = subprocess.run( + command, + input=json.dumps(request, separators=(",", ":")) + "\n", + text=True, + capture_output=True, + timeout=timeout, + check=False, + ) + wall_ms = (time.perf_counter() - started) * 1_000.0 + if process.returncode != 0: + raise RuntimeError( + f"tool executor exited {process.returncode}: {process.stderr.strip()}" + ) + envelope = json.loads(process.stdout) + result = envelope.get("result") if isinstance(envelope, dict) else None + if not envelope.get("ok") or not isinstance(result, dict): + raise RuntimeError(f"tool executor returned invalid data: {envelope!r}") + if ( + result.get("call_sha256") != call_sha256(call) + or result.get("call_ref") != call_ref(call) + or result.get("tool_name") != call.get("name") + or result.get("side_effects") is not False + ): + raise RuntimeError("tool result does not exactly match the read-only call") + return {"wall_ms": wall_ms, "result": result} + + +def run_branch( + args: argparse.Namespace, + pattern: CompiledPattern, + root: dict[str, str], + label: str, +) -> dict[str, Any]: + started = time.perf_counter() + steps = [] + previous: dict[str, Any] | None = None + for index in range(len(pattern.steps)): + call = pattern.instantiate(root, previous, index) + tool = execute_tool_safe( + args.binary, + call, + args.tool_cpus, + args.timeout, + f"{label}-step-{index + 1}", + ) + previous = tool["result"] + steps.append({"call": call, "tool_result": previous, "tool_wall_ms": tool["wall_ms"]}) + return { + "root": root, + "steps": steps, + "final_ref": steps[-1]["tool_result"]["call_ref"], + "wall_ms": (time.perf_counter() - started) * 1_000.0, + } + + +def start_graph( + args: argparse.Namespace, + pattern: CompiledPattern, + items: list[dict[str, str]], + label: str, +) -> dict[str, Any]: + pool = ThreadPoolExecutor(max_workers=len(items), thread_name_prefix="tool-graph") + started = time.perf_counter() + futures: list[Future[dict[str, Any]]] = [ + pool.submit(run_branch, args, pattern, root, f"{label}-branch-{index}") + for index, root in enumerate(items) + ] + return {"pool": pool, "futures": futures, "started": started} + + +def finish_graph(handle: dict[str, Any], timeout: float) -> dict[str, Any]: + pool: ThreadPoolExecutor = handle["pool"] + futures: list[Future[dict[str, Any]]] = handle["futures"] + try: + branches = [future.result(timeout=timeout) for future in futures] + finally: + pool.shutdown(wait=True, cancel_futures=True) + return { + "branches": branches, + "wall_ms": (time.perf_counter() - float(handle["started"])) * 1_000.0, + } + + +def expected_final(branches: list[dict[str, Any]]) -> str: + return "workflow_complete:" + ",".join(branch["final_ref"] for branch in branches) + + +def final_answer_correct(content: str, expected: str) -> bool: + """Accept the receipt literally or as the equivalent one-field JSON object.""" + content = content.strip() + if content == expected: + return True + prefix = "workflow_complete:" + if not expected.startswith(prefix): + return False + receipt = expected[len(prefix) :] + if content == receipt: + return True + try: + parsed = json.loads(content) + except json.JSONDecodeError: + return False + return parsed == {"workflow_complete": receipt} + + +def post_final( + args: argparse.Namespace, + expected: str, +) -> dict[str, Any]: + """Measure the same minimal final-response turn for every benchmark arm.""" + return post_turn( + args, + [ + { + "role": "system", + "content": ( + "Return the opaque workflow receipt from the user message exactly. " + "Do not explain, reformat, or add any characters." + ), + }, + {"role": "user", "content": expected}, + ], + [], + "none", + args.final_max_tokens, + ) + + +def flatten_graph_calls(graph: dict[str, Any]) -> list[str]: + return [ + canonical_call(step["call"]) + for branch in graph["branches"] + for step in branch["steps"] + ] + + +def flatten_graph_results(graph: dict[str, Any]) -> list[str]: + return [ + step["tool_result"]["call_sha256"] + for branch in graph["branches"] + for step in branch["steps"] + ] + + +def run_stage_batched( + args: argparse.Namespace, + task: dict[str, Any], + pattern: CompiledPattern, + label: str, +) -> dict[str, Any]: + """Run a strong non-speculative baseline with parallel calls per stage.""" + started = time.perf_counter() + stage_messages = stage_batched_messages(task, pattern) + current_tools: list[dict[str, Any]] = [] + branches = [ + {"root": root, "steps": [], "final_ref": ""} for root in task["items"] + ] + previous_results: list[dict[str, Any] | None] = [None] * len(branches) + turns = [] + exposed_tool_wait_ms = 0.0 + + for step_index, step in enumerate(pattern.steps): + expected_calls = [ + pattern.instantiate(branch["root"], previous_results[index], step_index) + for index, branch in enumerate(branches) + ] + batch_name = f"batch_{step.tool}" + stage_ref = stage_reference(task, pattern, step_index) + current_tools = [ + stage_batch_tool(pattern, step_index, args.max_branches, stage_ref) + ] + expected_batch = { + "name": batch_name, + "arguments": {"stage_ref": stage_ref}, + } + model = post_turn( + args, + stage_messages, + current_tools, + {"type": "function", "function": {"name": batch_name}}, + args.call_max_tokens, + ) + if len(model["calls"]) != 1 or model["calls"][0]["call"] != expected_batch: + raise RuntimeError( + f"{task['id']} stage {step_index + 1}: batch call " + f"{[item['call'] for item in model['calls']]!r} != " + f"{expected_batch!r}; content={model['content']!r}" + ) + stage_started = time.perf_counter() + with ThreadPoolExecutor( + max_workers=len(expected_calls), thread_name_prefix="batched-tool-stage" + ) as pool: + futures = [ + pool.submit( + execute_tool_safe, + args.binary, + call, + args.tool_cpus, + args.timeout, + f"{label}-stage-{step_index}-branch-{branch_index}", + ) + for branch_index, call in enumerate(expected_calls) + ] + stage_tools = [future.result(timeout=args.timeout) for future in futures] + exposed_tool_wait_ms += (time.perf_counter() - stage_started) * 1_000.0 + + for branch_index, (call, tool) in enumerate( + zip(expected_calls, stage_tools, strict=True) + ): + result = tool["result"] + previous_results[branch_index] = result + branches[branch_index]["steps"].append( + { + "call": call, + "tool_result": result, + "tool_wall_ms": tool["wall_ms"], + } + ) + turns.append(model) + + for branch in branches: + branch["final_ref"] = branch["steps"][-1]["tool_result"]["call_ref"] + expected = expected_final(branches) + final = post_final(args, expected) + all_turns = [*turns, final] + graph = {"branches": branches} + return { + "task_ms": (time.perf_counter() - started) * 1_000.0, + "model_turns": len(all_turns), + "call_turns": turns, + "final": final, + "expected_final": expected, + "final_correct": final_answer_correct(final["content"], expected), + "graph": graph, + "underlying_calls": flatten_graph_calls(graph), + "tool_results": flatten_graph_results(graph), + "model_compute_ms": sum(turn["model_compute_ms"] for turn in all_turns), + "decode_ms": sum(turn["decode_ms"] for turn in all_turns), + "completion_tokens": sum(turn["completion_tokens"] for turn in all_turns), + "exposed_tool_wait_ms": exposed_tool_wait_ms, + "all_ds4_active": all(turn["accept_rate"] > 0.0 for turn in turns), + } + + +def macro_result_message( + pattern: CompiledPattern, + graph: dict[str, Any], + tool_call_id: str, +) -> dict[str, Any]: + content = { + "workflow": pattern.macro_name, + "call_count": sum(len(branch["steps"]) for branch in graph["branches"]), + "items": [ + { + **branch["root"], + "final_ref": branch["final_ref"], + } + for branch in graph["branches"] + ], + "side_effects": False, + } + return { + "role": "tool", + "tool_call_id": tool_call_id, + "name": pattern.macro_name, + "content": json.dumps(content, sort_keys=True, separators=(",", ":")), + } + + +def graph_from_speculative_result( + metadata: dict[str, Any], pattern: CompiledPattern, call: dict[str, Any] +) -> dict[str, Any]: + result = metadata.get("result") + if not isinstance(result, dict): + raise RuntimeError("engine speculation hit has no compiled workflow result") + branches = result.get("branches") + if ( + result.get("call_sha256") != call_sha256(call) + or result.get("call_ref") != call_ref(call) + or result.get("tool_name") != pattern.macro_name + or result.get("workflow_fingerprint") != pattern.fingerprint + or result.get("side_effects") is not False + or not isinstance(branches, list) + ): + raise RuntimeError("engine returned an invalid compiled workflow result") + return {"branches": branches, "wall_ms": float(result.get("elapsed_ms", 0.0))} + + +def run_macro( + args: argparse.Namespace, + task: dict[str, Any], + pattern: CompiledPattern, + speculative: bool, + label: str, +) -> dict[str, Any]: + started = time.perf_counter() + messages = macro_messages(task, pattern) + workflow_ref = workflow_reference(task, pattern) + tools = [pattern.macro_tool(args.max_branches, workflow_ref)] + parsed_items = parse_request_customers(messages[-1]["content"]) + if parsed_items != task["items"]: + raise RuntimeError("event extractor did not reproduce structured request data") + expected_call = { + "name": pattern.macro_name, + "arguments": {"workflow_ref": workflow_ref}, + } + model = post_turn( + args, + messages, + tools, + "required", + args.macro_max_tokens, + automatic_tool_speculation=speculative, + ) + if len(model["calls"]) != 1: + raise RuntimeError(f"{task['id']}: macro turn emitted {len(model['calls'])} calls") + emitted = model["calls"][0] + macro_correct = emitted["call"] == expected_call + if not macro_correct: + raise RuntimeError( + f"{task['id']}: macro call {emitted['call']!r} != {expected_call!r}" + ) + + metadata = model.get("speculation") if speculative else None + if speculative and not isinstance(metadata, dict): + raise RuntimeError("automatic Qwen speculation returned no engine metadata") + prediction = metadata.get("prediction") if isinstance(metadata, dict) else None + prediction_hit = ( + speculative + and metadata.get("status") == "hit" + and prediction == expected_call + ) + predictor_ms = ( + float(metadata.get("predictor_wall_ms", 0.0)) + if isinstance(metadata, dict) + else 0.0 + ) + if prediction_hit: + graph = graph_from_speculative_result(metadata, pattern, emitted["call"]) + exposed_wait_ms = float(metadata.get("commit_wait_ms", 0.0)) + else: + graph_handle = start_graph(args, pattern, task["items"], f"{label}-authoritative") + wait_started = time.perf_counter() + graph = finish_graph(graph_handle, args.timeout) + exposed_wait_ms = (time.perf_counter() - wait_started) * 1_000.0 + expected_calls = [ + canonical_call(call) + for root in task["items"] + for call in pattern.simulate(root) + ] + actual_calls = flatten_graph_calls(graph) + if actual_calls != expected_calls: + raise RuntimeError(f"{task['id']}: compiled graph diverged from learned pattern") + messages.extend( + [ + model["assistant_message"], + macro_result_message(pattern, graph, emitted["id"]), + ] + ) + expected = expected_final(graph["branches"]) + final = post_final(args, expected) + all_turns = [model, final] + return { + "task_ms": (time.perf_counter() - started) * 1_000.0, + "model_turns": len(all_turns), + "call_turns": [model], + "macro_call": emitted["call"], + "macro_correct": macro_correct, + "prediction_hit": prediction_hit, + "prediction_source": ( + metadata.get("prediction_source") if isinstance(metadata, dict) else None + ), + "prediction_status": metadata.get("status") if isinstance(metadata, dict) else None, + "prediction_reason": metadata.get("reason") if isinstance(metadata, dict) else None, + "predictor_ms": predictor_ms, + "graph": graph, + "graph_wall_ms": graph["wall_ms"], + "exposed_tool_wait_ms": exposed_wait_ms, + "underlying_calls": actual_calls, + "tool_results": flatten_graph_results(graph), + "final": final, + "expected_final": expected, + "final_correct": final_answer_correct(final["content"], expected), + "model_compute_ms": sum(turn["model_compute_ms"] for turn in all_turns), + "decode_ms": sum(turn["decode_ms"] for turn in all_turns), + "completion_tokens": sum(turn["completion_tokens"] for turn in all_turns), + "all_ds4_active": model["accept_rate"] > 0.0, + } + + +def macro_signature(arm: dict[str, Any]) -> dict[str, Any]: + turn = arm["call_turns"][0] + return { + "macro_call": canonical_call(arm["macro_call"]), + "turn_content": turn["content_sha256"], + "turn_tokens": turn["completion_tokens"], + "final_content": arm["final"]["content_sha256"], + "final_tokens": arm["final"]["completion_tokens"], + } + + +def bootstrap_speedup_ci( + pairs: list[dict[str, Any]], numerator: str, denominator: str, resamples: int, seed: int +) -> list[float]: + generator = random.Random(seed) + values = [] + for _ in range(resamples): + sample = [pairs[generator.randrange(len(pairs))] for _ in pairs] + values.append( + statistics.median( + pair[numerator]["task_ms"] / pair[denominator]["task_ms"] + for pair in sample + ) + ) + return [percentile(values, 0.025), percentile(values, 0.975)] + + +def paired_slowdown( + pairs: list[dict[str, Any]], metric: str, quantile: float +) -> float: + ratios = [ + pair["speculative"][metric] / pair["compiled"][metric] + for pair in pairs + if pair["compiled"][metric] > 0.0 + ] + return 100.0 * (percentile(ratios, quantile) - 1.0) + + +def summarize( + pairs: list[dict[str, Any]], resamples: int, seed: int +) -> dict[str, Any]: + baseline_speedups = [ + pair["stage_batched"]["task_ms"] / pair["speculative"]["task_ms"] + for pair in pairs + ] + compiled_speedups = [ + pair["compiled"]["task_ms"] / pair["speculative"]["task_ms"] for pair in pairs + ] + continuation_turns = [ + turn + for pair in pairs + for arm_name in ("stage_batched", "compiled", "speculative") + for turn in [*pair[arm_name]["call_turns"][1:], pair[arm_name]["final"]] + ] + speedup_by_call_count = {} + for call_count in sorted({pair["task"]["call_count"] for pair in pairs}): + bucket = [pair for pair in pairs if pair["task"]["call_count"] == call_count] + speedup_by_call_count[str(call_count)] = { + "tasks": len(bucket), + "stage_batched_task_p50_ms": statistics.median( + pair["stage_batched"]["task_ms"] for pair in bucket + ), + "compiled_task_p50_ms": statistics.median( + pair["compiled"]["task_ms"] for pair in bucket + ), + "speculative_task_p50_ms": statistics.median( + pair["speculative"]["task_ms"] for pair in bucket + ), + "combined_speedup_p50": statistics.median( + pair["stage_batched"]["task_ms"] + / pair["speculative"]["task_ms"] + for pair in bucket + ), + "speculation_speedup_p50": statistics.median( + pair["compiled"]["task_ms"] / pair["speculative"]["task_ms"] + for pair in bucket + ), + } + return { + "tasks": len(pairs), + "calls_per_task": [pair["task"]["call_count"] for pair in pairs], + "stage_batched_task_p50_ms": statistics.median( + pair["stage_batched"]["task_ms"] for pair in pairs + ), + "compiled_task_p50_ms": statistics.median( + pair["compiled"]["task_ms"] for pair in pairs + ), + "speculative_task_p50_ms": statistics.median( + pair["speculative"]["task_ms"] for pair in pairs + ), + "stage_batched_to_speculative_speedup_p50": statistics.median( + baseline_speedups + ), + "stage_batched_to_speculative_speedup_p05": percentile( + baseline_speedups, 0.05 + ), + "stage_batched_to_speculative_speedup_min": min(baseline_speedups), + "stage_batched_to_speculative_bootstrap_95ci": bootstrap_speedup_ci( + pairs, "stage_batched", "speculative", resamples, seed + ), + "stage_batched_to_compiled_speedup_p50": statistics.median( + pair["stage_batched"]["task_ms"] / pair["compiled"]["task_ms"] + for pair in pairs + ), + "compiled_to_speculative_speedup_p50": statistics.median(compiled_speedups), + "compiled_to_speculative_speedup_p05": percentile(compiled_speedups, 0.05), + "compiled_to_speculative_bootstrap_95ci": bootstrap_speedup_ci( + pairs, "compiled", "speculative", resamples, seed + 1 + ), + "total_wall_speedup": sum( + pair["stage_batched"]["task_ms"] for pair in pairs + ) + / sum(pair["speculative"]["task_ms"] for pair in pairs), + "stage_batched_model_turns_p50": statistics.median( + pair["stage_batched"]["model_turns"] for pair in pairs + ), + "compiled_model_turns_p50": statistics.median( + pair["compiled"]["model_turns"] for pair in pairs + ), + "stage_batched_exposed_tool_wait_p50_ms": statistics.median( + pair["stage_batched"]["exposed_tool_wait_ms"] for pair in pairs + ), + "compiled_exposed_tool_wait_p50_ms": statistics.median( + pair["compiled"]["exposed_tool_wait_ms"] for pair in pairs + ), + "speculative_exposed_tool_wait_p50_ms": statistics.median( + pair["speculative"]["exposed_tool_wait_ms"] for pair in pairs + ), + "pattern_prediction_hit_rate": sum( + pair["speculative"]["prediction_hit"] for pair in pairs + ) + / len(pairs), + "all_predictions_from_qwen": all( + pair["speculative"]["prediction_source"] == NATIVE_PREDICTION_SOURCE + for pair in pairs + ), + "predictor_p50_ms": statistics.median( + pair["speculative"]["predictor_ms"] for pair in pairs + ), + "model_compute_slowdown_p50_percent": paired_slowdown( + pairs, "model_compute_ms", 0.50 + ), + "model_compute_slowdown_p95_percent": paired_slowdown( + pairs, "model_compute_ms", 0.95 + ), + "decode_slowdown_p50_percent": paired_slowdown(pairs, "decode_ms", 0.50), + "decode_slowdown_p95_percent": paired_slowdown(pairs, "decode_ms", 0.95), + "continuation_cache_hit_rate": sum( + turn["cache_hit"] and turn["cached_prefix_tokens"] > 0 + for turn in continuation_turns + ) + / len(continuation_turns), + "speedup_by_call_count": speedup_by_call_count, + "all_calls_stable": all( + pair["stage_batched"]["underlying_calls"] + == pair["compiled"]["underlying_calls"] + == pair["speculative"]["underlying_calls"] + for pair in pairs + ), + "all_tool_results_stable": all( + pair["stage_batched"]["tool_results"] + == pair["compiled"]["tool_results"] + == pair["speculative"]["tool_results"] + for pair in pairs + ), + "macro_output_stability_rate": sum( + macro_signature(pair["compiled"]) == macro_signature(pair["speculative"]) + for pair in pairs + ) + / len(pairs), + "all_final_answers_correct": all( + pair[arm]["final_correct"] + for pair in pairs + for arm in ("stage_batched", "compiled", "speculative") + ), + "all_final_outputs_stable": all( + pair["stage_batched"]["final"]["content"].strip() + == pair["compiled"]["final"]["content"].strip() + == pair["speculative"]["final"]["content"].strip() + for pair in pairs + ), + "all_macro_calls_correct": all( + pair[arm]["macro_correct"] + for pair in pairs + for arm in ("compiled", "speculative") + ), + "all_ds4_active": all( + pair[arm]["all_ds4_active"] + for pair in pairs + for arm in ("stage_batched", "compiled", "speculative") + ), + } + + +def production_checks(summary: dict[str, Any], args: argparse.Namespace) -> dict[str, bool]: + return { + "sample_size": summary["tasks"] >= args.min_production_pairs, + "end_to_end_speedup": summary["stage_batched_to_speculative_speedup_p50"] + >= args.min_e2e_speedup, + "end_to_end_ci": summary[ + "stage_batched_to_speculative_bootstrap_95ci" + ][0] + > 1.0, + "end_to_end_tail": summary["stage_batched_to_speculative_speedup_p05"] + >= args.min_e2e_speedup_p05, + "speculation_incremental_gain": summary["compiled_to_speculative_speedup_p50"] + >= args.min_incremental_speedup, + "speculation_incremental_ci": summary[ + "compiled_to_speculative_bootstrap_95ci" + ][0] + > 1.0, + "prediction_hit_rate": summary["pattern_prediction_hit_rate"] == 1.0, + "prediction_source": summary["all_predictions_from_qwen"], + "model_slowdown_p50": summary["model_compute_slowdown_p50_percent"] + <= args.max_model_slowdown_percent, + "model_slowdown_p95": summary["model_compute_slowdown_p95_percent"] + <= args.max_model_slowdown_p95_percent, + "decode_slowdown_p50": summary["decode_slowdown_p50_percent"] + <= args.max_decode_slowdown_percent, + "decode_slowdown_p95": summary["decode_slowdown_p95_percent"] + <= args.max_decode_slowdown_p95_percent, + "prefix_cache_configured": summary["prefix_cache_configured"], + "calls_stable": summary["all_calls_stable"], + "tool_results_stable": summary["all_tool_results_stable"], + "macro_outputs_stable": summary["macro_output_stability_rate"] == 1.0, + "final_answers_correct": summary["all_final_answers_correct"], + "final_outputs_stable": summary["all_final_outputs_stable"], + "macro_calls_correct": summary["all_macro_calls_correct"], + "ds4_active": summary["all_ds4_active"], + } + + +def compact_arm(arm: dict[str, Any]) -> dict[str, Any]: + """Keep auditable outputs and timings without embedding the full graph.""" + fields = ( + "task_ms", + "model_turns", + "model_compute_ms", + "decode_ms", + "completion_tokens", + "exposed_tool_wait_ms", + "all_ds4_active", + "final_correct", + "graph_wall_ms", + "macro_call", + "macro_correct", + "prediction_hit", + "prediction_reason", + "prediction_source", + "prediction_status", + "predictor_ms", + ) + compact = {field: arm[field] for field in fields if field in arm} + for field in ("underlying_calls", "tool_results"): + values = arm.get(field) + if isinstance(values, list): + compact[f"{field}_count"] = len(values) + compact[f"{field}_sha256"] = hashlib.sha256( + json.dumps(values, separators=(",", ":")).encode() + ).hexdigest() + final = arm.get("final") + if isinstance(final, dict): + compact["final"] = { + field: final[field] + for field in ( + "content_sha256", + "completion_tokens", + "accept_rate", + ) + if field in final + } + return compact + + +def compact_pair(pair: dict[str, Any]) -> dict[str, Any]: + task = pair["task"] + return { + "pair_index": pair["pair_index"], + "task": { + field: task[field] + for field in ("id", "branch_count", "call_count") + if field in task + }, + "arm_order": pair["arm_order"], + **{ + arm: compact_arm(pair[arm]) + for arm in ("stage_batched", "compiled", "speculative") + }, + } + + +def load_partial_pairs( + path: Path, + measured_tasks: list[dict[str, Any]], + arm_orders: list[list[str]], +) -> list[dict[str, Any]]: + """Load and strictly validate a checkpoint before resuming a long run.""" + checkpoint = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(checkpoint, dict): + raise ValueError("benchmark checkpoint is not an object") + pairs = checkpoint.get("pairs") + if ( + checkpoint.get("schema_version") != 1 + or checkpoint.get("complete") is not False + or not isinstance(pairs, list) + or len(pairs) > len(measured_tasks) + ): + raise ValueError("benchmark checkpoint is not resumable") + for index, pair in enumerate(pairs): + if ( + not isinstance(pair, dict) + or pair.get("pair_index") != index + or pair.get("task") != measured_tasks[index] + or pair.get("arm_order") != arm_orders[index] + or not all( + isinstance(pair.get(arm), dict) + for arm in ("stage_batched", "compiled", "speculative") + ) + ): + raise ValueError(f"benchmark checkpoint pair {index} does not match this run") + for arm in ("stage_batched", "compiled", "speculative"): + result = pair[arm] + final = result.get("final") + expected = result.get("expected_final") + if ( + not isinstance(final, dict) + or not isinstance(final.get("content"), str) + or not isinstance(expected, str) + ): + raise ValueError( + f"benchmark checkpoint pair {index} has an invalid {arm} final" + ) + result["final_correct"] = final_answer_correct( + final["content"], expected + ) + return pairs + + +def validate_args(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None: + if not args.binary.is_file() or not os.access(args.binary, os.X_OK): + parser.error("--binary must be an executable tool adapter") + if not args.training_report.is_file(): + parser.error("--training-report must be an existing trace file or report") + if ( + args.pairs <= 0 + or args.warmup_tasks < 0 + or not 2 <= args.min_branches <= args.max_branches <= 4 + or args.timeout <= 0 + or min(args.call_max_tokens, args.macro_max_tokens, args.final_max_tokens) <= 0 + or args.bootstrap_resamples <= 0 + or args.min_production_pairs < 2 + or args.min_e2e_speedup <= 1.0 + or args.min_e2e_speedup_p05 <= 1.0 + or args.min_incremental_speedup <= 1.0 + ): + parser.error("benchmark counts and thresholds are invalid") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--url", default="http://127.0.0.1:18145/v1/chat/completions") + parser.add_argument("--binary", type=Path, required=True) + parser.add_argument("--training-report", type=Path, required=True) + parser.add_argument( + "--workflow-registry", + type=Path, + default=Path(__file__).with_name("results") / "trace-workflow-registry.json", + ) + parser.add_argument("--tool-cpus", type=parse_cpu_list, default="14-15,30-31") + parser.add_argument("--pairs", type=int, default=6) + parser.add_argument("--warmup-tasks", type=int, default=1) + parser.add_argument("--min-branches", type=int, default=2) + parser.add_argument("--max-branches", type=int, default=4) + parser.add_argument("--call-max-tokens", type=int, default=160) + parser.add_argument("--macro-max-tokens", type=int, default=512) + parser.add_argument("--final-max-tokens", type=int, default=96) + parser.add_argument("--timeout", type=float, default=180.0) + parser.add_argument("--seed", type=int, default=814) + parser.add_argument("--bootstrap-resamples", type=int, default=20_000) + parser.add_argument("--min-production-pairs", type=int, default=6) + parser.add_argument("--min-e2e-speedup", type=float, default=2.0) + parser.add_argument("--min-e2e-speedup-p05", type=float, default=1.5) + parser.add_argument("--min-incremental-speedup", type=float, default=1.05) + parser.add_argument("--max-model-slowdown-percent", type=float, default=1.0) + parser.add_argument("--max-model-slowdown-p95-percent", type=float, default=5.0) + parser.add_argument("--max-decode-slowdown-percent", type=float, default=1.0) + parser.add_argument("--max-decode-slowdown-p95-percent", type=float, default=5.0) + parser.add_argument( + "--resume-partial", + action="store_true", + help="resume the strictly matching .partial checkpoint", + ) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + args.binary = args.binary.resolve() + args.training_report = args.training_report.resolve() + args.workflow_registry = args.workflow_registry.resolve() + validate_args(parser, args) + + traces = load_training_traces(args.training_report, required_steps=5) + pattern = mine_pattern(traces) + props = get_json(props_url(args.url), args.timeout) + prefix_cache = props.get("prefix_cache") + if not isinstance(prefix_cache, dict) or int(prefix_cache.get("capacity", 0)) <= 0: + raise SystemExit("prefix cache is not enabled") + tool_speculation = props.get("tool_speculation") + required_tool_props = { + "enabled": True, + "automatic_prediction_enabled": True, + "prediction_source": NATIVE_PREDICTION_SOURCE, + "predictor_schedule": "before-model", + "execution_mode": "child_process_cpu_affinity", + "cpu_affinity_isolated": True, + "preserves_token_speculation": True, + } + if not isinstance(tool_speculation, dict): + raise SystemExit("engine tool speculation is not enabled") + for key, expected in required_tool_props.items(): + if tool_speculation.get(key) != expected: + raise SystemExit( + f"engine tool_speculation.{key}={tool_speculation.get(key)!r}, " + f"expected {expected!r}" + ) + if pattern.macro_name not in tool_speculation.get("allowed_tools", []): + raise SystemExit(f"engine does not allow compiled macro {pattern.macro_name!r}") + + branch_span = args.max_branches - args.min_branches + 1 + warmup_tasks = [ + make_task(-1000 - index, args.min_branches, pattern) + for index in range(args.warmup_tasks) + ] + measured_tasks = [ + make_task( + pair_index, + args.min_branches + pair_index % branch_span, + pattern, + ) + for pair_index in range(args.pairs) + ] + write_workflow_registry( + args.workflow_registry, pattern, [*warmup_tasks, *measured_tasks] + ) + generator = random.Random(args.seed) + arm_orders = [] + for _ in measured_tasks: + order = ["stage_batched", "compiled", "speculative"] + generator.shuffle(order) + arm_orders.append(order) + partial_output = args.output.with_suffix(args.output.suffix + ".partial") + pairs = ( + load_partial_pairs(partial_output, measured_tasks, arm_orders) + if args.resume_partial + else [] + ) + if pairs: + print(json.dumps({"resumed_pairs": len(pairs)}), flush=True) + else: + for warmup, task in enumerate(warmup_tasks): + run_stage_batched(args, task, pattern, f"warm-{warmup}-stage-batched") + run_macro(args, task, pattern, False, f"warm-{warmup}-compiled") + run_macro(args, task, pattern, True, f"warm-{warmup}-speculative") + + for pair_index in range(len(pairs), len(measured_tasks)): + task = measured_tasks[pair_index] + order = arm_orders[pair_index] + arms = {} + for arm in order: + if arm == "stage_batched": + arms[arm] = run_stage_batched( + args, task, pattern, f"pair-{pair_index}-stage-batched" + ) + else: + arms[arm] = run_macro( + args, + task, + pattern, + arm == "speculative", + f"pair-{pair_index}-{arm}", + ) + pair = {"pair_index": pair_index, "task": task, "arm_order": order, **arms} + pairs.append(pair) + partial_output.parent.mkdir(parents=True, exist_ok=True) + partial_output.write_text( + json.dumps({"schema_version": 1, "complete": False, "pairs": pairs}, indent=2) + + "\n", + encoding="utf-8", + ) + print( + json.dumps( + { + "pair": pair_index + 1, + "calls": task["call_count"], + "order": order, + "stage_batched_ms": round( + arms["stage_batched"]["task_ms"], 1 + ), + "compiled_ms": round(arms["compiled"]["task_ms"], 1), + "speculative_ms": round(arms["speculative"]["task_ms"], 1), + "end_to_end_speedup": round( + arms["stage_batched"]["task_ms"] + / arms["speculative"]["task_ms"], + 3, + ), + "incremental_speedup": round( + arms["compiled"]["task_ms"] + / arms["speculative"]["task_ms"], + 3, + ), + "prediction_hit": arms["speculative"]["prediction_hit"], + "correct": all(arms[arm]["final_correct"] for arm in arms), + }, + sort_keys=True, + ), + flush=True, + ) + + summary = summarize(pairs, args.bootstrap_resamples, args.seed) + ending_props = get_json(props_url(args.url), args.timeout) + ending_prefix_cache = ending_props.get("prefix_cache") + if not isinstance(ending_prefix_cache, dict): + raise RuntimeError("prefix cache disappeared during the benchmark") + summary["prefix_cache_lifetime_hit_delta"] = int( + ending_prefix_cache.get("lifetime_hits", 0) + ) - int(prefix_cache.get("lifetime_hits", 0)) + summary["prefix_cache_configured"] = int(prefix_cache.get("capacity", 0)) > 0 + checks = production_checks(summary, args) + report = { + "schema_version": 1, + "host": "lucebox5", + "feature": "no-training trace-compiled speculative tool graphs", + "pattern": { + "macro_name": pattern.macro_name, + "fingerprint": pattern.fingerprint, + "training_traces": pattern.training_traces, + "training_report": str(args.training_report), + "training_report_sha256": file_sha256(args.training_report), + "workflow_registry": str(args.workflow_registry), + "workflow_registry_sha256": file_sha256(args.workflow_registry), + "root_fields": list(pattern.root_fields), + "steps": [ + { + "tool": step.tool, + "arguments": { + name: {"source": binding.source, "key": binding.key} + for name, binding in step.arguments + }, + } + for step in pattern.steps + ], + "model_training": False, + "side_effects_allowed": False, + }, + "workload": { + "tasks": args.pairs, + "branches_per_task": f"{args.min_branches}-{args.max_branches}", + "calls_per_task": f"{args.min_branches * len(pattern.steps)}-" + f"{args.max_branches * len(pattern.steps)}", + "dependency": "five serial calls per branch; branches are independent", + "tool_adapter": "deterministic read-only 2-second API replay", + }, + "methodology": { + "stage_batched": ( + "DS4+DSpark sees only the currently-ready typed batch, authorizes " + "one per dependency stage, runs its calls concurrently, and receives " + "a compact rolling state instead of replaying old tool history" + ), + "compiled": ( + "one DS4+DSpark macro authorization; independent branches execute " + "concurrently on the Strix CPU lane" + ), + "speculative": ( + "Qwen predicts the trace-derived macro through the engine; its CPU " + "graph overlaps DS4+DSpark and commits only on an exact call match" + ), + "arm_order": "randomized per task", + "measured_wall_time": "request through all tools and exact final answer", + "oracle_prediction": False, + "argument_binding": ( + "the harness binds validated structured inputs to a request-scoped " + "workflow_ref before either model runs" + ), + "model_seed": args.seed, + "warmup_tasks": args.warmup_tasks, + }, + "server_snapshot": { + "prefix_cache_before": prefix_cache, + "prefix_cache_after": ending_prefix_cache, + "tool_speculation": tool_speculation, + "model": props.get("model"), + }, + "production_gate": { + "passed": all(checks.values()), + "checks": checks, + "thresholds": { + "min_e2e_speedup": args.min_e2e_speedup, + "min_e2e_speedup_p05": args.min_e2e_speedup_p05, + "min_incremental_speedup": args.min_incremental_speedup, + "min_production_pairs": args.min_production_pairs, + "max_model_slowdown_percent": args.max_model_slowdown_percent, + "max_model_slowdown_p95_percent": args.max_model_slowdown_p95_percent, + "max_decode_slowdown_percent": args.max_decode_slowdown_percent, + "max_decode_slowdown_p95_percent": args.max_decode_slowdown_p95_percent, + }, + }, + "summary": summary, + "pairs": [compact_pair(pair) for pair in pairs], + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + if report["production_gate"]["passed"]: + partial_output.unlink(missing_ok=True) + print( + json.dumps( + {"production_gate": report["production_gate"], "summary": summary}, + indent=2, + sort_keys=True, + ), + flush=True, + ) + return 0 if report["production_gate"]["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/optimizations/ooo_spec_lucebox5_cpu/bfcl_replay_tool_executor.py b/optimizations/ooo_spec_lucebox5_cpu/bfcl_replay_tool_executor.py new file mode 100755 index 000000000..e1d79a101 --- /dev/null +++ b/optimizations/ooo_spec_lucebox5_cpu/bfcl_replay_tool_executor.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""Read-only executor for end-to-end BFCL tool-speculation replay. + +BFCL functions are specifications rather than deployable APIs. This adapter +therefore performs no external action: it waits for a fixed, documented API +latency and returns a deterministic digest of the canonical call. The engine's +own allowlist remains the authority for which predictions may reach it. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import sys +import time +from typing import Any + + +PROTOCOL = "dflash.tool-speculation.v1" +LATENCY_MS = 2_000 +REFERENCE_WORDS = ( + "amber", "azure", "cedar", "coral", "gold", "ivory", "maple", "olive", + "pearl", "plum", "sable", "silver", "teal", "violet", "willow", "jade", +) + + +def canonical_call(call: dict[str, Any]) -> str: + return json.dumps(call, sort_keys=True, separators=(",", ":")) + + +def call_sha256(call: dict[str, Any]) -> str: + return hashlib.sha256(canonical_call(call).encode()).hexdigest() + + +def call_ref(call: dict[str, Any]) -> str: + return REFERENCE_WORDS[int(call_sha256(call)[:8], 16) % len(REFERENCE_WORDS)] + + +def execute(request: dict[str, Any]) -> dict[str, Any]: + if request.get("protocol") != PROTOCOL: + raise ValueError("unsupported protocol") + call = request.get("call") + if not isinstance(call, dict): + raise ValueError("call must be an object") + name = call.get("name") + arguments = call.get("arguments") + if not isinstance(name, str) or not name: + raise ValueError("tool name must be a non-empty string") + if not isinstance(arguments, dict): + raise ValueError("tool arguments must be an object") + + expected = request.get("cpu_affinity") or [] + if not isinstance(expected, list) or not all( + isinstance(cpu, int) and cpu >= 0 for cpu in expected + ): + raise ValueError("cpu_affinity must contain non-negative integers") + observed = sorted(os.sched_getaffinity(0)) if hasattr( + os, "sched_getaffinity" + ) else [] + if expected and observed != sorted(set(expected)): + raise ValueError("observed CPU affinity does not match request") + + canonical = {"name": name, "arguments": arguments} + digest = call_sha256(canonical) + reference = call_ref(canonical) + started = time.perf_counter() + time.sleep(LATENCY_MS / 1_000.0) + elapsed = (time.perf_counter() - started) * 1_000.0 + return { + "ok": True, + "result": { + "call_sha256": digest, + "call_ref": reference, + "tool_name": name, + "latency_ms": LATENCY_MS, + "elapsed_ms": elapsed, + "cpu_affinity": observed, + "side_effects": False, + }, + } + + +def main() -> int: + if sys.argv[1:] != ["--dflash-tool-spec-v1"]: + print("expected --dflash-tool-spec-v1", file=sys.stderr) + return 2 + try: + line = sys.stdin.readline() + if not line: + raise ValueError("missing request") + request = json.loads(line) + if not isinstance(request, dict): + raise ValueError("request must be an object") + print(json.dumps(execute(request), separators=(",", ":")), flush=True) + return 0 + except (OSError, ValueError, json.JSONDecodeError) as error: + print(str(error), file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/optimizations/ooo_spec_lucebox5_cpu/dflash_server_native_tool_predictor_wrapper.sh b/optimizations/ooo_spec_lucebox5_cpu/dflash_server_native_tool_predictor_wrapper.sh new file mode 100755 index 000000000..d8e118d85 --- /dev/null +++ b/optimizations/ooo_spec_lucebox5_cpu/dflash_server_native_tool_predictor_wrapper.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Preserve every argument from the qualified 0731 launcher and add the native +# Qwen3 tool-prediction lane on the Strix GPU. The qualified launcher clears +# ambient variables, so an adjacent `candidate-build` symlink is the durable +# deployment override; direct launches may still use CANDIDATE_BUILD. +wrapper_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +default_candidate="/home/lucebox5/tool-spec-cpu-20260813/engine-ooo-spec/server/build-hip-dual" +if [[ -d "${wrapper_dir}/candidate-build" ]]; then + default_candidate="${wrapper_dir}/candidate-build" +fi +CANDIDATE_BUILD="${CANDIDATE_BUILD:-${default_candidate}}" +PREDICTOR_MODEL="${PREDICTOR_MODEL:-/home/lucebox5/tool-spec-cpu-20260813/models/Qwen3-0.6B-Q8_0.gguf}" +PREDICTOR_IPC_BIN="${PREDICTOR_IPC_BIN:-${CANDIDATE_BUILD}/backend_ipc_daemon}" +PREDICTOR_GPU="${PREDICTOR_GPU:-1}" +PREDICTOR_MAX_CTX="${PREDICTOR_MAX_CTX:-4096}" +PREDICTOR_MAX_TOKENS="${PREDICTOR_MAX_TOKENS:-256}" +PREDICTOR_CONFIDENCE="${PREDICTOR_CONFIDENCE:-0.75}" +PREDICTOR_SCHEDULE="${PREDICTOR_SCHEDULE:-before-model}" +# The qualified 0731 launcher disables caches for cold throughput benchmarks. +# Tool-using agent loops need turn-boundary reuse; this later CLI flag wins +# without modifying the qualified model/DSpark arguments. +PREFIX_CACHE_SLOTS_OVERRIDE="${PREFIX_CACHE_SLOTS_OVERRIDE:-32}" + +cache_args=() +if [[ -n "${PREFIX_CACHE_SLOTS_OVERRIDE}" ]]; then + [[ "${PREFIX_CACHE_SLOTS_OVERRIDE}" =~ ^[1-9][0-9]*$ ]] || { + printf 'invalid PREFIX_CACHE_SLOTS_OVERRIDE: %s\n' \ + "${PREFIX_CACHE_SLOTS_OVERRIDE}" >&2 + exit 2 + } + (( PREFIX_CACHE_SLOTS_OVERRIDE <= 64 )) || { + printf 'PREFIX_CACHE_SLOTS_OVERRIDE exceeds the 64-slot engine limit\n' >&2 + exit 2 + } + cache_args+=(--prefix-cache-slots "${PREFIX_CACHE_SLOTS_OVERRIDE}") +fi + +for required in \ + "${CANDIDATE_BUILD}/dflash_server" \ + "${PREDICTOR_IPC_BIN}" \ + "${PREDICTOR_MODEL}"; do + [[ -e "${required}" ]] || { + printf 'missing Qwen tool-predictor path: %s\n' "${required}" >&2 + exit 2 + } +done + +export LD_LIBRARY_PATH="${CANDIDATE_BUILD}/deps/llama.cpp/ggml/src:${CANDIDATE_BUILD}/deps/llama.cpp/ggml/src/ggml-hip:${LD_LIBRARY_PATH:-}" +export LUCE_MMVQ_MAX_NCOLS=5 + +exec "${CANDIDATE_BUILD}/dflash_server" "$@" \ + --tool-hint-native-model "${PREDICTOR_MODEL}" \ + --tool-hint-native-ipc-bin "${PREDICTOR_IPC_BIN}" \ + --tool-hint-native-gpu "${PREDICTOR_GPU}" \ + --tool-hint-native-max-ctx "${PREDICTOR_MAX_CTX}" \ + --tool-hint-sidecar-max-tokens "${PREDICTOR_MAX_TOKENS}" \ + --tool-hint-native-schedule "${PREDICTOR_SCHEDULE}" \ + --tool-hint-execution-confidence "${PREDICTOR_CONFIDENCE}" \ + "${cache_args[@]}" diff --git a/optimizations/ooo_spec_lucebox5_cpu/results/trace-compiled-engine-qwen-production-6pairs-compact.json b/optimizations/ooo_spec_lucebox5_cpu/results/trace-compiled-engine-qwen-production-6pairs-compact.json new file mode 100644 index 000000000..3031e16c2 --- /dev/null +++ b/optimizations/ooo_spec_lucebox5_cpu/results/trace-compiled-engine-qwen-production-6pairs-compact.json @@ -0,0 +1,872 @@ +{ + "feature": "no-training trace-compiled speculative tool graphs", + "host": "lucebox5", + "methodology": { + "argument_binding": "the harness binds validated structured inputs to a request-scoped workflow_ref before either model runs", + "arm_order": "randomized per task", + "compiled": "one DS4+DSpark macro authorization; independent branches execute concurrently on the Strix CPU lane", + "measured_wall_time": "request through all tools and exact final answer", + "model_seed": 814, + "oracle_prediction": false, + "speculative": "Qwen predicts the trace-derived macro through the engine; its CPU graph overlaps DS4+DSpark and commits only on an exact call match", + "stage_batched": "DS4+DSpark sees only the currently-ready typed batch, authorizes one per dependency stage, runs its calls concurrently, and receives a compact rolling state instead of replaying old tool history", + "warmup_tasks": 1 + }, + "pairs": [ + { + "arm_order": [ + "compiled", + "stage_batched", + "speculative" + ], + "compiled": { + "all_ds4_active": true, + "completion_tokens": 50, + "decode_ms": 2097.5, + "exposed_tool_wait_ms": 10124.96868299786, + "final": { + "accept_rate": 0.9166666865348816, + "completion_tokens": 14, + "content_sha256": "cfbe812f379c62a39fb1047ae84da4e1c121dc72dc0bcadda519322908c0d43e" + }, + "final_correct": true, + "graph_wall_ms": 10126.189057999, + "macro_call": { + "arguments": { + "workflow_ref": "workflow_taska" + }, + "name": "execute_customer_workflows" + }, + "macro_correct": true, + "model_compute_ms": 13641.900000000001, + "model_turns": 2, + "prediction_hit": false, + "prediction_reason": null, + "prediction_source": null, + "prediction_status": null, + "predictor_ms": 0.0, + "task_ms": 23787.2067290009, + "tool_results_count": 10, + "tool_results_sha256": "eb2ed6a561ebc1e15c6f1a5226fb751de351f983d52dfe47f94c23a97d8c9cb2", + "underlying_calls_count": 10, + "underlying_calls_sha256": "b17c72ad474ba5d0a431bd35335fcb839c0a329504f6eed9eda5d211249e6841" + }, + "pair_index": 0, + "speculative": { + "all_ds4_active": true, + "completion_tokens": 50, + "decode_ms": 2087.8, + "exposed_tool_wait_ms": 0.019826, + "final": { + "accept_rate": 0.9166666865348816, + "completion_tokens": 14, + "content_sha256": "cfbe812f379c62a39fb1047ae84da4e1c121dc72dc0bcadda519322908c0d43e" + }, + "final_correct": true, + "graph_wall_ms": 10002.203055999416, + "macro_call": { + "arguments": { + "workflow_ref": "workflow_taska" + }, + "name": "execute_customer_workflows" + }, + "macro_correct": true, + "model_compute_ms": 13568.7, + "model_turns": 2, + "prediction_hit": true, + "prediction_reason": null, + "prediction_source": "native-qwen3", + "prediction_status": "hit", + "predictor_ms": 198.696329, + "task_ms": 13771.538261993555, + "tool_results_count": 10, + "tool_results_sha256": "eb2ed6a561ebc1e15c6f1a5226fb751de351f983d52dfe47f94c23a97d8c9cb2", + "underlying_calls_count": 10, + "underlying_calls_sha256": "b17c72ad474ba5d0a431bd35335fcb839c0a329504f6eed9eda5d211249e6841" + }, + "stage_batched": { + "all_ds4_active": true, + "completion_tokens": 204, + "decode_ms": 8934.7, + "exposed_tool_wait_ms": 10135.228522005491, + "final": { + "accept_rate": 0.9166666865348816, + "completion_tokens": 14, + "content_sha256": "cfbe812f379c62a39fb1047ae84da4e1c121dc72dc0bcadda519322908c0d43e" + }, + "final_correct": true, + "model_compute_ms": 67834.7, + "model_turns": 6, + "task_ms": 77984.87404600019, + "tool_results_count": 10, + "tool_results_sha256": "eb2ed6a561ebc1e15c6f1a5226fb751de351f983d52dfe47f94c23a97d8c9cb2", + "underlying_calls_count": 10, + "underlying_calls_sha256": "b17c72ad474ba5d0a431bd35335fcb839c0a329504f6eed9eda5d211249e6841" + }, + "task": { + "branch_count": 2, + "call_count": 10, + "id": "trace_compiled_000" + } + }, + { + "arm_order": [ + "compiled", + "speculative", + "stage_batched" + ], + "compiled": { + "all_ds4_active": true, + "completion_tokens": 51, + "decode_ms": 2239.5, + "exposed_tool_wait_ms": 10143.1347070029, + "final": { + "accept_rate": 0.6875, + "completion_tokens": 15, + "content_sha256": "e9618263bdfa8c2639cf7326931749b0625a0f5ca309f638c95ad621c5809303" + }, + "final_correct": true, + "graph_wall_ms": 10144.398914002522, + "macro_call": { + "arguments": { + "workflow_ref": "workflow_taskb" + }, + "name": "execute_customer_workflows" + }, + "macro_correct": true, + "model_compute_ms": 14251.7, + "model_turns": 2, + "prediction_hit": false, + "prediction_reason": null, + "prediction_source": null, + "prediction_status": null, + "predictor_ms": 0.0, + "task_ms": 24427.28014900058, + "tool_results_count": 15, + "tool_results_sha256": "2037fc8f56106ba9bc50763cc4b82c0f3262f0ffb8682ca5ca727eaf9d46b07b", + "underlying_calls_count": 15, + "underlying_calls_sha256": "2263e8d373d489cb82f03725feec381fb4ec218f6a706ebff2de102ed1e7c40b" + }, + "pair_index": 1, + "speculative": { + "all_ds4_active": true, + "completion_tokens": 51, + "decode_ms": 2241.1, + "exposed_tool_wait_ms": 0.035275, + "final": { + "accept_rate": 0.6875, + "completion_tokens": 15, + "content_sha256": "e9618263bdfa8c2639cf7326931749b0625a0f5ca309f638c95ad621c5809303" + }, + "final_correct": true, + "graph_wall_ms": 10001.984403999813, + "macro_call": { + "arguments": { + "workflow_ref": "workflow_taskb" + }, + "name": "execute_customer_workflows" + }, + "macro_correct": true, + "model_compute_ms": 14184.8, + "model_turns": 2, + "prediction_hit": true, + "prediction_reason": null, + "prediction_source": "native-qwen3", + "prediction_status": "hit", + "predictor_ms": 201.174369, + "task_ms": 14390.547144001175, + "tool_results_count": 15, + "tool_results_sha256": "2037fc8f56106ba9bc50763cc4b82c0f3262f0ffb8682ca5ca727eaf9d46b07b", + "underlying_calls_count": 15, + "underlying_calls_sha256": "2263e8d373d489cb82f03725feec381fb4ec218f6a706ebff2de102ed1e7c40b" + }, + "stage_batched": { + "all_ds4_active": true, + "completion_tokens": 210, + "decode_ms": 9601.1, + "exposed_tool_wait_ms": 10159.580159001052, + "final": { + "accept_rate": 0.6875, + "completion_tokens": 15, + "content_sha256": "e9618263bdfa8c2639cf7326931749b0625a0f5ca309f638c95ad621c5809303" + }, + "final_correct": true, + "model_compute_ms": 70723.3, + "model_turns": 6, + "task_ms": 80902.57541000028, + "tool_results_count": 15, + "tool_results_sha256": "2037fc8f56106ba9bc50763cc4b82c0f3262f0ffb8682ca5ca727eaf9d46b07b", + "underlying_calls_count": 15, + "underlying_calls_sha256": "2263e8d373d489cb82f03725feec381fb4ec218f6a706ebff2de102ed1e7c40b" + }, + "task": { + "branch_count": 3, + "call_count": 15, + "id": "trace_compiled_001" + } + }, + { + "arm_order": [ + "compiled", + "stage_batched", + "speculative" + ], + "compiled": { + "all_ds4_active": true, + "completion_tokens": 55, + "decode_ms": 2136.2, + "exposed_tool_wait_ms": 10164.059007001924, + "final": { + "accept_rate": 0.9375, + "completion_tokens": 19, + "content_sha256": "77d293cd086d290a81c7ba651034f8d95d944b3305f9956ae1dd9dae33671953" + }, + "final_correct": true, + "graph_wall_ms": 10165.094661002513, + "macro_call": { + "arguments": { + "workflow_ref": "workflow_taskc" + }, + "name": "execute_customer_workflows" + }, + "macro_correct": true, + "model_compute_ms": 14691.8, + "model_turns": 2, + "prediction_hit": false, + "prediction_reason": null, + "prediction_source": null, + "prediction_status": null, + "predictor_ms": 0.0, + "task_ms": 24958.613470000273, + "tool_results_count": 20, + "tool_results_sha256": "8eb88b665ebcafe693c031c44e5c724a2f3aebe00f5e726008dc316050fa211c", + "underlying_calls_count": 20, + "underlying_calls_sha256": "467ec65a34e98b78d3d4a6990fef3a6292140cb019e9ec1635fd7b5c7aa9b3ef" + }, + "pair_index": 2, + "speculative": { + "all_ds4_active": true, + "completion_tokens": 55, + "decode_ms": 2135.2, + "exposed_tool_wait_ms": 0.03236, + "final": { + "accept_rate": 0.9375, + "completion_tokens": 19, + "content_sha256": "77d293cd086d290a81c7ba651034f8d95d944b3305f9956ae1dd9dae33671953" + }, + "final_correct": true, + "graph_wall_ms": 10002.267649004352, + "macro_call": { + "arguments": { + "workflow_ref": "workflow_taskc" + }, + "name": "execute_customer_workflows" + }, + "macro_correct": true, + "model_compute_ms": 14628.900000000001, + "model_turns": 2, + "prediction_hit": true, + "prediction_reason": null, + "prediction_source": "native-qwen3", + "prediction_status": "hit", + "predictor_ms": 205.821132, + "task_ms": 14839.366328000324, + "tool_results_count": 20, + "tool_results_sha256": "8eb88b665ebcafe693c031c44e5c724a2f3aebe00f5e726008dc316050fa211c", + "underlying_calls_count": 20, + "underlying_calls_sha256": "467ec65a34e98b78d3d4a6990fef3a6292140cb019e9ec1635fd7b5c7aa9b3ef" + }, + "stage_batched": { + "all_ds4_active": true, + "completion_tokens": 211, + "decode_ms": 9963.0, + "exposed_tool_wait_ms": 10172.528195005725, + "final": { + "accept_rate": 0.9375, + "completion_tokens": 19, + "content_sha256": "77d293cd086d290a81c7ba651034f8d95d944b3305f9956ae1dd9dae33671953" + }, + "final_correct": true, + "model_compute_ms": 73757.7, + "model_turns": 6, + "task_ms": 83947.50960399688, + "tool_results_count": 20, + "tool_results_sha256": "8eb88b665ebcafe693c031c44e5c724a2f3aebe00f5e726008dc316050fa211c", + "underlying_calls_count": 20, + "underlying_calls_sha256": "467ec65a34e98b78d3d4a6990fef3a6292140cb019e9ec1635fd7b5c7aa9b3ef" + }, + "task": { + "branch_count": 4, + "call_count": 20, + "id": "trace_compiled_002" + } + }, + { + "arm_order": [ + "speculative", + "stage_batched", + "compiled" + ], + "compiled": { + "all_ds4_active": true, + "completion_tokens": 50, + "decode_ms": 2561.7, + "exposed_tool_wait_ms": 10127.542959999118, + "final": { + "accept_rate": 0.9166666865348816, + "completion_tokens": 14, + "content_sha256": "b8f612f1f27ff9b32b2fb5d747b5b09e9ff0de8e4241ea1810893b522b673906" + }, + "final_correct": true, + "graph_wall_ms": 10128.030801002751, + "macro_call": { + "arguments": { + "workflow_ref": "workflow_taskd" + }, + "name": "execute_customer_workflows" + }, + "macro_correct": true, + "model_compute_ms": 14129.5, + "model_turns": 2, + "prediction_hit": false, + "prediction_reason": null, + "prediction_source": null, + "prediction_status": null, + "predictor_ms": 0.0, + "task_ms": 24262.652850004088, + "tool_results_count": 10, + "tool_results_sha256": "c2041b2f6e3aeddaafa6ffb35bcd8affbc01e75cc62de4258206a53a6559a771", + "underlying_calls_count": 10, + "underlying_calls_sha256": "ff04b3b23c0147caffa852e474ed9f9fec517c227e12674dfacdba771bf34e33" + }, + "pair_index": 3, + "speculative": { + "all_ds4_active": true, + "completion_tokens": 50, + "decode_ms": 2557.7, + "exposed_tool_wait_ms": 0.020318, + "final": { + "accept_rate": 0.9166666865348816, + "completion_tokens": 14, + "content_sha256": "b8f612f1f27ff9b32b2fb5d747b5b09e9ff0de8e4241ea1810893b522b673906" + }, + "final_correct": true, + "graph_wall_ms": 10002.225474003353, + "macro_call": { + "arguments": { + "workflow_ref": "workflow_taskd" + }, + "name": "execute_customer_workflows" + }, + "macro_correct": true, + "model_compute_ms": 14066.4, + "model_turns": 2, + "prediction_hit": true, + "prediction_reason": null, + "prediction_source": "native-qwen3", + "prediction_status": "hit", + "predictor_ms": 198.264642, + "task_ms": 14328.593565005576, + "tool_results_count": 10, + "tool_results_sha256": "c2041b2f6e3aeddaafa6ffb35bcd8affbc01e75cc62de4258206a53a6559a771", + "underlying_calls_count": 10, + "underlying_calls_sha256": "ff04b3b23c0147caffa852e474ed9f9fec517c227e12674dfacdba771bf34e33" + }, + "stage_batched": { + "all_ds4_active": true, + "completion_tokens": 204, + "decode_ms": 8685.0, + "exposed_tool_wait_ms": 10141.129875002662, + "final": { + "accept_rate": 0.9166666865348816, + "completion_tokens": 14, + "content_sha256": "b8f612f1f27ff9b32b2fb5d747b5b09e9ff0de8e4241ea1810893b522b673906" + }, + "final_correct": true, + "model_compute_ms": 67694.4, + "model_turns": 6, + "task_ms": 77853.58790800092, + "tool_results_count": 10, + "tool_results_sha256": "c2041b2f6e3aeddaafa6ffb35bcd8affbc01e75cc62de4258206a53a6559a771", + "underlying_calls_count": 10, + "underlying_calls_sha256": "ff04b3b23c0147caffa852e474ed9f9fec517c227e12674dfacdba771bf34e33" + }, + "task": { + "branch_count": 2, + "call_count": 10, + "id": "trace_compiled_003" + } + }, + { + "arm_order": [ + "speculative", + "stage_batched", + "compiled" + ], + "compiled": { + "all_ds4_active": true, + "completion_tokens": 54, + "decode_ms": 2277.8, + "exposed_tool_wait_ms": 10143.549353000708, + "final": { + "accept_rate": 0.8125, + "completion_tokens": 17, + "content_sha256": "781a9894a33fb9de5e33c892e322d4181cd9a5838ea34bfacb5bcd6dda59e9e4" + }, + "final_correct": true, + "graph_wall_ms": 10144.438636001723, + "macro_call": { + "arguments": { + "workflow_ref": "workflow_taske" + }, + "name": "execute_customer_workflows" + }, + "macro_correct": true, + "model_compute_ms": 14573.5, + "model_turns": 2, + "prediction_hit": false, + "prediction_reason": null, + "prediction_source": null, + "prediction_status": null, + "predictor_ms": 0.0, + "task_ms": 24722.972717005177, + "tool_results_count": 15, + "tool_results_sha256": "968113db6b1f63c186b8124472fb64ac38b8d81957a7d9a05898a41eb23af577", + "underlying_calls_count": 15, + "underlying_calls_sha256": "0be2134f957d10c601ed26e39380751ede7d2b05ec5419d36680e9f981c2ba57" + }, + "pair_index": 4, + "speculative": { + "all_ds4_active": true, + "completion_tokens": 54, + "decode_ms": 2283.9, + "exposed_tool_wait_ms": 0.021269, + "final": { + "accept_rate": 0.8125, + "completion_tokens": 17, + "content_sha256": "781a9894a33fb9de5e33c892e322d4181cd9a5838ea34bfacb5bcd6dda59e9e4" + }, + "final_correct": true, + "graph_wall_ms": 10001.9813550025, + "macro_call": { + "arguments": { + "workflow_ref": "workflow_taske" + }, + "name": "execute_customer_workflows" + }, + "macro_correct": true, + "model_compute_ms": 14529.800000000001, + "model_turns": 2, + "prediction_hit": true, + "prediction_reason": null, + "prediction_source": "native-qwen3", + "prediction_status": "hit", + "predictor_ms": 244.327376, + "task_ms": 14804.260248994979, + "tool_results_count": 15, + "tool_results_sha256": "968113db6b1f63c186b8124472fb64ac38b8d81957a7d9a05898a41eb23af577", + "underlying_calls_count": 15, + "underlying_calls_sha256": "0be2134f957d10c601ed26e39380751ede7d2b05ec5419d36680e9f981c2ba57" + }, + "stage_batched": { + "all_ds4_active": true, + "completion_tokens": 213, + "decode_ms": 8830.1, + "exposed_tool_wait_ms": 10159.158977992774, + "final": { + "accept_rate": 0.8125, + "completion_tokens": 17, + "content_sha256": "781a9894a33fb9de5e33c892e322d4181cd9a5838ea34bfacb5bcd6dda59e9e4" + }, + "final_correct": true, + "model_compute_ms": 70980.1, + "model_turns": 6, + "task_ms": 81157.67812900594, + "tool_results_count": 15, + "tool_results_sha256": "968113db6b1f63c186b8124472fb64ac38b8d81957a7d9a05898a41eb23af577", + "underlying_calls_count": 15, + "underlying_calls_sha256": "0be2134f957d10c601ed26e39380751ede7d2b05ec5419d36680e9f981c2ba57" + }, + "task": { + "branch_count": 3, + "call_count": 15, + "id": "trace_compiled_004" + } + }, + { + "arm_order": [ + "stage_batched", + "compiled", + "speculative" + ], + "compiled": { + "all_ds4_active": true, + "completion_tokens": 46, + "decode_ms": 3468.2, + "exposed_tool_wait_ms": 10165.250458005175, + "final": { + "accept_rate": 0.5833333134651184, + "completion_tokens": 10, + "content_sha256": "4323774d53ee50de45337a0ce501766a1b32b5398ac8d64044006ed6f5143ef7" + }, + "final_correct": true, + "graph_wall_ms": 10166.563869002857, + "macro_call": { + "arguments": { + "workflow_ref": "workflow_taskf" + }, + "name": "execute_customer_workflows" + }, + "macro_correct": true, + "model_compute_ms": 16199.5, + "model_turns": 2, + "prediction_hit": false, + "prediction_reason": null, + "prediction_source": null, + "prediction_status": null, + "predictor_ms": 0.0, + "task_ms": 26370.94549000176, + "tool_results_count": 20, + "tool_results_sha256": "de71e45a390fca728e07a8bf4cdc467e91604fc52f68abe6ed0bd75ca77e87d2", + "underlying_calls_count": 20, + "underlying_calls_sha256": "bd91a2316a76490265d64557860d9a2acdc97a8f1a18014dacaa0a21cd241e2f" + }, + "pair_index": 5, + "speculative": { + "all_ds4_active": true, + "completion_tokens": 46, + "decode_ms": 2451.7, + "exposed_tool_wait_ms": 0.032681, + "final": { + "accept_rate": 0.5833333134651184, + "completion_tokens": 10, + "content_sha256": "4323774d53ee50de45337a0ce501766a1b32b5398ac8d64044006ed6f5143ef7" + }, + "final_correct": true, + "graph_wall_ms": 10002.300546002516, + "macro_call": { + "arguments": { + "workflow_ref": "workflow_taskf" + }, + "name": "execute_customer_workflows" + }, + "macro_correct": true, + "model_compute_ms": 15157.8, + "model_turns": 2, + "prediction_hit": true, + "prediction_reason": null, + "prediction_source": "native-qwen3", + "prediction_status": "hit", + "predictor_ms": 214.897278, + "task_ms": 15378.521895996528, + "tool_results_count": 20, + "tool_results_sha256": "de71e45a390fca728e07a8bf4cdc467e91604fc52f68abe6ed0bd75ca77e87d2", + "underlying_calls_count": 20, + "underlying_calls_sha256": "bd91a2316a76490265d64557860d9a2acdc97a8f1a18014dacaa0a21cd241e2f" + }, + "stage_batched": { + "all_ds4_active": true, + "completion_tokens": 191, + "decode_ms": 11077.9, + "exposed_tool_wait_ms": 10172.786328992515, + "final": { + "accept_rate": 0.5833333134651184, + "completion_tokens": 10, + "content_sha256": "4323774d53ee50de45337a0ce501766a1b32b5398ac8d64044006ed6f5143ef7" + }, + "final_correct": true, + "model_compute_ms": 75437.0, + "model_turns": 6, + "task_ms": 85663.01394799666, + "tool_results_count": 20, + "tool_results_sha256": "de71e45a390fca728e07a8bf4cdc467e91604fc52f68abe6ed0bd75ca77e87d2", + "underlying_calls_count": 20, + "underlying_calls_sha256": "bd91a2316a76490265d64557860d9a2acdc97a8f1a18014dacaa0a21cd241e2f" + }, + "task": { + "branch_count": 4, + "call_count": 20, + "id": "trace_compiled_005" + } + } + ], + "pattern": { + "fingerprint": "06d95882b485daf3f30f3fc12ea62ccfd18656de9deafa2f4ed77c49bb8c0645", + "macro_name": "execute_customer_workflows", + "model_training": false, + "root_fields": [ + "customer_email", + "destination" + ], + "side_effects_allowed": false, + "steps": [ + { + "arguments": { + "customer_email": { + "key": "customer_email", + "source": "root" + } + }, + "tool": "resolve_customer" + }, + { + "arguments": { + "customer_ref": { + "key": "call_ref", + "source": "previous_result" + } + }, + "tool": "list_open_orders" + }, + { + "arguments": { + "orders_ref": { + "key": "call_ref", + "source": "previous_result" + } + }, + "tool": "get_order_details" + }, + { + "arguments": { + "destination": { + "key": "destination", + "source": "root" + }, + "order_ref": { + "key": "call_ref", + "source": "previous_result" + } + }, + "tool": "calculate_shipping" + }, + { + "arguments": { + "shipping_ref": { + "key": "call_ref", + "source": "previous_result" + } + }, + "tool": "prepare_customer_summary" + } + ], + "training_report": "/home/lucebox5/tool-spec-cpu-20260813/results/multiturn-cached-wordref-production-6tasks.json", + "training_report_sha256": "2475697d418bffed0e9668da26ce6c88a85a952ce97d99749f440f97f9ac5bf9", + "training_traces": 2, + "workflow_registry": "/home/lucebox5/tool-spec-cpu-20260813/results/trace-workflow-registry.json", + "workflow_registry_sha256": "0b73cb4f45284f07a4d6890906d1eaf5a25469c2b80eb542b1d74f1e3a9a0e1b" + }, + "production_gate": { + "checks": { + "calls_stable": true, + "decode_slowdown_p50": true, + "decode_slowdown_p95": true, + "ds4_active": true, + "end_to_end_ci": true, + "end_to_end_speedup": true, + "end_to_end_tail": true, + "final_answers_correct": true, + "final_outputs_stable": true, + "macro_calls_correct": true, + "macro_outputs_stable": true, + "model_slowdown_p50": true, + "model_slowdown_p95": true, + "prediction_hit_rate": true, + "prediction_source": true, + "prefix_cache_configured": true, + "sample_size": true, + "speculation_incremental_ci": true, + "speculation_incremental_gain": true, + "tool_results_stable": true + }, + "passed": true, + "thresholds": { + "max_decode_slowdown_p95_percent": 5.0, + "max_decode_slowdown_percent": 1.0, + "max_model_slowdown_p95_percent": 5.0, + "max_model_slowdown_percent": 1.0, + "min_e2e_speedup": 2.0, + "min_e2e_speedup_p05": 1.5, + "min_incremental_speedup": 1.05, + "min_production_pairs": 6 + } + }, + "schema_version": 1, + "server_snapshot": { + "model": { + "arch": "deepseek4", + "draft_path": null, + "tokenizer_id": null + }, + "prefix_cache_after": { + "capacity": 32, + "in_use": 0, + "lifetime_hits": 0 + }, + "prefix_cache_before": { + "capacity": 32, + "in_use": 0, + "lifetime_hits": 0 + }, + "tool_speculation": { + "allowed_tools": [ + "calculate_shipping", + "execute_customer_workflows", + "get_order_details", + "list_open_orders", + "prepare_customer_summary", + "resolve_customer" + ], + "automatic_prediction_enabled": true, + "compute_isolation": "disjoint_cpu_affinity", + "cpu_affinity_isolated": true, + "enabled": true, + "execution_mode": "child_process_cpu_affinity", + "executor_contract": "child_process_cpu_affinity", + "hip_reserved_tool_compute_units": 0, + "hip_tool_device": null, + "max_model_slowdown_ratio": 1.05, + "model_cpu_affinity": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29 + ], + "model_expert_ownership_unique": false, + "model_routing_static": false, + "prediction_confidence": 0.75, + "prediction_source": "native-qwen3", + "predictor_decode_isolated": true, + "predictor_schedule": "before-model", + "preserves_token_speculation": true, + "profile_lanes": [ + { + "accelerator_relation": "non_accelerator", + "decode_interference_qualified": true, + "model_slowdown_ratio": 1.0011341375399527, + "requires_static_model_routing": false, + "requires_unique_expert_ownership": false, + "resource_percentage": 100 + } + ], + "profile_status": "qualified", + "protocol": "dflash.tool-speculation.v1", + "requires_client_support": false, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "unqualified_lane_policy": "defer" + } + }, + "summary": { + "all_calls_stable": true, + "all_ds4_active": true, + "all_final_answers_correct": true, + "all_final_outputs_stable": true, + "all_macro_calls_correct": true, + "all_predictions_from_qwen": true, + "all_tool_results_stable": true, + "calls_per_task": [ + 10, + 15, + 20, + 10, + 15, + 20 + ], + "compiled_exposed_tool_wait_p50_ms": 10143.342030001804, + "compiled_model_turns_p50": 2.0, + "compiled_task_p50_ms": 24575.12643300288, + "compiled_to_speculative_bootstrap_95ci": [ + 1.6759547511337496, + 1.7210318416877688 + ], + "compiled_to_speculative_speedup_p05": 1.6729725830674844, + "compiled_to_speculative_speedup_p50": 1.6953781778482404, + "continuation_cache_hit_rate": 0.0, + "decode_slowdown_p50_percent": -0.10147920266865285, + "decode_slowdown_p95_percent": 0.21871282872425457, + "macro_output_stability_rate": 1.0, + "model_compute_slowdown_p50_percent": -0.4580005364335671, + "model_compute_slowdown_p95_percent": -0.3319269946081671, + "pattern_prediction_hit_rate": 1.0, + "predictor_p50_ms": 203.4977505, + "prefix_cache_configured": true, + "prefix_cache_lifetime_hit_delta": 0, + "speculative_exposed_tool_wait_p50_ms": 0.026814499999999998, + "speculative_task_p50_ms": 14597.403696498077, + "speedup_by_call_count": { + "10": { + "combined_speedup_p50": 5.548099679951088, + "compiled_task_p50_ms": 24024.929789502494, + "speculation_speedup_p50": 1.7102881019726668, + "speculative_task_p50_ms": 14050.065913499566, + "stage_batched_task_p50_ms": 77919.23097700055, + "tasks": 2 + }, + "15": { + "combined_speedup_p50": 5.551986886996318, + "compiled_task_p50_ms": 24575.12643300288, + "speculation_speedup_p50": 1.683721802277213, + "speculative_task_p50_ms": 14597.403696498077, + "stage_batched_task_p50_ms": 81030.12676950311, + "tasks": 2 + }, + "20": { + "combined_speedup_p50": 5.613692001294089, + "compiled_task_p50_ms": 25664.779480001016, + "speculation_speedup_p50": 1.698354866419879, + "speculative_task_p50_ms": 15108.944111998426, + "stage_batched_task_p50_ms": 84805.26177599677, + "tasks": 2 + } + }, + "stage_batched_exposed_tool_wait_p50_ms": 10159.369568496913, + "stage_batched_model_turns_p50": 6.0, + "stage_batched_task_p50_ms": 81030.12676950311, + "stage_batched_to_compiled_speedup_p50": 3.2805602399674787, + "stage_batched_to_speculative_bootstrap_95ci": [ + 5.457745637116071, + 5.659919390842823 + ], + "stage_batched_to_speculative_speedup_min": 5.433442406946423, + "stage_batched_to_speculative_speedup_p05": 5.445594022031247, + "stage_batched_to_speculative_speedup_p50": 5.5961135402826, + "tasks": 6, + "total_wall_speedup": 5.570717496895011 + }, + "workload": { + "branches_per_task": "2-4", + "calls_per_task": "10-20", + "dependency": "five serial calls per branch; branches are independent", + "tasks": 6, + "tool_adapter": "deterministic read-only 2-second API replay" + } +} diff --git a/optimizations/ooo_spec_lucebox5_cpu/results/trace-compiled-training-traces.json b/optimizations/ooo_spec_lucebox5_cpu/results/trace-compiled-training-traces.json new file mode 100644 index 000000000..089220994 --- /dev/null +++ b/optimizations/ooo_spec_lucebox5_cpu/results/trace-compiled-training-traces.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "source": "successful read-only workflow traces", + "traces": [ + { + "root": { + "customer_email": "agent-benchmark-2@example.test", + "destination": "Turin" + }, + "calls": [ + {"name": "resolve_customer", "arguments": {"customer_email": "agent-benchmark-2@example.test"}}, + {"name": "list_open_orders", "arguments": {"customer_ref": "plum"}}, + {"name": "get_order_details", "arguments": {"orders_ref": "jade"}}, + {"name": "calculate_shipping", "arguments": {"destination": "Turin", "order_ref": "amber"}}, + {"name": "prepare_customer_summary", "arguments": {"shipping_ref": "amber"}} + ], + "results": [ + {"call_ref": "plum", "call_sha256": "93824d1968f2c3ab058a3ef69d2625c0fea88fad3b27b8c30f95418fb83ae6cf", "tool_name": "resolve_customer", "side_effects": false}, + {"call_ref": "jade", "call_sha256": "772a4b9fde9e69fb1da323ba4c22e7a7d01061c0600c133e74de7bd73bba931f", "tool_name": "list_open_orders", "side_effects": false}, + {"call_ref": "amber", "call_sha256": "428dbfc05f17e7a9295c2b4741dbae2bb91d3f107150b77bb34c6ca25bd8af23", "tool_name": "get_order_details", "side_effects": false}, + {"call_ref": "amber", "call_sha256": "21ec3c60e83bfbc0ab4e6e8356a5c87e7210ffd16b53ca8299017261d97f70a1", "tool_name": "calculate_shipping", "side_effects": false}, + {"call_ref": "ivory", "call_sha256": "adaa761561a84b7e457acb7881e856285fcdccf21a6d82751dd2bea984b1d6d0", "tool_name": "prepare_customer_summary", "side_effects": false} + ] + }, + { + "root": { + "customer_email": "agent-benchmark-5@example.test", + "destination": "Naples" + }, + "calls": [ + {"name": "resolve_customer", "arguments": {"customer_email": "agent-benchmark-5@example.test"}}, + {"name": "list_open_orders", "arguments": {"customer_ref": "maple"}}, + {"name": "get_order_details", "arguments": {"orders_ref": "willow"}}, + {"name": "calculate_shipping", "arguments": {"destination": "Naples", "order_ref": "olive"}}, + {"name": "prepare_customer_summary", "arguments": {"shipping_ref": "cedar"}} + ], + "results": [ + {"call_ref": "maple", "call_sha256": "71063336b1901dd2a0050d4a7a0ebe52ad0b0755f5af079efe22db9c1df00b29", "tool_name": "resolve_customer", "side_effects": false}, + {"call_ref": "willow", "call_sha256": "8da0147e626bec41f9bafd2ec61d07d01a33e176990483fd62a2b93eeea82e2f", "tool_name": "list_open_orders", "side_effects": false}, + {"call_ref": "olive", "call_sha256": "66ea7a270ffff62c364786d69dc72144b0b0eb928c262f682dff72635964c6a9", "tool_name": "get_order_details", "side_effects": false}, + {"call_ref": "cedar", "call_sha256": "db784ed2c9e33569d9c6bdd36e6963076032cd8d0bf48cf954420a843cb8d593", "tool_name": "calculate_shipping", "side_effects": false}, + {"call_ref": "amber", "call_sha256": "6ba7d6c0326fbe23206f51a9a4268beb41a3050097565421946c4ac32900f9bc", "tool_name": "prepare_customer_summary", "side_effects": false} + ] + } + ] +} diff --git a/optimizations/ooo_spec_lucebox5_cpu/run_native_cpu_server_lucebox5.sh b/optimizations/ooo_spec_lucebox5_cpu/run_native_cpu_server_lucebox5.sh index 7fdae08ed..2e067aab9 100755 --- a/optimizations/ooo_spec_lucebox5_cpu/run_native_cpu_server_lucebox5.sh +++ b/optimizations/ooo_spec_lucebox5_cpu/run_native_cpu_server_lucebox5.sh @@ -3,11 +3,22 @@ set -euo pipefail root="/home/lucebox5" experiment="$root/tool-spec-cpu-20260813" -launcher="$experiment/run-deepseek-0731-cpu-tool.sh" -executor="$experiment/cpu_sparse_tool_executor" -profile="$experiment/profiles/lucebox5-cpu-lane-qualified.json" +launcher="${LUCEBOX_LAUNCHER:-$experiment/run-deepseek-0731-cpu-tool.sh}" +executor="${TOOL_SPEC_EXECUTOR:-$experiment/cpu_sparse_tool_executor}" +profile="${TOOL_SPEC_PROFILE:-$experiment/profiles/lucebox5-cpu-lane-qualified.json}" +allowed="${TOOL_SPEC_ALLOW:-benchmark_cpu_sparse}" +native_wrapper_dir="${NATIVE_WRAPPER_DIR:-$experiment/native-wrapper}" +candidate_build="${CANDIDATE_BUILD:-$experiment/engine-ooo-spec/server/build-hip-dual}" +predictor_model="${PREDICTOR_MODEL:-$experiment/models/Qwen3-0.6B-Q8_0.gguf}" -for required in "$launcher" "$executor" "$profile"; do +for required in \ + "$launcher" \ + "$executor" \ + "$profile" \ + "$native_wrapper_dir/dflash_server" \ + "$candidate_build/dflash_server" \ + "$candidate_build/backend_ipc_daemon" \ + "$predictor_model"; do [[ -e "$required" ]] || { printf 'missing required path: %s\n' "$required" >&2 exit 2 @@ -28,7 +39,15 @@ exec env \ USER="lucebox5" \ PATH="$root/.local/bin:/opt/rocm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \ ENGINE_DIR="$root/lucebox-engine-0731" \ - BUILD_DIR="$root/codex-ds4-tool-spec-fix-20260812/build-tool-spec" \ + BUILD_DIR="$native_wrapper_dir" \ + CANDIDATE_BUILD="$candidate_build" \ + PREDICTOR_MODEL="$predictor_model" \ + PREDICTOR_GPU="${PREDICTOR_GPU:-1}" \ + PREDICTOR_MAX_CTX="${PREDICTOR_MAX_CTX:-4096}" \ + PREDICTOR_MAX_TOKENS="${PREDICTOR_MAX_TOKENS:-256}" \ + PREDICTOR_CONFIDENCE="${PREDICTOR_CONFIDENCE:-0.75}" \ + PREDICTOR_SCHEDULE="${PREDICTOR_SCHEDULE:-before-model}" \ + PREFIX_CACHE_SLOTS_OVERRIDE="${PREFIX_CACHE_SLOTS_OVERRIDE:-}" \ QUALIFIED_CONFIG_DIR="/opt/lucebox-manage/qualified/r9700_deepseek/runtime-config" \ TARGET_MODEL="$root/lucebox-models/DeepSeek-V4-Flash-0731-ROCMFPX-MIX-STRIX.gguf" \ DRAFT_MODEL="$root/lucebox-models/DeepSeek-V4-Flash-0731-DSpark-draft-Q4RMFP4-denseF16.gguf" \ @@ -36,7 +55,7 @@ exec env \ MODEL_CPU_AFFINITY="0-13,16-29" \ TOOL_SPEC_EXECUTOR="$executor" \ TOOL_SPEC_PROFILE="$profile" \ - TOOL_SPEC_ALLOW="benchmark_cpu_sparse" \ + TOOL_SPEC_ALLOW="$allowed" \ TOOL_SPEC_CPU_AFFINITY="14-15,30-31" \ TOOL_SPEC_MAX_MODEL_SLOWDOWN="1.05" \ LUCEBOX_INFERENCE_PROFILE="quality" \ diff --git a/optimizations/ooo_spec_lucebox5_cpu/test_benchmark_cpu_tool_speculation.py b/optimizations/ooo_spec_lucebox5_cpu/test_benchmark_cpu_tool_speculation.py index cf8df1a45..8e03e399b 100644 --- a/optimizations/ooo_spec_lucebox5_cpu/test_benchmark_cpu_tool_speculation.py +++ b/optimizations/ooo_spec_lucebox5_cpu/test_benchmark_cpu_tool_speculation.py @@ -6,7 +6,9 @@ from benchmark_cpu_tool_speculation import ( TOOL_NAME, expected_arguments, + normalize_tool_call, parse_cpu_list, + percentile, props_url, request_body, ) @@ -35,6 +37,46 @@ def test_props_url_uses_server_origin(self) -> None: "http://127.0.0.1:18145/props", ) + def test_percentile_interpolates_sorted_values(self) -> None: + self.assertEqual(percentile([4, 1, 3, 2], 0.0), 1.0) + self.assertEqual(percentile([4, 1, 3, 2], 0.5), 2.5) + self.assertEqual(percentile([4, 1, 3, 2], 1.0), 4.0) + + def test_automatic_qwen_arm_has_no_oracle_prediction(self) -> None: + arguments = {"iterations": 77} + body = request_body( + arguments, + 32, + prediction=None, + automatic_prediction=True, + tool_choice="required", + ) + self.assertNotIn("tool_speculation", body) + self.assertTrue(body["automatic_tool_speculation"]) + self.assertEqual(body["tool_choice"], "required") + + def test_normalizes_deepseek_single_parameter_envelope(self) -> None: + result = { + "choices": [ + { + "message": { + "content": ( + '{"function":"batch_resolve_customer",' + '"parameter":"stage_ref",' + '"parameter_value":"workflow_taskf_stage_one"}' + ) + } + } + ] + } + self.assertEqual( + normalize_tool_call(result), + { + "name": "batch_resolve_customer", + "arguments": {"stage_ref": "workflow_taskf_stage_one"}, + }, + ) + if __name__ == "__main__": unittest.main() diff --git a/optimizations/ooo_spec_lucebox5_cpu/test_benchmark_trace_compiled_workflows.py b/optimizations/ooo_spec_lucebox5_cpu/test_benchmark_trace_compiled_workflows.py new file mode 100644 index 000000000..7f4ff69a2 --- /dev/null +++ b/optimizations/ooo_spec_lucebox5_cpu/test_benchmark_trace_compiled_workflows.py @@ -0,0 +1,400 @@ +from __future__ import annotations + +import argparse +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from benchmark_trace_compiled_workflows import ( + alphabetic_identifier, + compact_arm, + final_answer_correct, + load_training_traces, + load_partial_pairs, + make_task, + mine_pattern, + model_observation, + parse_request_customers, + post_final, + production_checks, + simulated_tool_result, + stage_batch_tool, + stage_batched_messages, + stage_reference, + workflow_reference, +) + + +def workflow_trace(email: str, destination: str) -> dict: + root = {"customer_email": email, "destination": destination} + calls = [] + results = [] + + def add(name: str, arguments: dict) -> None: + call = {"name": name, "arguments": arguments} + calls.append(call) + results.append(simulated_tool_result(call)) + + add("resolve_customer", {"customer_email": email}) + add("list_open_orders", {"customer_ref": results[-1]["call_ref"]}) + add("get_order_details", {"orders_ref": results[-1]["call_ref"]}) + add( + "calculate_shipping", + {"order_ref": results[-1]["call_ref"], "destination": destination}, + ) + add("prepare_customer_summary", {"shipping_ref": results[-1]["call_ref"]}) + return {"root": root, "calls": calls, "results": results} + + +class TraceCompiledWorkflowBenchmarkTest(unittest.TestCase): + def setUp(self) -> None: + self.pattern = mine_pattern( + [ + workflow_trace("first@example.test", "Rome"), + workflow_trace("second@example.test", "Milan"), + ] + ) + + def test_mines_control_flow_and_late_bound_arguments(self) -> None: + self.assertEqual(self.pattern.training_traces, 2) + self.assertEqual( + [step.tool for step in self.pattern.steps], + [ + "resolve_customer", + "list_open_orders", + "get_order_details", + "calculate_shipping", + "prepare_customer_summary", + ], + ) + shipping_bindings = dict(self.pattern.steps[3].arguments) + self.assertEqual(shipping_bindings["order_ref"].source, "previous_result") + self.assertEqual(shipping_bindings["order_ref"].key, "call_ref") + self.assertEqual(shipping_bindings["destination"].source, "root") + + def test_loads_compact_training_trace_file(self) -> None: + traces = [ + workflow_trace("first@example.test", "Rome"), + workflow_trace("second@example.test", "Milan"), + ] + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "traces.json" + path.write_text(json.dumps({"traces": traces}), encoding="utf-8") + loaded = load_training_traces(path, required_steps=5) + self.assertEqual(loaded, traces) + + def test_compact_arm_keeps_evidence_but_drops_full_graph(self) -> None: + arm = { + "task_ms": 12.0, + "underlying_calls": ["call"], + "tool_results": ["digest"], + "graph": {"large": "payload"}, + "call_turns": [{"large": "payload"}], + "final": { + "content": "done", + "content_sha256": "hash", + "completion_tokens": 1, + "unused": "payload", + }, + } + compact = compact_arm(arm) + self.assertEqual(compact["underlying_calls_count"], 1) + self.assertEqual( + compact["underlying_calls_sha256"], + "4f2a91df1674ac67599f9835f2d43b0ca94e1e769f6a666ce448ae07ac1d94f7", + ) + self.assertEqual(compact["final"]["content_sha256"], "hash") + self.assertNotIn("underlying_calls", compact) + self.assertNotIn("content", compact["final"]) + self.assertNotIn("graph", compact) + self.assertNotIn("call_turns", compact) + self.assertNotIn("unused", compact["final"]) + + def test_pattern_expands_unseen_request_without_literals(self) -> None: + root = {"customer_email": "new@example.test", "destination": "Turin"} + calls = self.pattern.simulate(root) + self.assertEqual(calls[0]["arguments"], {"customer_email": root["customer_email"]}) + self.assertEqual( + calls[1]["arguments"], {"customer_ref": simulated_tool_result(calls[0])["call_ref"]} + ) + self.assertEqual(calls[3]["arguments"]["destination"], "Turin") + + def test_compiler_rejects_side_effecting_trace(self) -> None: + traces = [ + workflow_trace("first@example.test", "Rome"), + workflow_trace("second@example.test", "Milan"), + ] + traces[1]["results"][2]["side_effects"] = True + with self.assertRaisesRegex(ValueError, "side-effect-free"): + mine_pattern(traces) + + def test_compiler_rejects_literal_argument(self) -> None: + traces = [ + workflow_trace("first@example.test", "Rome"), + workflow_trace("second@example.test", "Milan"), + ] + for trace in traces: + trace["calls"][0]["arguments"]["constant"] = "not-in-request" + trace["results"][0] = simulated_tool_result(trace["calls"][0]) + with self.assertRaisesRegex(ValueError, "literal"): + mine_pattern(traces) + + def test_macro_schema_is_closed_and_typed(self) -> None: + tool = self.pattern.macro_tool(4)["function"] + parameters = tool["parameters"] + item = parameters["properties"]["customers"]["items"] + self.assertFalse(parameters["additionalProperties"]) + self.assertFalse(item["additionalProperties"]) + self.assertEqual(set(item["required"]), set(self.pattern.root_fields)) + self.assertEqual(parameters["properties"]["customers"]["maxItems"], 4) + + def test_bound_macro_uses_a_short_single_value_reference(self) -> None: + task = make_task(4, 4, self.pattern) + workflow_ref = workflow_reference(task, self.pattern) + parameters = self.pattern.macro_tool(4, workflow_ref)["function"]["parameters"] + self.assertEqual(set(parameters["properties"]), {"workflow_ref"}) + self.assertEqual( + parameters["properties"]["workflow_ref"]["enum"], [workflow_ref] + ) + self.assertEqual(workflow_ref, "workflow_taske") + + def test_generated_branches_do_not_collapse_on_short_refs(self) -> None: + task = make_task(3, 4, self.pattern) + refs_by_stage = list( + zip( + *[ + [simulated_tool_result(call)["call_ref"] for call in self.pattern.simulate(root)] + for root in task["items"] + ], + strict=True, + ) + ) + self.assertTrue(all(len(set(refs)) == 4 for refs in refs_by_stage)) + + def test_generated_identifiers_avoid_ambiguous_digits(self) -> None: + self.assertEqual(alphabetic_identifier(0), "taska") + self.assertEqual(alphabetic_identifier(26), "taskaa") + task = make_task(50, 2, self.pattern) + self.assertTrue( + all(not any(character.isdigit() for character in item["customer_email"]) + for item in task["items"]) + ) + + def test_event_extractor_recovers_macro_arguments_without_a_model(self) -> None: + content = "Customers: a@example.test to Rome; b@example.test to Milan." + self.assertEqual( + parse_request_customers(content), + [ + {"customer_email": "a@example.test", "destination": "Rome"}, + {"customer_email": "b@example.test", "destination": "Milan"}, + ], + ) + + def test_stage_batch_schema_preserves_every_call(self) -> None: + task = make_task(2, 3, self.pattern) + tool = stage_batch_tool(self.pattern, 3, 4)["function"] + calls = tool["parameters"]["properties"]["calls"] + self.assertEqual(tool["name"], "batch_calculate_shipping") + self.assertEqual(calls["maxItems"], 4) + self.assertEqual( + set(calls["items"]["required"]), {"order_ref", "destination"} + ) + self.assertIn( + "exactly one currently-ready batch tool", + stage_batched_messages(task, self.pattern)[0]["content"], + ) + + def test_bound_stage_uses_the_request_scoped_reference(self) -> None: + task = make_task(2, 4, self.pattern) + stage_ref = stage_reference(task, self.pattern, 2) + parameters = stage_batch_tool( + self.pattern, 2, 4, stage_ref + )["function"]["parameters"] + self.assertEqual(set(parameters["properties"]), {"stage_ref"}) + self.assertEqual(parameters["properties"]["stage_ref"]["enum"], [stage_ref]) + self.assertEqual(stage_ref, "workflow_taskc_stage_three") + + def test_parses_multiple_native_tool_calls(self) -> None: + response = { + "choices": [ + { + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "resolve_customer", + "arguments": '{"customer_email":"a@example.test"}', + }, + }, + { + "id": "call_2", + "type": "function", + "function": { + "name": "resolve_customer", + "arguments": {"customer_email": "b@example.test"}, + }, + }, + ], + } + } + ], + "usage": {"timings": {}}, + } + observed = model_observation(response, 12.0) + self.assertEqual(len(observed["calls"]), 2) + self.assertEqual(observed["calls"][0]["id"], "call_1") + self.assertEqual( + observed["calls"][1]["call"]["arguments"]["customer_email"], + "b@example.test", + ) + + def test_normalizes_content_format_call_for_conversation_history(self) -> None: + response = { + "choices": [ + { + "message": { + "role": "assistant", + "content": ( + '{"function":"resolve_customer","parameters":' + '{"customer_email":"a@example.test"},"type":"function"}' + ), + } + } + ], + "usage": {"timings": {}}, + } + observed = model_observation(response, 12.0) + self.assertTrue(observed["content_format_call"]) + self.assertEqual( + observed["calls"][0]["call"], + { + "name": "resolve_customer", + "arguments": {"customer_email": "a@example.test"}, + }, + ) + self.assertEqual( + observed["assistant_message"]["tool_calls"][0]["id"], + observed["calls"][0]["id"], + ) + + def test_production_gate_requires_two_x_and_exact_outputs(self) -> None: + summary = { + "tasks": 6, + "stage_batched_to_speculative_speedup_p50": 1.99, + "stage_batched_to_speculative_bootstrap_95ci": [1.5, 2.4], + "stage_batched_to_speculative_speedup_p05": 1.6, + "compiled_to_speculative_speedup_p50": 1.1, + "compiled_to_speculative_bootstrap_95ci": [1.01, 1.2], + "pattern_prediction_hit_rate": 1.0, + "all_predictions_from_qwen": True, + "model_compute_slowdown_p50_percent": 0.0, + "model_compute_slowdown_p95_percent": 0.0, + "decode_slowdown_p50_percent": 0.0, + "decode_slowdown_p95_percent": 0.0, + "continuation_cache_hit_rate": 1.0, + "prefix_cache_configured": True, + "all_calls_stable": True, + "all_tool_results_stable": True, + "macro_output_stability_rate": 1.0, + "all_final_answers_correct": True, + "all_final_outputs_stable": True, + "all_macro_calls_correct": True, + "all_ds4_active": True, + } + args = argparse.Namespace( + min_e2e_speedup=2.0, + min_e2e_speedup_p05=1.5, + min_incremental_speedup=1.05, + min_production_pairs=6, + max_model_slowdown_percent=1.0, + max_model_slowdown_p95_percent=5.0, + max_decode_slowdown_percent=1.0, + max_decode_slowdown_p95_percent=5.0, + ) + checks = production_checks(summary, args) + self.assertFalse(checks["end_to_end_speedup"]) + self.assertTrue( + all(value for key, value in checks.items() if key != "end_to_end_speedup") + ) + + def test_final_turn_is_identical_and_context_free_for_every_arm(self) -> None: + args = argparse.Namespace(final_max_tokens=32) + with patch( + "benchmark_trace_compiled_workflows.post_turn", + return_value={"content": "workflow_complete:plum"}, + ) as mocked: + post_final(args, "workflow_complete:plum") + + call_args = mocked.call_args.args + self.assertEqual(call_args[2], []) + self.assertEqual(call_args[3], "none") + self.assertEqual(call_args[4], 32) + self.assertEqual( + call_args[1][-1], + {"role": "user", "content": "workflow_complete:plum"}, + ) + + def test_final_receipt_accepts_literal_or_equivalent_json(self) -> None: + expected = "workflow_complete:plum,ivory" + self.assertTrue(final_answer_correct(expected, expected)) + self.assertTrue( + final_answer_correct('{"workflow_complete": "plum,ivory"}', expected) + ) + self.assertTrue(final_answer_correct("plum,ivory", expected)) + self.assertFalse(final_answer_correct('{"workflow_complete": "plum"}', expected)) + self.assertFalse(final_answer_correct("workflow_complete:ivory,plum", expected)) + + def test_resume_checkpoint_requires_matching_task_and_arm_order(self) -> None: + tasks = [make_task(0, 2, self.pattern), make_task(1, 3, self.pattern)] + orders = [ + ["compiled", "stage_batched", "speculative"], + ["speculative", "stage_batched", "compiled"], + ] + checkpoint = { + "schema_version": 1, + "complete": False, + "pairs": [ + { + "pair_index": 0, + "task": tasks[0], + "arm_order": orders[0], + "stage_batched": { + "final": {"content": "plum,ivory"}, + "expected_final": "workflow_complete:plum,ivory", + }, + "compiled": { + "final": {"content": "plum,ivory"}, + "expected_final": "workflow_complete:plum,ivory", + }, + "speculative": { + "final": {"content": "plum,ivory"}, + "expected_final": "workflow_complete:plum,ivory", + }, + } + ], + } + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "run.json.partial" + path.write_text(json.dumps(checkpoint), encoding="utf-8") + resumed = load_partial_pairs(path, tasks, orders) + self.assertEqual(len(resumed), 1) + self.assertTrue( + all( + resumed[0][arm]["final_correct"] + for arm in ("stage_batched", "compiled", "speculative") + ) + ) + checkpoint["pairs"][0]["arm_order"] = orders[1] + path.write_text(json.dumps(checkpoint), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "does not match"): + load_partial_pairs(path, tasks, orders) + + +if __name__ == "__main__": + unittest.main() diff --git a/optimizations/ooo_spec_lucebox5_cpu/test_trace_compiled_tool_executor.py b/optimizations/ooo_spec_lucebox5_cpu/test_trace_compiled_tool_executor.py new file mode 100644 index 000000000..9c95884bf --- /dev/null +++ b/optimizations/ooo_spec_lucebox5_cpu/test_trace_compiled_tool_executor.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import unittest + +from benchmark_trace_compiled_workflows import mine_pattern, simulated_tool_result +from test_benchmark_trace_compiled_workflows import workflow_trace +from trace_compiled_tool_executor import execute_macro, resolve_items + + +class TraceCompiledToolExecutorTest(unittest.TestCase): + def setUp(self) -> None: + self.pattern = mine_pattern( + [ + workflow_trace("first@example.test", "Rome"), + workflow_trace("second@example.test", "Milan"), + ] + ) + self.items = [ + {"customer_email": "alice@example.test", "destination": "Turin"}, + {"customer_email": "bob@example.test", "destination": "Naples"}, + ] + self.workflow_ref = "workflow_alpha" + self.registry = { + "schema_version": 1, + "pattern_fingerprint": self.pattern.fingerprint, + "workflows": { + self.workflow_ref: { + "pattern_fingerprint": self.pattern.fingerprint, + "items": self.items, + } + }, + } + + @staticmethod + def fake_leaf(request: dict) -> dict: + return {"ok": True, "result": simulated_tool_result(request["call"])} + + def test_executes_every_branch_and_preserves_order(self) -> None: + call = { + "name": self.pattern.macro_name, + "arguments": {"workflow_ref": self.workflow_ref}, + } + request = {"call": call} + envelope = execute_macro( + request, self.pattern, self.fake_leaf, self.registry + ) + result = envelope["result"] + self.assertEqual(result["call_count"], 10) + self.assertEqual([branch["root"] for branch in result["branches"]], self.items) + self.assertEqual( + [len(branch["steps"]) for branch in result["branches"]], [5, 5] + ) + self.assertFalse(result["side_effects"]) + + def test_rejects_unknown_or_missing_inputs(self) -> None: + with self.assertRaisesRegex(ValueError, "fields"): + resolve_items( + {"workflow_ref": self.workflow_ref}, + self.pattern, + { + **self.registry, + "workflows": { + self.workflow_ref: { + "pattern_fingerprint": self.pattern.fingerprint, + "items": [{**self.items[0], "undeclared": "value"}], + } + }, + }, + ) + with self.assertRaisesRegex(ValueError, "workflow_ref"): + resolve_items({"items": self.items}, self.pattern, self.registry) + + +if __name__ == "__main__": + unittest.main() diff --git a/optimizations/ooo_spec_lucebox5_cpu/trace_compiled_tool_executor.py b/optimizations/ooo_spec_lucebox5_cpu/trace_compiled_tool_executor.py new file mode 100755 index 000000000..20cd771fc --- /dev/null +++ b/optimizations/ooo_spec_lucebox5_cpu/trace_compiled_tool_executor.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +"""Execute a trace-compiled read-only workflow through the tool-spec protocol. + +The engine treats the compiled workflow like any other predicted tool: Qwen +proposes its typed arguments, DS4 remains authoritative, and the private result +is released only after an exact call match. Independent workflow branches run +concurrently inside the CPU-pinned executor process. +""" + +from __future__ import annotations + +import json +import os +import re +import sys +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Any, Callable + +from benchmark_trace_compiled_workflows import CompiledPattern, load_training_traces, mine_pattern +from bfcl_replay_tool_executor import ( + PROTOCOL, + call_ref, + call_sha256, + execute as execute_leaf, +) + + +MAX_BRANCHES = 4 +TRAINING_REPORT_ENV = "DFLASH_TRACE_TRAINING_REPORT" +WORKFLOW_REGISTRY_ENV = "DFLASH_TRACE_WORKFLOW_REGISTRY" +LeafExecutor = Callable[[dict[str, Any]], dict[str, Any]] + + +def load_pattern() -> CompiledPattern: + default = Path(__file__).with_name("results") / "trace-compiled-training-traces.json" + report = Path(os.environ.get(TRAINING_REPORT_ENV, str(default))) + return mine_pattern(load_training_traces(report, required_steps=5)) + + +def load_registry() -> dict[str, Any]: + default = Path(__file__).with_name("results") / "trace-workflow-registry.json" + path = Path(os.environ.get(WORKFLOW_REGISTRY_ENV, str(default))) + registry = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(registry, dict) or registry.get("schema_version") != 1: + raise ValueError("compiled workflow registry is invalid") + return registry + + +def validate_items(items: Any, pattern: CompiledPattern) -> list[dict[str, str]]: + if not isinstance(items, list) or not 1 <= len(items) <= MAX_BRANCHES: + raise ValueError(f"customers must contain between 1 and {MAX_BRANCHES} items") + expected_fields = set(pattern.root_fields) + validated = [] + for item in items: + if not isinstance(item, dict) or set(item) != expected_fields: + raise ValueError("customer fields do not match the compiled workflow") + if not all(isinstance(value, str) and value for value in item.values()): + raise ValueError("compiled workflow inputs must be non-empty strings") + validated.append(dict(item)) + return validated + + +def resolve_items( + arguments: Any, pattern: CompiledPattern, registry: dict[str, Any] +) -> list[dict[str, str]]: + if not isinstance(arguments, dict) or set(arguments) != {"workflow_ref"}: + raise ValueError("compiled workflow requires only a workflow_ref") + workflow_ref = arguments["workflow_ref"] + if not isinstance(workflow_ref, str) or re.fullmatch( + r"workflow_[a-z]+", workflow_ref + ) is None: + raise ValueError("workflow_ref is malformed") + if registry.get("pattern_fingerprint") != pattern.fingerprint: + raise ValueError("workflow registry pattern does not match the executor") + workflows = registry.get("workflows") + entry = workflows.get(workflow_ref) if isinstance(workflows, dict) else None + if ( + not isinstance(entry, dict) + or entry.get("pattern_fingerprint") != pattern.fingerprint + ): + raise ValueError("workflow_ref is unknown or bound to another pattern") + return validate_items(entry.get("items"), pattern) + + +def execute_branch( + request: dict[str, Any], + pattern: CompiledPattern, + root: dict[str, str], + leaf_executor: LeafExecutor, +) -> dict[str, Any]: + previous: dict[str, Any] | None = None + steps = [] + for index in range(len(pattern.steps)): + call = pattern.instantiate(root, previous, index) + leaf_request = {**request, "call": call} + envelope = leaf_executor(leaf_request) + result = envelope.get("result") if isinstance(envelope, dict) else None + if not envelope.get("ok") or not isinstance(result, dict): + raise RuntimeError("leaf tool returned an invalid result") + if ( + result.get("call_sha256") != call_sha256(call) + or result.get("call_ref") != call_ref(call) + or result.get("tool_name") != call["name"] + or result.get("side_effects") is not False + ): + raise RuntimeError("leaf tool result did not match its compiled call") + previous = result + steps.append({"call": call, "tool_result": result}) + return {"root": root, "steps": steps, "final_ref": steps[-1]["tool_result"]["call_ref"]} + + +def execute_macro( + request: dict[str, Any], + pattern: CompiledPattern, + leaf_executor: LeafExecutor = execute_leaf, + registry: dict[str, Any] | None = None, +) -> dict[str, Any]: + call = request.get("call") + if not isinstance(call, dict) or call.get("name") != pattern.macro_name: + raise ValueError("request is not for the compiled workflow") + items = resolve_items( + call.get("arguments"), pattern, registry if registry is not None else load_registry() + ) + started = time.perf_counter() + with ThreadPoolExecutor(max_workers=len(items), thread_name_prefix="compiled-workflow") as pool: + futures = [ + pool.submit(execute_branch, request, pattern, root, leaf_executor) + for root in items + ] + branches = [future.result() for future in futures] + elapsed_ms = (time.perf_counter() - started) * 1_000.0 + affinity = sorted(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else [] + return { + "ok": True, + "result": { + "call_sha256": call_sha256(call), + "call_ref": call_ref(call), + "tool_name": pattern.macro_name, + "workflow_fingerprint": pattern.fingerprint, + "branches": branches, + "call_count": len(items) * len(pattern.steps), + "elapsed_ms": elapsed_ms, + "cpu_affinity": affinity, + "side_effects": False, + }, + } + + +def execute(request: dict[str, Any], pattern: CompiledPattern) -> dict[str, Any]: + call = request.get("call") + if isinstance(call, dict) and call.get("name") == pattern.macro_name: + return execute_macro(request, pattern) + return execute_leaf(request) + + +def main() -> int: + if sys.argv[1:] != ["--dflash-tool-spec-v1"]: + print("expected --dflash-tool-spec-v1", file=sys.stderr) + return 2 + try: + line = sys.stdin.readline() + if not line: + raise ValueError("missing request") + request = json.loads(line) + if not isinstance(request, dict) or request.get("protocol") != PROTOCOL: + raise ValueError("unsupported tool-speculation request") + print(json.dumps(execute(request, load_pattern()), separators=(",", ":")), flush=True) + return 0 + except (OSError, RuntimeError, ValueError, json.JSONDecodeError) as error: + print(str(error), file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index b4876f957..b0c0a3146 100644 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -419,6 +419,7 @@ add_library(dflash_common STATIC src/common/dflash_draft_ipc.cpp src/common/dflash_draft_ipc_daemon.cpp src/common/pflash_drafter_ipc.cpp + src/common/qwen3_tool_predictor_ipc.cpp src/common/dflash_draft_graph.cpp src/common/dflash_draft_kv.cpp src/common/dflash_spec_decode.cpp @@ -468,6 +469,8 @@ add_library(dflash_common STATIC src/server/chat_template.cpp src/server/tool_parser.cpp src/server/tool_hint.cpp + src/server/semantic_tool_hint.cpp + src/server/native_semantic_tool_predictor.cpp src/server/reasoning.cpp src/server/tool_memory.cpp src/server/sse_emitter.cpp @@ -1277,6 +1280,14 @@ if(DFLASH27B_TESTS) target_include_directories(smoke_qwen3_forward PRIVATE ${DFLASH27B_SRC_INCLUDE_DIRS}) target_link_libraries(smoke_qwen3_forward PRIVATE dflash_common ggml ${DFLASH27B_GGML_BACKEND_TARGET}) endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/smoke_qwen3_tool_predictor_ipc.cpp") + add_executable(smoke_qwen3_tool_predictor_ipc + test/smoke_qwen3_tool_predictor_ipc.cpp) + target_include_directories(smoke_qwen3_tool_predictor_ipc PRIVATE + ${DFLASH27B_SRC_INCLUDE_DIRS}) + target_link_libraries(smoke_qwen3_tool_predictor_ipc PRIVATE + dflash_common ggml ${DFLASH27B_GGML_BACKEND_TARGET}) + endif() if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_vs_oracle.cpp") add_executable(test_vs_oracle test/test_vs_oracle.cpp) target_include_directories(test_vs_oracle PRIVATE ${DFLASH27B_SRC_INCLUDE_DIRS}) @@ -1467,6 +1478,7 @@ if(DFLASH27B_TESTS) test/test_unit_main.cpp test/test_server_unit.cpp test/test_tool_speculation.cpp + test/test_semantic_tool_hint.cpp test/test_anchor_params.cpp test/test_derived_scalars.cpp test/test_adaptive_keep_ratio.cpp @@ -1809,6 +1821,7 @@ if(DFLASH27B_SERVER) add_executable(backend_ipc_daemon src/ipc/backend_ipc_main.cpp src/common/pflash_drafter_ipc_daemon.cpp + src/common/qwen3_tool_predictor_ipc_daemon.cpp ) target_include_directories(backend_ipc_daemon PRIVATE ${DFLASH27B_SRC_INCLUDE_DIRS}) if(DFLASH27B_GPU_BACKEND STREQUAL "hip") diff --git a/server/src/common/backend_ipc.cpp b/server/src/common/backend_ipc.cpp index 2a98b8e9c..4e72cd86a 100644 --- a/server/src/common/backend_ipc.cpp +++ b/server/src/common/backend_ipc.cpp @@ -29,6 +29,7 @@ const char * backend_ipc_mode_name(BackendIpcMode mode) { case BackendIpcMode::Invalid: return "invalid"; case BackendIpcMode::DFlashDraft: return "dflash-draft"; case BackendIpcMode::PFlashCompress: return "pflash-compress"; + case BackendIpcMode::Qwen3ToolPredict: return "qwen3-tool-predict"; case BackendIpcMode::Qwen35TargetShard: return "qwen35-target-shard"; case BackendIpcMode::Gemma4TargetShard: return "gemma4-target-shard"; case BackendIpcMode::LagunaTargetShard: return "laguna-target-shard"; @@ -47,6 +48,10 @@ bool parse_backend_ipc_mode(const std::string & value, BackendIpcMode & out) { out = BackendIpcMode::PFlashCompress; return true; } + if (value == "qwen3-tool-predict") { + out = BackendIpcMode::Qwen3ToolPredict; + return true; + } if (value == "qwen35-target-shard") { out = BackendIpcMode::Qwen35TargetShard; return true; diff --git a/server/src/common/backend_ipc.h b/server/src/common/backend_ipc.h index d995759ac..8731f3714 100644 --- a/server/src/common/backend_ipc.h +++ b/server/src/common/backend_ipc.h @@ -25,6 +25,7 @@ enum class BackendIpcMode { Invalid, DFlashDraft, PFlashCompress, + Qwen3ToolPredict, Qwen35TargetShard, Gemma4TargetShard, LagunaTargetShard, diff --git a/server/src/common/qwen3_tool_predictor_ipc.cpp b/server/src/common/qwen3_tool_predictor_ipc.cpp new file mode 100644 index 000000000..a7bdf038c --- /dev/null +++ b/server/src/common/qwen3_tool_predictor_ipc.cpp @@ -0,0 +1,119 @@ +#include "qwen3_tool_predictor_ipc.h" + +#include "io_utils.h" + +#include +#include + +namespace dflash::common { + +bool Qwen3ToolPredictorIpcClient::start( + const std::string & bin, + const std::string & model_path, + int gpu, + int max_ctx, + const std::string & work_dir) { +#if defined(_WIN32) + (void)bin; (void)model_path; (void)gpu; (void)max_ctx; (void)work_dir; + std::fprintf(stderr, + "Qwen3 tool-predictor IPC is only implemented on POSIX hosts\n"); + return false; +#else + std::lock_guard lock(mutex_); + close_locked(); + if (bin.empty() || model_path.empty() || max_ctx <= 0) return false; + + BackendIpcLaunchConfig launch; + launch.bin = bin; + launch.mode = BackendIpcMode::Qwen3ToolPredict; + launch.payload_path = model_path; + launch.work_dir = work_dir; + launch.args.push_back("--target-gpu=" + std::to_string(std::max(0, gpu))); + launch.args.push_back("--max-ctx=" + std::to_string(max_ctx)); + if (!process_.start(launch)) { + std::fprintf(stderr, "[tool-predictor-ipc] backend process start failed\n"); + return false; + } + active_ = true; + std::fprintf(stderr, + "[tool-predictor-ipc] ready model=%s gpu=%d max_ctx=%d work_dir=%s\n", + model_path.c_str(), std::max(0, gpu), max_ctx, + process_.work_dir().c_str()); + return true; +#endif +} + +bool Qwen3ToolPredictorIpcClient::predict( + const std::vector & prompt_ids, + int max_tokens, + std::vector & output_ids, + std::string & error) { + output_ids.clear(); + error.clear(); +#if defined(_WIN32) + (void)prompt_ids; (void)max_tokens; + error = "native_predictor_ipc_unsupported"; + return false; +#else + std::lock_guard lock(mutex_); + FILE * command = process_.command_stream(); + const int stream_fd = process_.stream_fd(); + if (!active_ || !command || stream_fd < 0) { + error = "native_predictor_not_active"; + return false; + } + if (prompt_ids.empty() || max_tokens <= 0) { + error = "native_predictor_invalid_request"; + return false; + } + + const std::string path = process_.next_path("tool_predictor_prompt"); + if (!write_int32_file(path, prompt_ids)) { + error = "native_predictor_prompt_write_failed"; + return false; + } + + std::fprintf(command, "predict %d %s\n", max_tokens, path.c_str()); + std::fflush(command); + + int32_t status = -1; + bool ok = read_exact_fd(stream_fd, &status, sizeof(status)) && status == 0; + if (ok) { + int32_t count = -1; + ok = read_exact_fd(stream_fd, &count, sizeof(count)) && + count > 0 && count <= max_tokens; + if (ok) { + output_ids.assign(static_cast(count), 0); + ok = read_exact_fd(stream_fd, output_ids.data(), + output_ids.size() * sizeof(int32_t)); + } + } + std::remove(path.c_str()); + if (!ok) { + error = status == 0 + ? "native_predictor_invalid_response" + : "native_predictor_generation_failed"; + output_ids.clear(); + close_locked(); + return false; + } + return true; +#endif +} + +bool Qwen3ToolPredictorIpcClient::active() const { + std::lock_guard lock(mutex_); + return active_; +} + +void Qwen3ToolPredictorIpcClient::close_locked() { + process_.close(); + active_ = false; +} + +void Qwen3ToolPredictorIpcClient::close() { + std::lock_guard lock(mutex_); + close_locked(); +} + +} // namespace dflash::common diff --git a/server/src/common/qwen3_tool_predictor_ipc.h b/server/src/common/qwen3_tool_predictor_ipc.h new file mode 100644 index 000000000..5cab9513c --- /dev/null +++ b/server/src/common/qwen3_tool_predictor_ipc.h @@ -0,0 +1,57 @@ +// Persistent Qwen3 tool-predictor IPC lane. +// +// The HTTP server tokenizes the predictor prompt with the predictor's own +// vocabulary, then sends token IDs to a small out-of-process Qwen3 backend. +// Keeping this lane behind BackendIpcProcess isolates the target decoder from +// predictor crashes and lets heterogeneous deployments choose a different GPU. + +#pragma once + +#include "backend_ipc.h" + +#include +#include +#include +#include +#include + +namespace dflash::common { + +class Qwen3ToolPredictorIpcClient { +public: + Qwen3ToolPredictorIpcClient() = default; + Qwen3ToolPredictorIpcClient(const Qwen3ToolPredictorIpcClient &) = delete; + Qwen3ToolPredictorIpcClient & operator=( + const Qwen3ToolPredictorIpcClient &) = delete; + ~Qwen3ToolPredictorIpcClient() { close(); } + + bool start(const std::string & bin, + const std::string & model_path, + int gpu, + int max_ctx, + const std::string & work_dir); + + // Requests are serialized: one compact predictor model owns one KV cache. + // On transport or generation failure the lane closes and fails shut. + bool predict(const std::vector & prompt_ids, + int max_tokens, + std::vector & output_ids, + std::string & error); + + bool active() const; + void close(); + +private: + void close_locked(); + + mutable std::mutex mutex_; + BackendIpcProcess process_; + bool active_ = false; +}; + +int run_qwen3_tool_predictor_ipc_daemon(const char * model_path, + int gpu, + int max_ctx, + int stream_fd); + +} // namespace dflash::common diff --git a/server/src/common/qwen3_tool_predictor_ipc_daemon.cpp b/server/src/common/qwen3_tool_predictor_ipc_daemon.cpp new file mode 100644 index 000000000..9ebe96fd3 --- /dev/null +++ b/server/src/common/qwen3_tool_predictor_ipc_daemon.cpp @@ -0,0 +1,120 @@ +#include "qwen3_tool_predictor_ipc.h" + +#include "io_utils.h" +#include "model_backend.h" +#include "qwen3/qwen3_backend.h" + +#include +#include +#include +#include +#include + +namespace dflash::common { +namespace { + +bool send_status(int stream_fd, int32_t status) { + return write_exact_fd(stream_fd, &status, sizeof(status)); +} + +} // namespace + +int run_qwen3_tool_predictor_ipc_daemon( + const char * model_path, + int gpu, + int max_ctx, + int stream_fd) { +#if defined(_WIN32) + (void)model_path; (void)gpu; (void)max_ctx; (void)stream_fd; + return 2; +#else + if (!model_path || !*model_path || stream_fd < 0 || max_ctx <= 0) { + std::fprintf(stderr, + "usage: backend_ipc_daemon --backend-ipc-mode=qwen3-tool-predict " + " --stream-fd=FD --target-gpu=N --max-ctx=N\n"); + return 2; + } + + Qwen3BackendConfig config; + config.model_path = model_path; + config.device.backend = PlacementBackend::Auto; + config.device.gpu = std::max(0, gpu); + config.device.max_ctx = max_ctx; + config.chunk = 512; + + Qwen3Backend backend(config); + if (!backend.init()) { + std::fprintf(stderr, "[tool-predictor-daemon] Qwen3 init failed\n"); + send_status(stream_fd, -1); + return 1; + } + std::fprintf(stderr, + "[tool-predictor-daemon] ready gpu=%d max_ctx=%d\n", + std::max(0, gpu), max_ctx); + send_status(stream_fd, 0); + + std::string line; + while (std::getline(std::cin, line)) { + std::istringstream input(line); + std::string command; + input >> command; + if (command == "quit" || command == "exit") break; + if (command != "predict") { + std::fprintf(stderr, + "[tool-predictor-daemon] unknown command: %s\n", + line.c_str()); + send_status(stream_fd, -1); + continue; + } + + int max_tokens = 0; + input >> max_tokens; + const std::string path = read_line_tail(input); + if (max_tokens <= 0 || path.empty()) { + send_status(stream_fd, -1); + continue; + } + const auto prompt = read_int32_file(path); + if (prompt.empty() || + prompt.size() + static_cast(max_tokens) > + static_cast(max_ctx)) { + std::fprintf(stderr, + "[tool-predictor-daemon] invalid context prompt=%zu max_tokens=%d max_ctx=%d\n", + prompt.size(), max_tokens, max_ctx); + send_status(stream_fd, -1); + continue; + } + + GenerateRequest request; + request.prompt = prompt; + request.n_gen = max_tokens; + request.do_sample = false; + request.stream = false; + DaemonIO io; + const GenerateResult result = backend.generate(request, io); + if (!result.ok() || result.tokens.empty()) { + std::fprintf(stderr, + "[tool-predictor-daemon] generation failed code=%s\n", + result.error_code().data()); + send_status(stream_fd, -1); + continue; + } + + const int32_t count = static_cast(result.tokens.size()); + if (!send_status(stream_fd, 0) || + !write_exact_fd(stream_fd, &count, sizeof(count)) || + !write_exact_fd(stream_fd, result.tokens.data(), + result.tokens.size() * sizeof(int32_t))) { + std::fprintf(stderr, + "[tool-predictor-daemon] response write failed\n"); + break; + } + } + + backend.shutdown(); + std::fprintf(stderr, "[tool-predictor-daemon] stopped\n"); + return 0; +#endif +} + +} // namespace dflash::common diff --git a/server/src/ipc/backend_ipc_main.cpp b/server/src/ipc/backend_ipc_main.cpp index b2fae7791..e2764ef9d 100644 --- a/server/src/ipc/backend_ipc_main.cpp +++ b/server/src/ipc/backend_ipc_main.cpp @@ -8,6 +8,7 @@ #include "gemma4/gemma4_layer_split_adapter.h" #include "laguna/laguna_layer_split_adapter.h" #include "pflash_drafter_ipc.h" +#include "qwen3_tool_predictor_ipc.h" #include "common/platform_env.h" #include "qwen35_target_shard_ipc.h" @@ -117,6 +118,8 @@ int main(int argc, char ** argv) { "[--shared-payload-fd=FD --shared-payload-bytes=N] [--draft-gpu=N]\n" " or: %s --backend-ipc-mode=pflash-compress " "--stream-fd=FD [--draft-gpu=N]\n" + " or: %s --backend-ipc-mode=qwen3-tool-predict " + "--stream-fd=FD --target-gpu=N --max-ctx=N\n" " or: %s --backend-ipc-mode=qwen35-target-shard " "--stream-fd=FD --target-gpu=N --layer-begin=N --layer-end=N " "--max-ctx=N [--hidden=N --vocab=N --max-tokens=N]\n" @@ -138,6 +141,8 @@ int main(int argc, char ** argv) { argv[0], argv[0], argv[0], + argv[0], + argv[0], argv[0]); return 2; } @@ -315,6 +320,9 @@ int main(int argc, char ** argv) { shared_payload_bytes); case BackendIpcMode::PFlashCompress: return run_pflash_drafter_ipc_daemon(payload_path, draft_gpu, stream_fd); + case BackendIpcMode::Qwen3ToolPredict: + return run_qwen3_tool_predictor_ipc_daemon( + payload_path, target_gpu, max_ctx, stream_fd); case BackendIpcMode::Qwen35TargetShard: if (target_gpus.empty()) target_gpus.push_back(target_gpu); if (layer_begins.empty()) layer_begins.push_back(layer_begin); diff --git a/server/src/qwen3/qwen3_loader.cpp b/server/src/qwen3/qwen3_loader.cpp index 583261992..52bc3291e 100644 --- a/server/src/qwen3/qwen3_loader.cpp +++ b/server/src/qwen3/qwen3_loader.cpp @@ -134,7 +134,11 @@ bool load_qwen3_drafter_model(const std::string & path, out.head_dim = (int)get_u32(gctx, "qwen3.attention.key_length", 128); out.rope_theta = get_f32(gctx, "qwen3.rope.freq_base", 1000000.0f); - // Detect weight quant type from blk.0.attn_q.weight; support BF16 and Q8_0. + // Preserve Q8_0 storage when the predictor reuses the production compact + // GGUF. Activations/KV still use the backend precision policy; ggml's + // mul_mat and get_rows kernels dequantize Q8_0 weights as they are read. + // This avoids expanding a 0.6B sidecar to BF16 merely to use the native + // IPC lane. ggml_type wtype = GGML_TYPE_BF16; { int64_t tidx = gguf_find_tensor(gctx, "blk.0.attn_q.weight"); @@ -142,8 +146,16 @@ bool load_qwen3_drafter_model(const std::string & path, wtype = gguf_get_tensor_type(gctx, tidx); } } + if (wtype == GGML_TYPE_Q8_0) { + out.weight_type = GGML_TYPE_Q8_0; + } else if (wtype != GGML_TYPE_BF16 && wtype != GGML_TYPE_F16) { + set_last_error(std::string("unsupported Qwen3-0.6B weight type: ") + + ggml_type_name(wtype)); + gguf_free(gctx); + return false; + } std::fprintf(stderr, "[qwen3-0.6b] detected weight type: %s\n", - wtype == GGML_TYPE_Q8_0 ? "Q8_0" : "BF16"); + ggml_type_name(wtype)); std::fflush(stderr); // Compute total tensor metadata size for context allocation. diff --git a/server/src/server/chat_template.cpp b/server/src/server/chat_template.cpp index 939386bab..149a3f0fb 100644 --- a/server/src/server/chat_template.cpp +++ b/server/src/server/chat_template.cpp @@ -44,6 +44,23 @@ static const char QWEN3_TOOL_SUFFIX[] = "current knowledge and do not tell the user about function calls\n" ""; +static const char DEEPSEEK4_TOOL_PREAMBLE[] = + "# Tools\n\nYou have access to these functions:\n\n"; + +static const char DEEPSEEK4_TOOL_FORMAT[] = + "\n\n\n" + "When calling a function, reply with exactly one call in this format " + "and nothing after it:\n" + "\n" + "\n" + "\n" + "PARAMETER_VALUE\n" + "\n" + "\n" + "\n" + "Use the exact function and parameter names from , and include all " + "required parameters."; + ChatFormat chat_format_for_arch(const std::string & arch) { if (arch == "deepseek4") return ChatFormat::DEEPSEEK4; if (arch == "laguna") return ChatFormat::LAGUNA; @@ -57,7 +74,8 @@ std::string render_chat_template( ChatFormat format, bool add_generation_prompt, bool enable_thinking, - const std::string & tools_json) + const std::string & tools_json, + bool tool_call_required) { std::string result; bool has_tools = !tools_json.empty() && tools_json != "[]" && tools_json != "null"; @@ -370,10 +388,12 @@ std::string render_chat_template( result = "<｜begin▁of▁sentence｜>"; if (has_tools) { - // Tool schema rendering is not implemented for the native DSML - // path yet; keep the JSON visible in the system prefix rather than - // silently dropping it. + result += DEEPSEEK4_TOOL_PREAMBLE; result += tools_json; + result += DEEPSEEK4_TOOL_FORMAT; + result += tool_call_required + ? "\nYou MUST call exactly one of these functions to answer this request." + : "\nIf a function is applicable, call it instead of answering from memory."; if (has_system) result += "\n\n"; } result += system_content; diff --git a/server/src/server/chat_template.h b/server/src/server/chat_template.h index ecade9217..b7825153c 100644 --- a/server/src/server/chat_template.h +++ b/server/src/server/chat_template.h @@ -39,14 +39,17 @@ enum class ChatFormat { // false → assistant starts with \n\n\n\n (skip thinking) // // `tools_json` is an optional JSON string containing the tool definitions -// array. When non-empty, the Qwen3/3.5 template injects a tool preamble -// into the system message instructing the model how to emit tags. +// array. When non-empty, tool-capable templates inject a tool preamble into +// the system message instructing the model how to emit tags. +// `tool_call_required` strengthens that instruction for OpenAI +// `tool_choice="required"` and forced-function requests. std::string render_chat_template( const std::vector & messages, ChatFormat format, bool add_generation_prompt = true, bool enable_thinking = false, - const std::string & tools_json = ""); + const std::string & tools_json = "", + bool tool_call_required = false); // Detect the appropriate chat format for an architecture. ChatFormat chat_format_for_arch(const std::string & arch); diff --git a/server/src/server/http_server.cpp b/server/src/server/http_server.cpp index 11218f8f1..cb7928aab 100644 --- a/server/src/server/http_server.cpp +++ b/server/src/server/http_server.cpp @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -71,6 +72,7 @@ static inline bool sock_is_eintr (int e) { return e == WSAEINTR; } static inline bool sock_is_eagain(int e) { return e == WSAEWOULDBLOCK; } #else #include +#include #include static inline int sock_get_flags(SocketHandle fd) { return fcntl(fd, F_GETFL, 0); } static inline void sock_set_nonblock(SocketHandle fd) { int f = fcntl(fd, F_GETFL, 0); if (f >= 0) fcntl(fd, F_SETFL, f | O_NONBLOCK); } @@ -409,6 +411,262 @@ static bool curl_forward(const std::string & url, } #endif // DFLASH_HAS_CURL +namespace { + +struct SemanticSidecarUrl { + std::string host; + std::string port; + std::string path; +}; + +bool parse_semantic_sidecar_url( + const std::string & value, SemanticSidecarUrl & out) { + constexpr char kHttpPrefix[] = "http://"; + if (value.rfind(kHttpPrefix, 0) != 0) return false; + const size_t authority_begin = sizeof(kHttpPrefix) - 1; + const size_t path_begin = value.find('/', authority_begin); + const std::string authority = value.substr( + authority_begin, + path_begin == std::string::npos + ? std::string::npos + : path_begin - authority_begin); + if (authority.empty() || authority.find('@') != std::string::npos) { + return false; + } + + const size_t colon = authority.rfind(':'); + if (colon == std::string::npos) { + out.host = authority; + out.port = "80"; + } else { + out.host = authority.substr(0, colon); + out.port = authority.substr(colon + 1); + } + out.path = path_begin == std::string::npos + ? "/" : value.substr(path_begin); + if (out.host.empty() || out.port.empty() || out.path.empty()) return false; + if (out.host.front() == '[' || out.host.find(':') != std::string::npos) { + // The production bridge is loopback IPv4. Reject ambiguous IPv6 + // authority parsing instead of silently connecting to the wrong host. + return false; + } + return std::all_of(out.port.begin(), out.port.end(), [](unsigned char ch) { + return std::isdigit(ch) != 0; + }); +} + +bool semantic_sidecar_send_all( + SocketHandle fd, const char * data, size_t size) { + size_t sent = 0; + while (sent < size) { +#if defined(_WIN32) + const int n = ::send( + fd, data + sent, + static_cast(std::min(size - sent, INT_MAX)), 0); +#else + const ssize_t n = ::send( + fd, data + sent, size - sent, MSG_NOSIGNAL); +#endif + if (n <= 0) return false; + sent += static_cast(n); + } + return true; +} + +void set_semantic_sidecar_socket_timeout(SocketHandle fd, int timeout_ms) { +#if defined(_WIN32) + const DWORD timeout = static_cast(timeout_ms); + setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, + reinterpret_cast(&timeout), sizeof(timeout)); + setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, + reinterpret_cast(&timeout), sizeof(timeout)); +#else + struct timeval timeout{}; + timeout.tv_sec = timeout_ms / 1000; + timeout.tv_usec = (timeout_ms % 1000) * 1000; + setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout)); + setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(timeout)); +#endif +} + +std::string lowercase_ascii(std::string value) { + std::transform(value.begin(), value.end(), value.begin(), + [](unsigned char ch) { return static_cast(std::tolower(ch)); }); + return value; +} + +bool decode_chunked_http_body( + const std::string & encoded, std::string & decoded) { + size_t offset = 0; + while (true) { + const size_t line_end = encoded.find("\r\n", offset); + if (line_end == std::string::npos) return false; + const std::string size_text = encoded.substr(offset, line_end - offset); + const size_t extension = size_text.find(';'); + const std::string hex = size_text.substr(0, extension); + size_t parsed = 0; + unsigned long chunk_size = 0; + try { + chunk_size = std::stoul(hex, &parsed, 16); + } catch (...) { + return false; + } + if (parsed != hex.size()) return false; + offset = line_end + 2; + if (chunk_size == 0) return true; + if (chunk_size > encoded.size() - std::min(offset, encoded.size())) { + return false; + } + decoded.append(encoded, offset, static_cast(chunk_size)); + offset += static_cast(chunk_size); + if (offset + 2 > encoded.size() || + encoded.compare(offset, 2, "\r\n") != 0) { + return false; + } + offset += 2; + } +} + +SemanticToolPrediction request_semantic_tool_prediction( + const SemanticToolPredictorConfig & config, + const json & payload, + const json & request_tools) { + const auto started = std::chrono::steady_clock::now(); + SemanticToolPrediction prediction; + prediction.source = config.model; + auto finish = [&]() { + prediction.wall_ms = std::chrono::duration( + std::chrono::steady_clock::now() - started).count(); + return prediction; + }; + + SemanticSidecarUrl url; + if (!parse_semantic_sidecar_url(config.url, url)) { + prediction.error = "predictor_url_must_be_http_host_port_path"; + return finish(); + } + + struct addrinfo hints{}; + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + struct addrinfo * addresses = nullptr; + if (getaddrinfo(url.host.c_str(), url.port.c_str(), &hints, &addresses) != 0) { + prediction.error = "predictor_host_resolution_failed"; + return finish(); + } + + SocketHandle fd = kInvalidSocket; + for (auto * address = addresses; address; address = address->ai_next) { + fd = socket(address->ai_family, address->ai_socktype, + address->ai_protocol); + if (!socket_is_valid(fd)) continue; + set_semantic_sidecar_socket_timeout(fd, config.timeout_ms); + if (connect(fd, address->ai_addr, + static_cast(address->ai_addrlen)) == 0) { + break; + } + socket_close(fd); + fd = kInvalidSocket; + } + freeaddrinfo(addresses); + if (!socket_is_valid(fd)) { + prediction.error = "predictor_connect_failed"; + return finish(); + } + + const std::string body = payload.dump(); + const std::string request = + "POST " + url.path + " HTTP/1.1\r\n" + + "Host: " + url.host + ":" + url.port + "\r\n" + + "Content-Type: application/json\r\n" + + "Accept: application/json\r\n" + + "Connection: close\r\n" + + "Content-Length: " + std::to_string(body.size()) + "\r\n\r\n" + + body; + if (!semantic_sidecar_send_all(fd, request.data(), request.size())) { + socket_close(fd); + prediction.error = "predictor_send_failed"; + return finish(); + } + + constexpr size_t kMaxResponseBytes = 1024 * 1024; + std::string response; + std::array buffer{}; + while (response.size() < kMaxResponseBytes) { +#if defined(_WIN32) + const int received = recv( + fd, buffer.data(), static_cast(buffer.size()), 0); +#else + const ssize_t received = recv(fd, buffer.data(), buffer.size(), 0); +#endif + if (received == 0) break; + if (received < 0) { + socket_close(fd); + prediction.error = "predictor_receive_failed_or_timed_out"; + return finish(); + } + response.append(buffer.data(), static_cast(received)); + } + socket_close(fd); + if (response.size() >= kMaxResponseBytes) { + prediction.error = "predictor_response_too_large"; + return finish(); + } + + const size_t header_end = response.find("\r\n\r\n"); + const size_t status_end = response.find("\r\n"); + if (header_end == std::string::npos || status_end == std::string::npos) { + prediction.error = "predictor_malformed_http_response"; + return finish(); + } + const std::string status_line = response.substr(0, status_end); + const size_t status_space = status_line.find(' '); + if (status_space == std::string::npos || + status_line.size() < status_space + 4) { + prediction.error = "predictor_malformed_http_status"; + return finish(); + } + const int status = std::atoi(status_line.c_str() + status_space + 1); + if (status < 200 || status >= 300) { + prediction.error = "predictor_http_status_" + std::to_string(status); + return finish(); + } + + const std::string headers = lowercase_ascii( + response.substr(status_end + 2, header_end - status_end - 2)); + const std::string encoded_body = response.substr(header_end + 4); + std::string response_body; + if (headers.find("transfer-encoding: chunked") != std::string::npos) { + if (!decode_chunked_http_body(encoded_body, response_body)) { + prediction.error = "predictor_invalid_chunked_response"; + return finish(); + } + } else { + response_body = encoded_body; + } + + json response_json; + try { + response_json = json::parse(response_body); + } catch (...) { + prediction.error = "predictor_response_invalid_json"; + return finish(); + } + if (!parse_semantic_tool_prediction( + response_json, request_tools, prediction.call, + prediction.error)) { + return finish(); + } + if (!materialize_declared_tool_defaults( + request_tools, prediction.call, prediction.error)) { + return finish(); + } + prediction.ok = true; + return finish(); +} + +} // namespace + // ─── /props constants ─────────────────────────────────────────────────── // // SERVER_NAME / SERVER_VERSION mirror the Python server's identity strings @@ -549,6 +807,15 @@ static const json * find_tool_function(const json & tools, return nullptr; } +static bool tool_choice_requires_call(const json & tool_choice) { + if (tool_choice.is_string()) { + return tool_choice.get() == "required"; + } + return tool_choice.is_object() && tool_choice.contains("function") && + tool_choice["function"].is_object() && + !tool_choice["function"].value("name", "").empty(); +} + static std::string first_tool_parameter_name(const json & function_def) { const auto & params = function_def.value("parameters", json::object()); if (params.contains("required") && params["required"].is_array()) { @@ -796,6 +1063,27 @@ json build_props_body(const ServerConfig & config, }}, {"tool_speculation", { {"enabled", config.tool_speculation.enabled()}, + {"automatic_prediction_enabled", + config.tool_speculation.enabled() && + config.semantic_tool_predictor.enabled()}, + {"prediction_source", + config.semantic_tool_predictor.native_enabled() + ? json("native-qwen3") + : config.semantic_tool_predictor.http_enabled() + ? json(config.semantic_tool_predictor.model) + : json(nullptr)}, + {"prediction_confidence", + config.semantic_tool_predictor.enabled() + ? json(config.semantic_tool_predictor.execution_confidence) + : json(nullptr)}, + {"predictor_schedule", + config.semantic_tool_predictor.native_enabled() + ? json(native_tool_predictor_schedule_name( + config.semantic_tool_predictor.native_schedule)) + : config.semantic_tool_predictor.http_enabled() + ? json("overlap") : json(nullptr)}, + {"predictor_decode_isolated", + config.semantic_tool_predictor.native_runs_before_model()}, {"execution_mode", config.tool_speculation.execution_mode()}, {"profile_status", config.tool_speculation.policy.empty() @@ -806,7 +1094,9 @@ json build_props_body(const ServerConfig & config, ? json(nullptr) : json(config.tool_speculation.policy.executor_contract())}, {"protocol", "dflash.tool-speculation.v1"}, - {"requires_client_support", true}, + {"requires_client_support", + !(config.tool_speculation.enabled() && + config.semantic_tool_predictor.enabled())}, {"preserves_token_speculation", true}, {"unqualified_lane_policy", "defer"}, {"allowed_tools", config.tool_speculation.allowed_tools}, @@ -1272,6 +1562,24 @@ HttpServer::~HttpServer() { #endif } +bool HttpServer::init_semantic_tool_predictor(std::string & error) { + error.clear(); + if (!config_.semantic_tool_predictor.native_enabled()) return true; + native_semantic_predictor_ = NativeSemanticToolPredictor::create( + config_.semantic_tool_predictor, error); + if (native_semantic_predictor_) return true; + // An explicitly configured HTTP lane is a valid fail-open fallback. The + // target remains authoritative and still verifies every prediction. + if (config_.semantic_tool_predictor.http_enabled()) { + std::fprintf(stderr, + "[tool-hint] native predictor unavailable (%s); using HTTP fallback\n", + error.c_str()); + error.clear(); + return true; + } + return false; +} + void HttpServer::shutdown() { // Signal worker and accept loop to stop. stopping_.store(true); @@ -1664,6 +1972,15 @@ bool HttpServer::parse_common_request_fields( if (body.contains("tools")) req.tools = body["tools"]; // Tool choice constraint for hint generation. if (body.contains("tool_choice")) req.tool_choice = body["tool_choice"]; + if (body.contains("automatic_tool_speculation")) { + if (!body["automatic_tool_speculation"].is_boolean()) { + send_error(fd, 400, + "automatic_tool_speculation must be a boolean"); + return false; + } + req.automatic_tool_speculation_enabled = + body["automatic_tool_speculation"].get(); + } if (body.contains("tool_speculation")) { ToolSpeculationPrediction prediction; @@ -1940,7 +2257,8 @@ bool HttpServer::render_and_tokenize_request( } else { rendered = render_chat_template( chat_messages, chat_format_, /*add_generation_prompt=*/true, - req.thinking_enabled, tools_json); + req.thinking_enabled, tools_json, + tool_choice_requires_call(req.tool_choice)); } req.started_in_thinking = prompt_ends_in_open_think(rendered); @@ -1981,6 +2299,112 @@ void HttpServer::log_parsed_request(const ParsedRequest & req) const { req.stop_sequences.size(), req.model.c_str()); } +void HttpServer::launch_semantic_tool_prediction(ParsedRequest & req) const { + if (req.semantic_tool_prediction.valid() || + req.automatic_tool_speculation.valid()) { + return; + } + const bool automatic_execution = + req.automatic_tool_speculation_enabled && + !req.tool_speculation.has_value() && + config_.tool_speculation.enabled(); + if (!automatic_execution || + !config_.semantic_tool_predictor.enabled() || req.tools.empty() || + !req.raw_body.is_object()) { + return; + } + const SemanticToolPredictorConfig predictor = + config_.semantic_tool_predictor; + json semantic_request = req.raw_body; + // Endpoint parsers normalize Anthropic/Responses dialogue into req.messages. + // Supplying it here gives both transports one OpenAI-shaped semantic view. + semantic_request["messages"] = req.messages; + semantic_request["tools"] = req.tools; + if (!req.tool_choice.is_null()) { + semantic_request["tool_choice"] = req.tool_choice; + } + const json payload = build_semantic_tool_predictor_request( + semantic_request, + predictor.model.empty() ? "native-qwen3" : predictor.model, + predictor.max_tokens); + const json tools = req.tools; + const auto native = native_semantic_predictor_; + req.semantic_tool_prediction = std::async( + std::launch::async, + [predictor, payload, tools, native]() { + SemanticToolPrediction native_result; + if (native && native->active()) { + native_result = native->predict(payload, tools); + if (native_result.ok || !predictor.http_enabled()) { + return native_result; + } + } + if (predictor.http_enabled()) { + SemanticToolPrediction fallback = + request_semantic_tool_prediction(predictor, payload, tools); + if (!fallback.ok && !native_result.error.empty()) { + fallback.error = "native=" + native_result.error + + ";http=" + fallback.error; + } + return fallback; + } + if (native_result.error.empty()) { + native_result.error = "native_predictor_not_initialized"; + } + return native_result; + }).share(); + if (automatic_execution) { + const auto semantic_prediction = req.semantic_tool_prediction; + const ToolSpeculationConfig tool_config = config_.tool_speculation; + const double confidence = predictor.execution_confidence; + const std::string request_id = req.response_id; + req.automatic_tool_speculation = std::async( + std::launch::async, + [semantic_prediction, tool_config, confidence, request_id]() { + ParsedRequest::AutomaticToolSpeculationLaunch launch; + try { + const SemanticToolPrediction & semantic = + semantic_prediction.get(); + launch.predictor_wall_ms = semantic.wall_ms; + launch.prediction_source = semantic.source; + if (!semantic.ok) { + launch.predictor_error = semantic.error.empty() + ? "predictor_unavailable" : semantic.error; + return launch; + } + ToolSpeculationPrediction prediction; + std::string error; + const json arguments = json::parse( + semantic.call.arguments.dump()); + if (!build_tool_speculation_prediction( + semantic.call.name, arguments, confidence, + prediction, error)) { + launch.predictor_error = std::move(error); + return launch; + } + auto attempt = ToolSpeculationAttempt::create( + tool_config, prediction, request_id); + launch.attempt = std::shared_ptr( + attempt.release()); + launch.attempt->start(); + } catch (const std::exception & error) { + launch.predictor_error = + std::string("automatic_prediction_failed: ") + + error.what(); + } catch (...) { + launch.predictor_error = + "automatic_prediction_failed: unknown error"; + } + return launch; + }).share(); + } + std::fprintf(stderr, + "[tool-hint] launched predictor transport=%s%s tools=%zu execute=%s\n", + native ? "native-qwen3" : "http", + native && predictor.http_enabled() ? "+http-fallback" : "", + json_array_size(req.tools), automatic_execution ? "true" : "false"); +} + void HttpServer::enqueue_request_and_wait(SocketHandle fd, ParsedRequest req) { // Set socket non-blocking for send() stall detection during streaming. const int flags = sock_get_flags(fd); @@ -2078,6 +2502,9 @@ bool HttpServer::route_request(SocketHandle fd, const HttpRequest & hr) { } if (!validate_request_context(fd, req)) return true; + if (!config_.semantic_tool_predictor.native_runs_before_model()) { + launch_semantic_tool_prediction(req); + } log_parsed_request(req); enqueue_request_and_wait(fd, std::move(req)); return true; @@ -2621,7 +3048,8 @@ void HttpServer::apply_flowkv_compression( } else { rendered = render_chat_template( chat_messages, chat_format_, /*add_generation_prompt=*/true, - req.thinking_enabled, tools_json); + req.thinking_enabled, tools_json, + tool_choice_requires_call(req.tool_choice)); } const int tokens_before = (int) prepared.tokens.size(); @@ -3406,11 +3834,16 @@ void HttpServer::prepare_generation_inputs( inputs.request.n_gen = inputs.generation_cap; inputs.request.sampler = req.sampler; inputs.request.do_sample = req.sampler.needs_logit_processing(); - // An opted-in external tool may overlap the chosen decoder, but it must - // never cause speculative decoding to be retried as AR. If the selected - // decoder cannot produce output, surface that outcome unchanged. + // Tool prediction must never change the target decoder or trigger an + // autoregressive retry. DS4/DSpark remains authoritative on every arm. + const bool semantic_tool_request = + req.automatic_tool_speculation_enabled && + config_.tool_speculation.enabled() && + config_.semantic_tool_predictor.enabled() && + !req.tools.empty(); inputs.request.allow_decode_mode_retry = - !req.tool_speculation.has_value(); + !req.tool_speculation.has_value() && + !semantic_tool_request; // Tokens are delivered through DaemonIO so all API formats share the // same disconnect and streaming state machine. inputs.request.stream = false; @@ -3530,7 +3963,7 @@ void HttpServer::worker_loop() { void HttpServer::process_job(ServerJob * job) { SocketHandle fd = job->fd; - const auto & req = job->req; + auto & req = job->req; auto started_at = std::chrono::steady_clock::now(); // Track live status for /status page. RAII guard ensures idle on all paths. @@ -3628,6 +4061,26 @@ void HttpServer::process_job(ServerJob * job) { } if (req.stream) start_job_stream(job); + if (config_.semantic_tool_predictor.native_runs_before_model()) { + const auto predictor_wait_started = std::chrono::steady_clock::now(); + launch_semantic_tool_prediction(req); + if (req.automatic_tool_speculation.valid()) { + req.automatic_tool_speculation.wait(); + } else if (req.semantic_tool_prediction.valid()) { + req.semantic_tool_prediction.wait(); + } + const double predictor_wait_ms = + std::chrono::duration( + std::chrono::steady_clock::now() - predictor_wait_started) + .count(); + if (req.automatic_tool_speculation.valid() || + req.semantic_tool_prediction.valid()) { + std::fprintf(stderr, + "[tool-hint] before-model barrier complete %.1f ms\n", + predictor_wait_ms); + } + } + PreparedPrompt prepared = prepare_prompt(req); if (prepared.error_status != 0) { fail_request(prepared.error_status, prepared.error); @@ -3775,12 +4228,61 @@ void HttpServer::process_job(ServerJob * job) { if (job->client_disconnected.load(std::memory_order_acquire)) { client_disconnected = true; } + auto finish_tool_speculation = [&](bool cancel) -> std::optional { + if (tool_speculation) { + json metadata = cancel + ? tool_speculation->cancel("client_disconnected") + : tool_speculation->resolve(emitter.tool_calls()); + metadata["prediction_source"] = "client"; + return metadata; + } + if (!req.automatic_tool_speculation.valid()) return std::nullopt; + try { + const ParsedRequest::AutomaticToolSpeculationLaunch & launch = + req.automatic_tool_speculation.get(); + json metadata; + if (launch.attempt) { + metadata = cancel + ? launch.attempt->cancel("client_disconnected") + : launch.attempt->resolve(emitter.tool_calls()); + } else { + metadata = { + {"protocol", "dflash.tool-speculation.v1"}, + {"status", "deferred"}, + {"reason", "predictor_unavailable"}, + }; + if (!launch.predictor_error.empty()) { + metadata["detail"] = launch.predictor_error; + } + } + metadata["prediction_source"] = + launch.prediction_source.empty() + ? "predictor" : launch.prediction_source; + metadata["predictor_wall_ms"] = launch.predictor_wall_ms; + return metadata; + } catch (const std::exception & error) { + return json{ + {"protocol", "dflash.tool-speculation.v1"}, + {"status", "failed"}, + {"reason", "predictor_future_failure"}, + {"detail", error.what()}, + {"prediction_source", "predictor"}, + }; + } catch (...) { + return json{ + {"protocol", "dflash.tool-speculation.v1"}, + {"status", "failed"}, + {"reason", "predictor_future_failure"}, + {"detail", "unknown error"}, + {"prediction_source", "predictor"}, + }; + } + }; if (req.stream && !client_disconnected) { auto final_chunks = emitter.emit_finish(completion_tokens, &gen_timings); - if (tool_speculation) { - const json metadata = tool_speculation->resolve(emitter.tool_calls()); + if (auto metadata = finish_tool_speculation(false)) { const std::string extension = render_tool_speculation_sse( - req.format, req.response_id, req.model, metadata); + req.format, req.response_id, req.model, *metadata); // Keep the standard terminal event last: [DONE], message_stop, // or response.completed. Only opted-in clients see this extension. if (final_chunks.empty()) { @@ -3798,9 +4300,8 @@ void HttpServer::process_job(ServerJob * job) { } else if (!req.stream && !client_disconnected) { json response = build_non_streaming_response( req, result, n_gen_cap, gen_timings, tokenizer_, emitter); - if (tool_speculation) { - response["dflash_tool_speculation"] = - tool_speculation->resolve(emitter.tool_calls()); + if (auto metadata = finish_tool_speculation(false)) { + response["dflash_tool_speculation"] = std::move(*metadata); } // Streaming uses non-blocking sends; restore blocking mode before // writing a complete JSON response on this shared socket path. @@ -3808,8 +4309,8 @@ void HttpServer::process_job(ServerJob * job) { if (flags >= 0) sock_set_block(fd); send_response(fd, 200, "application/json", response.dump() + "\n"); - } else if (tool_speculation) { - tool_speculation->cancel("client_disconnected"); + } else { + finish_tool_speculation(true); } if (client_disconnected) { diff --git a/server/src/server/http_server.h b/server/src/server/http_server.h index 14a5a82cb..4ddb3a7a1 100644 --- a/server/src/server/http_server.h +++ b/server/src/server/http_server.h @@ -19,6 +19,8 @@ #include "chat_template.h" #include "tool_memory.h" #include "tool_speculation.h" +#include "semantic_tool_hint.h" +#include "native_semantic_tool_predictor.h" #include "prefix_cache.h" #include "disk_prefix_cache.h" #include "freeze_history.h" @@ -36,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -224,6 +227,10 @@ struct ServerConfig { // an executor, an empirical interference profile, and an explicit // read-only/idempotent tool allowlist. ToolSpeculationConfig tool_speculation; + // Model-agnostic tool-call prediction. Native predictors run before the + // model by default so shared accelerator compute cannot slow decoding; + // remote predictors may overlap. DS4 verifies the exact canonical call. + SemanticToolPredictorConfig semantic_tool_predictor; }; // ─── Parsed request ───────────────────────────────────────────────────── @@ -247,6 +254,18 @@ struct ParsedRequest { // The engine may execute it privately, but never exposes its result until // the model emits the exact canonical invocation. std::optional tool_speculation; + // Engine-side Qwen prediction may start a private external tool as soon + // as it is ready. The model's eventual canonical call remains authoritative. + struct AutomaticToolSpeculationLaunch { + std::shared_ptr attempt; + double predictor_wall_ms = 0.0; + std::string prediction_source; + std::string predictor_error; + }; + bool automatic_tool_speculation_enabled = true; + std::shared_future + automatic_tool_speculation; + std::shared_future semantic_tool_prediction; // Response ID std::string response_id; // Thinking/reasoning state @@ -314,6 +333,10 @@ class HttpServer { // Set the chat template format (detected from model arch). void set_chat_format(ChatFormat fmt) { chat_format_ = fmt; } + // Start the optional native Qwen predictor after target construction. + // HTTP-only predictor configurations need no persistent initialization. + bool init_semantic_tool_predictor(std::string & error); + // Start listening. Blocks until shutdown() is called. int run(); @@ -431,6 +454,7 @@ class HttpServer { ParsedRequest & req); bool validate_request_context(SocketHandle fd, const ParsedRequest & req); void log_parsed_request(const ParsedRequest & req) const; + void launch_semantic_tool_prediction(ParsedRequest & req) const; void enqueue_request_and_wait(SocketHandle fd, ParsedRequest req); // Send HTTP response helpers. @@ -457,6 +481,7 @@ class HttpServer { ServerConfig config_; ChatFormat chat_format_; PFlashDrafterIpcClient pflash_remote_; + std::shared_ptr native_semantic_predictor_; ToolMemory tool_memory_; PrefixCache prefix_cache_; DiskPrefixCache disk_cache_; diff --git a/server/src/server/native_semantic_tool_predictor.cpp b/server/src/server/native_semantic_tool_predictor.cpp new file mode 100644 index 000000000..9551d549c --- /dev/null +++ b/server/src/server/native_semantic_tool_predictor.cpp @@ -0,0 +1,85 @@ +#include "native_semantic_tool_predictor.h" + +#include +#include +#include + +namespace dflash::common { + +std::shared_ptr +NativeSemanticToolPredictor::create( + const SemanticToolPredictorConfig & config, + std::string & error) { + error.clear(); + if (!config.native_enabled()) { + error = "native_predictor_config_incomplete"; + return nullptr; + } + auto predictor = std::shared_ptr( + new NativeSemanticToolPredictor(config)); + if (!predictor->tokenizer_.load_from_gguf( + config.native_model_path.c_str())) { + error = "native_predictor_tokenizer_load_failed"; + return nullptr; + } + if (!predictor->ipc_.start( + config.native_ipc_bin, config.native_model_path, + config.native_gpu, config.native_max_ctx, + config.native_work_dir)) { + error = "native_predictor_ipc_start_failed"; + return nullptr; + } + return predictor; +} + +SemanticToolPrediction NativeSemanticToolPredictor::predict( + const json & predictor_request, + const json & request_tools, + std::string * generated_text) { + const auto started = std::chrono::steady_clock::now(); + SemanticToolPrediction prediction; + prediction.source = "native-qwen3"; + auto finish = [&]() { + prediction.wall_ms = std::chrono::duration( + std::chrono::steady_clock::now() - started).count(); + return prediction; + }; + + std::string prompt_error; + const std::string prompt = build_native_semantic_tool_predictor_prompt( + predictor_request, prompt_error); + if (prompt.empty()) { + prediction.error = std::move(prompt_error); + return finish(); + } + const std::vector prompt_ids = tokenizer_.encode(prompt); + if (prompt_ids.empty()) { + prediction.error = "native_predictor_prompt_tokenization_failed"; + return finish(); + } + if (prompt_ids.size() + static_cast(config_.max_tokens) > + static_cast(config_.native_max_ctx)) { + prediction.error = "native_predictor_context_overflow"; + return finish(); + } + + std::vector output_ids; + if (!ipc_.predict(prompt_ids, config_.max_tokens, + output_ids, prediction.error)) { + return finish(); + } + const std::string generated = tokenizer_.decode(output_ids); + if (generated_text) *generated_text = generated; + if (!parse_native_semantic_tool_prediction( + generated, request_tools, prediction.call, prediction.error)) { + return finish(); + } + if (!materialize_declared_tool_defaults( + request_tools, prediction.call, prediction.error)) { + return finish(); + } + prediction.ok = true; + return finish(); +} + +} // namespace dflash::common diff --git a/server/src/server/native_semantic_tool_predictor.h b/server/src/server/native_semantic_tool_predictor.h new file mode 100644 index 000000000..088db4344 --- /dev/null +++ b/server/src/server/native_semantic_tool_predictor.h @@ -0,0 +1,40 @@ +// Native Qwen semantic tool predictor built from the PFlash/Qwen runtime. + +#pragma once + +#include "semantic_tool_hint.h" + +#include "common/qwen3_tool_predictor_ipc.h" +#include "tokenizer.h" + +#include +#include + +namespace dflash::common { + +class NativeSemanticToolPredictor { +public: + static std::shared_ptr create( + const SemanticToolPredictorConfig & config, + std::string & error); + + NativeSemanticToolPredictor(const NativeSemanticToolPredictor &) = delete; + NativeSemanticToolPredictor & operator=( + const NativeSemanticToolPredictor &) = delete; + + SemanticToolPrediction predict(const json & predictor_request, + const json & request_tools, + std::string * generated_text = nullptr); + + bool active() const { return ipc_.active(); } + +private: + explicit NativeSemanticToolPredictor( + const SemanticToolPredictorConfig & config) : config_(config) {} + + SemanticToolPredictorConfig config_; + Tokenizer tokenizer_; + Qwen3ToolPredictorIpcClient ipc_; +}; + +} // namespace dflash::common diff --git a/server/src/server/semantic_tool_hint.cpp b/server/src/server/semantic_tool_hint.cpp new file mode 100644 index 000000000..cf7e7c3de --- /dev/null +++ b/server/src/server/semantic_tool_hint.cpp @@ -0,0 +1,500 @@ +#include "semantic_tool_hint.h" + +#include "tool_parser.h" + +#include +#include +#include + +namespace dflash::common { + +const char * native_tool_predictor_schedule_name( + NativeToolPredictorSchedule schedule) { + switch (schedule) { + case NativeToolPredictorSchedule::BeforeModel: + return "before-model"; + case NativeToolPredictorSchedule::Overlap: + return "overlap"; + } + return "unknown"; +} + +bool parse_native_tool_predictor_schedule( + const std::string & value, + NativeToolPredictorSchedule & out) { + if (value == "before-model") { + out = NativeToolPredictorSchedule::BeforeModel; + return true; + } + if (value == "overlap") { + out = NativeToolPredictorSchedule::Overlap; + return true; + } + return false; +} + +namespace { + +bool request_has_function(const json & tools, const std::string & name) { + if (!tools.is_array() || name.empty()) return false; + for (const auto & tool : tools) { + if (!tool.is_object()) continue; + if (tool.value("name", "") == name) return true; + const auto function = tool.find("function"); + if (function != tool.end() && function->is_object() && + function->value("name", "") == name) { + return true; + } + } + return false; +} + +std::string sole_request_function(const json & tools) { + if (!tools.is_array()) return {}; + std::string sole; + for (const auto & tool : tools) { + if (!tool.is_object()) continue; + std::string name = tool.value("name", ""); + const auto function = tool.find("function"); + if (name.empty() && function != tool.end() && function->is_object()) { + name = function->value("name", ""); + } + if (name.empty()) continue; + if (!sole.empty() && sole != name) return {}; + sole = std::move(name); + } + return sole; +} + +bool parse_arguments(const json & value, ordered_json & out) { + try { + if (value.is_string()) { + out = ordered_json::parse(value.get()); + } else if (value.is_object()) { + out = ordered_json::parse(value.dump()); + } else { + return false; + } + } catch (...) { + return false; + } + return out.is_object(); +} + +bool parse_call_object(const json & value, SemanticToolCall & out) { + if (!value.is_object()) return false; + const std::string name = value.value( + "name", value.value("function", std::string{})); + if (name.empty()) return false; + + const json * arguments = nullptr; + for (const char * key : {"arguments", "parameters", "params"}) { + const auto it = value.find(key); + if (it != value.end()) { + arguments = &*it; + break; + } + } + ordered_json parsed; + if (!arguments || !parse_arguments(*arguments, parsed)) return false; + out.name = name; + out.arguments = std::move(parsed); + return true; +} + +bool parse_content_call(const std::string & content, SemanticToolCall & out) { + for (size_t offset = 0; offset < content.size(); ++offset) { + if (content[offset] != '{') continue; + try { + const auto value = json::parse( + content.begin() + static_cast(offset), + content.end(), nullptr, false); + if (!value.is_discarded() && parse_call_object(value, out)) { + return true; + } + } catch (...) { + // Continue scanning for a later strict object. + } + } + return false; +} + +std::string trim_copy(std::string value) { + const auto is_space = [](unsigned char ch) { return std::isspace(ch); }; + value.erase(value.begin(), std::find_if_not( + value.begin(), value.end(), is_space)); + value.erase(std::find_if_not(value.rbegin(), value.rend(), is_space).base(), + value.end()); + return value; +} + +bool parse_qwen_tagged_call_repair( + const std::string & generated_text, + SemanticToolCall & out) { + const std::string open = ""; + const size_t open_pos = generated_text.find(open); + if (open_pos == std::string::npos) return false; + const size_t content_pos = open_pos + open.size(); + size_t end_pos = generated_text.find("", content_pos); + if (end_pos == std::string::npos) { + end_pos = generated_text.find("<|im_end|>", content_pos); + } + if (end_pos == std::string::npos) end_pos = generated_text.size(); + + std::string payload = trim_copy( + generated_text.substr(content_pos, end_pos - content_pos)); + if (payload.empty()) return false; + + // Qwen3-0.6B Q8 occasionally drops only the outer opening brace and + // emits a stray quote before the matching closing brace, while keeping + // the name and argument object strict JSON. Repair only that narrow + // envelope error; all semantic fields still pass the normal schema gate. + if (payload.front() != '{') payload.insert(payload.begin(), '{'); + if (payload.size() >= 3 && payload.back() == '}') { + size_t quote = payload.size() - 2; + while (quote > 0 && std::isspace( + static_cast(payload[quote]))) { + --quote; + } + if (payload[quote] == '"') payload.erase(quote, 1); + } + const json value = json::parse(payload, nullptr, false); + return !value.is_discarded() && parse_call_object(value, out); +} + +bool parse_qwen_bare_single_tool_arguments( + const std::string & generated_text, + const json & request_tools, + SemanticToolCall & out) { + const std::string name = sole_request_function(request_tools); + if (name.empty()) return false; + + size_t begin = generated_text.find(""); + begin = begin == std::string::npos + ? 0 : begin + std::string("").size(); + begin = generated_text.find('{', begin); + if (begin == std::string::npos) return false; + size_t end = generated_text.rfind('}'); + if (end == std::string::npos || end < begin) return false; + + const json arguments = json::parse( + generated_text.begin() + static_cast(begin), + generated_text.begin() + static_cast(end + 1), + nullptr, false); + if (arguments.is_discarded() || !arguments.is_object()) return false; + out.name = name; + out.arguments = ordered_json::parse(arguments.dump()); + return true; +} + +std::string semantic_message_content(const json & message) { + const auto content = message.find("content"); + if (content == message.end() || content->is_null()) return {}; + if (content->is_string()) return content->get(); + if (!content->is_array()) return content->dump(); + + std::string text; + for (const auto & part : *content) { + if (part.is_string()) { + text += part.get(); + continue; + } + if (!part.is_object()) continue; + const std::string type = part.value("type", ""); + if (type == "text" || type == "input_text" || + type == "output_text") { + text += part.value("text", ""); + } + } + return text; +} + +std::string forced_tool_name(const json & choice) { + if (!choice.is_object()) return {}; + const auto function = choice.find("function"); + if (function != choice.end() && function->is_object()) { + return function->value("name", ""); + } + return choice.value("name", ""); +} + +} // namespace + +bool parse_semantic_tool_prediction( + const json & response, + const json & request_tools, + SemanticToolCall & out, + std::string & error) { + error.clear(); + const auto choices = response.find("choices"); + if (choices == response.end() || !choices->is_array() || + choices->size() != 1 || !(*choices)[0].is_object()) { + error = "predictor_response_missing_single_choice"; + return false; + } + const auto message = (*choices)[0].find("message"); + if (message == (*choices)[0].end() || !message->is_object()) { + error = "predictor_response_missing_message"; + return false; + } + + bool parsed = false; + const auto calls = message->find("tool_calls"); + if (calls != message->end() && calls->is_array() && calls->size() == 1) { + const auto function = (*calls)[0].find("function"); + if (function != (*calls)[0].end()) { + parsed = parse_call_object(*function, out); + } + } + if (!parsed) { + const auto content = message->find("content"); + if (content != message->end() && content->is_string()) { + parsed = parse_content_call(content->get(), out); + } + } + if (!parsed) { + error = "predictor_response_has_no_valid_call"; + return false; + } + if (!request_has_function(request_tools, out.name)) { + error = "predictor_selected_unknown_function"; + return false; + } + return true; +} + +bool materialize_declared_tool_defaults( + const json & request_tools, + SemanticToolCall & call, + std::string & error) { + error.clear(); + if (!call.arguments.is_object()) { + error = "predictor_arguments_not_object"; + return false; + } + if (!request_tools.is_array()) { + error = "predictor_tools_not_array"; + return false; + } + + const json * function = nullptr; + for (const auto & tool : request_tools) { + if (!tool.is_object()) continue; + const json & candidate = tool.contains("function") && + tool["function"].is_object() + ? tool["function"] : tool; + if (candidate.value("name", "") == call.name) { + function = &candidate; + break; + } + } + if (!function) { + error = "predictor_selected_unknown_function"; + return false; + } + + const json * parameters = nullptr; + for (const char * key : {"parameters", "input_schema"}) { + const auto found = function->find(key); + if (found != function->end() && found->is_object()) { + parameters = &*found; + break; + } + } + if (!parameters) return true; + const auto properties = parameters->find("properties"); + if (properties == parameters->end() || !properties->is_object()) { + return true; + } + for (const auto & property : properties->items()) { + if (call.arguments.contains(property.key()) || + !property.value().is_object() || + !property.value().contains("default")) { + continue; + } + call.arguments[property.key()] = property.value()["default"]; + } + return true; +} + +json build_semantic_tool_predictor_request( + const json & target_request, + const std::string & sidecar_model, + int max_tokens) { + json request = { + {"model", sidecar_model}, + {"stream", false}, + {"temperature", 0}, + {"max_tokens", max_tokens}, + }; + for (const char * key : {"messages", "tools", "tool_choice"}) { + const auto it = target_request.find(key); + if (it != target_request.end()) request[key] = *it; + } + if (!request.contains("tool_choice")) request["tool_choice"] = "auto"; + return request; +} + +std::string build_native_semantic_tool_predictor_prompt( + const json & predictor_request, + std::string & error) { + error.clear(); + const auto messages = predictor_request.find("messages"); + if (messages == predictor_request.end() || !messages->is_array() || + messages->empty()) { + error = "native_predictor_missing_messages"; + return {}; + } + const json tools = predictor_request.value("tools", json::array()); + if (!tools.is_array() || tools.empty()) { + error = "native_predictor_missing_tools"; + return {}; + } + + struct PredictorMessage { + std::string role; + std::string content; + json tool_calls; + }; + std::vector chat; + chat.reserve(messages->size()); + for (const auto & message : *messages) { + if (!message.is_object()) continue; + std::string role = message.value("role", "user"); + if (role == "developer") role = "system"; + chat.push_back({ + std::move(role), semantic_message_content(message), + message.value("tool_calls", json::array()), + }); + } + if (chat.empty()) { + error = "native_predictor_empty_messages"; + return {}; + } + + std::string constraint; + const json choice = predictor_request.value("tool_choice", json("auto")); + if (choice.is_string() && choice.get() == "required") { + constraint = "You must call exactly one available function."; + } else if (const std::string name = forced_tool_name(choice); + !name.empty()) { + constraint = "You must call the function " + name + "."; + } + // Render the exact tokenizer.chat_template contract embedded in the + // Qwen3-0.6B GGUF. PFlash's generic Qwen3.5 renderer uses parameter XML, + // while this model was trained to emit one JSON object inside + // ; using the wrong contract destroys multi-tool accuracy. + size_t begin = 0; + std::string system_content; + if (!chat.empty() && chat.front().role == "system") { + system_content = chat.front().content; + begin = 1; + } + if (!constraint.empty()) { + if (!system_content.empty()) system_content += "\n\n"; + system_content += constraint; + } + + std::string rendered = "<|im_start|>system\n"; + if (!system_content.empty()) { + rendered += system_content; + rendered += "\n\n"; + } + rendered += + "# Tools\n\n" + "You may call one or more functions to assist with the user query.\n\n" + "You are provided with function signatures within XML tags:\n" + ""; + for (const auto & tool : tools) rendered += tool.dump(); + rendered += + "\n\n\n" + "For each function call, return a json object with function name and " + "arguments within XML tags:\n" + "\n" + "{\"name\": , \"arguments\": }\n" + "<|im_end|>\n"; + + bool in_tool_response = false; + for (size_t index = begin; index < chat.size(); ++index) { + const auto & message = chat[index]; + if (message.role == "tool") { + if (!in_tool_response) { + rendered += "<|im_start|>user"; + in_tool_response = true; + } + rendered += "\n\n" + message.content + + "\n"; + const bool next_is_tool = index + 1 < chat.size() && + chat[index + 1].role == "tool"; + if (!next_is_tool) { + rendered += "<|im_end|>\n"; + in_tool_response = false; + } + continue; + } + + rendered += "<|im_start|>" + message.role + "\n" + message.content; + if (message.role == "assistant" && message.tool_calls.is_array()) { + for (const auto & raw_call : message.tool_calls) { + if (!raw_call.is_object()) continue; + const json & call = raw_call.contains("function") && + raw_call["function"].is_object() + ? raw_call["function"] : raw_call; + const std::string name = call.value("name", ""); + if (name.empty() || !call.contains("arguments")) continue; + if (!message.content.empty()) rendered += "\n"; + rendered += "\n{\"name\": \"" + name + + "\", \"arguments\": "; + rendered += call["arguments"].is_string() + ? call["arguments"].get() + : call["arguments"].dump(); + rendered += "}\n"; + } + } + rendered += "<|im_end|>\n"; + } + rendered += "<|im_start|>assistant\n\n\n\n\n"; + return rendered; +} + +bool parse_native_semantic_tool_prediction( + const std::string & generated_text, + const json & request_tools, + SemanticToolCall & out, + std::string & error) { + error.clear(); + const ToolParseResult parsed = parse_tool_calls( + generated_text, request_tools); + if (parsed.tool_calls.size() == 1) { + try { + ordered_json arguments = ordered_json::parse( + parsed.tool_calls.front().arguments); + if (!arguments.is_object()) { + error = "native_predictor_arguments_not_object"; + return false; + } + out.name = parsed.tool_calls.front().name; + out.arguments = std::move(arguments); + } catch (...) { + error = "native_predictor_arguments_invalid_json"; + return false; + } + } else if (parsed.tool_calls.empty()) { + if (!parse_qwen_tagged_call_repair(generated_text, out) && + !parse_qwen_bare_single_tool_arguments( + generated_text, request_tools, out)) { + error = "native_predictor_response_has_no_valid_call"; + return false; + } + } else { + error = "native_predictor_response_has_multiple_calls"; + return false; + } + if (!request_has_function(request_tools, out.name)) { + error = "predictor_selected_unknown_function"; + return false; + } + return true; +} + +} // namespace dflash::common diff --git a/server/src/server/semantic_tool_hint.h b/server/src/server/semantic_tool_hint.h new file mode 100644 index 000000000..69c6f962c --- /dev/null +++ b/server/src/server/semantic_tool_hint.h @@ -0,0 +1,105 @@ +// Model-agnostic tool-call predictions shared by HTTP and native predictors. + +#pragma once + +#include + +#include + +namespace dflash::common { + +using json = nlohmann::json; +using ordered_json = nlohmann::ordered_json; + +enum class NativeToolPredictorSchedule { + BeforeModel, + Overlap, +}; + +const char * native_tool_predictor_schedule_name( + NativeToolPredictorSchedule schedule); + +bool parse_native_tool_predictor_schedule( + const std::string & value, + NativeToolPredictorSchedule & out); + +struct SemanticToolPredictorConfig { + std::string url; + std::string model; + std::string native_model_path; + std::string native_ipc_bin; + std::string native_work_dir; + int native_gpu = 0; + int native_max_ctx = 4096; + int timeout_ms = 2000; + int max_tokens = 96; + NativeToolPredictorSchedule native_schedule = + NativeToolPredictorSchedule::BeforeModel; + // Conservative prior used by the measured tool-execution admission + // policy. The base predictor currently emits no calibrated probability. + double execution_confidence = 0.75; + + bool http_enabled() const { return !url.empty() && !model.empty(); } + bool native_enabled() const { + return !native_model_path.empty() && !native_ipc_bin.empty(); + } + bool enabled() const { return native_enabled() || http_enabled(); } + bool native_runs_before_model() const { + return native_enabled() && + native_schedule == NativeToolPredictorSchedule::BeforeModel; + } +}; + +struct SemanticToolCall { + std::string name; + ordered_json arguments = ordered_json::object(); +}; + +struct SemanticToolPrediction { + bool ok = false; + std::string error; + // Actual predictor used for this result. Native and HTTP fallback paths + // share one execution gate, so response metadata must not guess. + std::string source; + SemanticToolCall call; + double wall_ms = 0.0; +}; + +// Parse one OpenAI-compatible sidecar response and reject calls whose +// function name is absent from the request schema. Arguments remain decoded +// JSON values; sidecar token IDs are never accepted by the target. +bool parse_semantic_tool_prediction( + const json & response, + const json & request_tools, + SemanticToolCall & out, + std::string & error); + +// Materialize top-level defaults declared by the selected function before a +// prediction is executed. This turns an omitted optional default into the +// exact explicit invocation the target may emit; the normal exact-match gate +// still rejects the result if the authoritative call differs. +bool materialize_declared_tool_defaults( + const json & request_tools, + SemanticToolCall & call, + std::string & error); + +// Build the small OpenAI-compatible request sent to the predictor. Only +// dialogue/tool semantics are forwarded; target-only extensions are omitted. +json build_semantic_tool_predictor_request( + const json & target_request, + const std::string & sidecar_model, + int max_tokens); + +// Native predictor bridge. The prompt uses the Qwen tool template and the +// decoded response is parsed semantically before any target token IDs exist. +std::string build_native_semantic_tool_predictor_prompt( + const json & predictor_request, + std::string & error); + +bool parse_native_semantic_tool_prediction( + const std::string & generated_text, + const json & request_tools, + SemanticToolCall & out, + std::string & error); + +} // namespace dflash::common diff --git a/server/src/server/server_main.cpp b/server/src/server/server_main.cpp index 7dc9b28ed..8485a79d6 100644 --- a/server/src/server/server_main.cpp +++ b/server/src/server/server_main.cpp @@ -178,6 +178,33 @@ static void print_usage(const char * prog) { " Drafter lifetime policy (default: auto)\n" " --lazy-draft Legacy alias for --draft-residency=request-scoped\n" "\n" + "Tool-call prediction (lossless exact verification):\n" + " --tool-hint-sidecar-url \n" + " OpenAI-compatible chat-completions endpoint.\n" + " --tool-hint-sidecar-model \n" + " Predictor model served by that endpoint.\n" + " --tool-hint-native-model \n" + " Qwen3-0.6B GGUF for the native PFlash runtime.\n" + " --tool-hint-native-ipc-bin \n" + " Matching backend_ipc_daemon executable.\n" + " --tool-hint-native-gpu Predictor GPU (default: 0).\n" + " --tool-hint-native-max-ctx \n" + " Predictor context capacity (default: 4096).\n" + " --tool-hint-native-schedule \n" + " before-model (default) runs Qwen before DS4\n" + " so shared-GPU decoding cannot be slowed;\n" + " overlap is an experimental throughput mode.\n" + " --tool-hint-native-work-dir \n" + " Optional private IPC scratch directory.\n" + " --tool-hint-sidecar-timeout-ms \n" + " Hard sidecar deadline (default: 2000).\n" + " --tool-hint-sidecar-max-tokens \n" + " Predictor completion cap (default: 96).\n" + " --tool-hint-execution-confidence

\n" + " Calibrated 0..1 prior for automatic external\n" + " tool admission (default: 0.75).\n" + " The target verifies every hint.\n" + "\n" "Speculative external tools (opt-in, POSIX):\n" " --tool-spec-executor Trusted executor adapter. Receives one\n" " dflash.tool-speculation.v1 JSON request\n" @@ -265,6 +292,7 @@ int main(int argc, char ** argv) { bool fast_rollback_forced_off = false; bool target_split_fast_rollback_cli = false; bool adaptive_experts_set = false; // --adaptive-experts (MoE architectures only) + bool native_tool_predictor_schedule_set = false; // Track which thinking-budget tunables the operator set via CLI. // Those values win over the model card (spec §3.1: "Explicit CLI @@ -544,6 +572,77 @@ int main(int argc, char ** argv) { } else if (std::strcmp(argv[i], "--lazy-draft") == 0) { sconfig.lazy_draft = true; sconfig.draft_residency = DraftResidencyPolicy::RequestScoped; + } else if (std::strcmp(argv[i], "--tool-hint-sidecar-url") == 0 && + i + 1 < argc) { + sconfig.semantic_tool_predictor.url = argv[++i]; + } else if (std::strcmp(argv[i], "--tool-hint-sidecar-model") == 0 && + i + 1 < argc) { + sconfig.semantic_tool_predictor.model = argv[++i]; + } else if (std::strcmp(argv[i], "--tool-hint-native-model") == 0 && + i + 1 < argc) { + sconfig.semantic_tool_predictor.native_model_path = argv[++i]; + } else if (std::strcmp(argv[i], "--tool-hint-native-ipc-bin") == 0 && + i + 1 < argc) { + sconfig.semantic_tool_predictor.native_ipc_bin = argv[++i]; + } else if (std::strcmp(argv[i], "--tool-hint-native-work-dir") == 0 && + i + 1 < argc) { + sconfig.semantic_tool_predictor.native_work_dir = argv[++i]; + } else if (std::strcmp(argv[i], "--tool-hint-native-gpu") == 0 && + i + 1 < argc) { + sconfig.semantic_tool_predictor.native_gpu = std::atoi(argv[++i]); + if (sconfig.semantic_tool_predictor.native_gpu < 0) { + std::fprintf(stderr, + "[server] --tool-hint-native-gpu must be non-negative\n"); + return 2; + } + } else if (std::strcmp(argv[i], "--tool-hint-native-max-ctx") == 0 && + i + 1 < argc) { + sconfig.semantic_tool_predictor.native_max_ctx = std::atoi(argv[++i]); + if (sconfig.semantic_tool_predictor.native_max_ctx <= 0) { + std::fprintf(stderr, + "[server] --tool-hint-native-max-ctx must be positive\n"); + return 2; + } + } else if (std::strcmp(argv[i], "--tool-hint-native-schedule") == 0 && + i + 1 < argc) { + native_tool_predictor_schedule_set = true; + if (!parse_native_tool_predictor_schedule( + argv[++i], + sconfig.semantic_tool_predictor.native_schedule)) { + std::fprintf(stderr, + "[server] --tool-hint-native-schedule must be " + "before-model or overlap\n"); + return 2; + } + } else if (std::strcmp(argv[i], "--tool-hint-sidecar-timeout-ms") == 0 && + i + 1 < argc) { + sconfig.semantic_tool_predictor.timeout_ms = std::atoi(argv[++i]); + if (sconfig.semantic_tool_predictor.timeout_ms <= 0) { + std::fprintf(stderr, + "[server] --tool-hint-sidecar-timeout-ms must be positive\n"); + return 2; + } + } else if (std::strcmp(argv[i], "--tool-hint-sidecar-max-tokens") == 0 && + i + 1 < argc) { + sconfig.semantic_tool_predictor.max_tokens = std::atoi(argv[++i]); + if (sconfig.semantic_tool_predictor.max_tokens <= 0) { + std::fprintf(stderr, + "[server] --tool-hint-sidecar-max-tokens must be positive\n"); + return 2; + } + } else if (std::strcmp( + argv[i], "--tool-hint-execution-confidence") == 0 && + i + 1 < argc) { + sconfig.semantic_tool_predictor.execution_confidence = + std::atof(argv[++i]); + if (!std::isfinite( + sconfig.semantic_tool_predictor.execution_confidence) || + sconfig.semantic_tool_predictor.execution_confidence < 0.0 || + sconfig.semantic_tool_predictor.execution_confidence > 1.0) { + std::fprintf(stderr, + "[server] --tool-hint-execution-confidence must be between 0 and 1\n"); + return 2; + } } else if (std::strcmp(argv[i], "--tool-spec-executor") == 0 && i + 1 < argc) { sconfig.tool_speculation.executor_path = argv[++i]; @@ -689,6 +788,28 @@ int main(int argc, char ** argv) { } #endif } + const bool semantic_http_predictor_requested = + !sconfig.semantic_tool_predictor.url.empty() || + !sconfig.semantic_tool_predictor.model.empty(); + if (semantic_http_predictor_requested && + !sconfig.semantic_tool_predictor.http_enabled()) { + std::fprintf(stderr, + "[server] HTTP semantic tool hints require both " + "--tool-hint-sidecar-url and --tool-hint-sidecar-model\n"); + return 2; + } + const bool semantic_native_predictor_requested = + !sconfig.semantic_tool_predictor.native_model_path.empty() || + !sconfig.semantic_tool_predictor.native_ipc_bin.empty() || + !sconfig.semantic_tool_predictor.native_work_dir.empty() || + native_tool_predictor_schedule_set; + if (semantic_native_predictor_requested && + !sconfig.semantic_tool_predictor.native_enabled()) { + std::fprintf(stderr, + "[server] native semantic tool hints require both " + "--tool-hint-native-model and --tool-hint-native-ipc-bin\n"); + return 2; + } const bool tool_speculation_requested = !sconfig.tool_speculation.executor_path.empty() || static_cast(sconfig.tool_speculation.in_process_executor) || @@ -1326,6 +1447,33 @@ int main(int argc, char ** argv) { std::fprintf(stderr, "[server] │ cors = %s\n", sconfig.enable_cors ? "ON" : "off"); std::fprintf(stderr, "[server] │ tool_speculation= %s\n", sconfig.tool_speculation.enabled() ? "ON" : "off"); + std::fprintf(stderr, "[server] │ tool_call_predictor= %s\n", + sconfig.semantic_tool_predictor.enabled() ? "ON" : "off"); + if (sconfig.semantic_tool_predictor.enabled()) { + const auto & predictor = sconfig.semantic_tool_predictor; + std::fprintf(stderr, "[server] │ tool_hint_transport= %s%s\n", + predictor.native_enabled() ? "native-qwen3" : "http", + predictor.native_enabled() && predictor.http_enabled() + ? "+http-fallback" : ""); + std::fprintf(stderr, "[server] │ tool_hint_model = %s\n", + predictor.native_enabled() + ? predictor.native_model_path.c_str() + : predictor.model.c_str()); + if (predictor.native_enabled()) { + std::fprintf(stderr, + "[server] │ tool_hint_gpu = %d (max_ctx=%d)\n", + predictor.native_gpu, predictor.native_max_ctx); + std::fprintf(stderr, + "[server] │ tool_hint_schedule= %s\n", + native_tool_predictor_schedule_name( + predictor.native_schedule)); + } + std::fprintf(stderr, "[server] │ tool_hint_timeout= %d ms\n", + predictor.timeout_ms); + std::fprintf(stderr, "[server] │ tool_hint_execute= %s (confidence=%.3f)\n", + sconfig.tool_speculation.enabled() ? "ON" : "off", + predictor.execution_confidence); + } if (sconfig.tool_speculation.enabled()) { std::fprintf(stderr, "[server] │ tool_spec_exec = %s\n", sconfig.tool_speculation.execution_mode()); @@ -1464,6 +1612,13 @@ int main(int argc, char ** argv) { HttpServer server(*backend, tokenizer, sconfig); server.set_chat_format(chat_format_for_arch(arch)); + std::string semantic_predictor_error; + if (!server.init_semantic_tool_predictor(semantic_predictor_error)) { + std::fprintf(stderr, + "[server] native semantic tool predictor initialization failed: %s\n", + semantic_predictor_error.c_str()); + return 1; + } g_server = &server; std::signal(SIGTERM, signal_handler); std::signal(SIGINT, signal_handler); diff --git a/server/src/server/tool_speculation.cpp b/server/src/server/tool_speculation.cpp index f3a055eba..c3d679a3c 100644 --- a/server/src/server/tool_speculation.cpp +++ b/server/src/server/tool_speculation.cpp @@ -229,6 +229,27 @@ bool CanonicalToolInvocation::from_tool_call( } } +bool build_tool_speculation_prediction( + const std::string & name, + const json & arguments, + double confidence, + ToolSpeculationPrediction & out, + std::string & error) { + if (!std::isfinite(confidence) || confidence < 0.0 || confidence > 1.0) { + error = "tool prediction confidence must be between 0 and 1"; + return false; + } + CanonicalToolInvocation invocation; + if (!CanonicalToolInvocation::from_parts( + name, arguments, invocation, error)) { + return false; + } + out.call = std::move(invocation); + out.confidence = confidence; + error.clear(); + return true; +} + bool parse_tool_speculation_prediction( const json & value, const json & tools, @@ -261,18 +282,17 @@ bool parse_tool_speculation_prediction( error = "tool_speculation.call.arguments is required"; return false; } - CanonicalToolInvocation invocation; - if (!CanonicalToolInvocation::from_parts( - call["name"].get(), call["arguments"], invocation, - error)) { + ToolSpeculationPrediction prediction; + if (!build_tool_speculation_prediction( + call["name"].get(), call["arguments"], confidence, + prediction, error)) { return false; } - if (!request_declares_tool(tools, invocation.name)) { + if (!request_declares_tool(tools, prediction.call.name)) { error = "tool_speculation.call.name is not declared in tools"; return false; } - out.call = std::move(invocation); - out.confidence = confidence; + out = std::move(prediction); error.clear(); return true; } diff --git a/server/src/server/tool_speculation.h b/server/src/server/tool_speculation.h index 0e862a683..2e38fadb9 100644 --- a/server/src/server/tool_speculation.h +++ b/server/src/server/tool_speculation.h @@ -44,6 +44,15 @@ struct ToolSpeculationPrediction { double confidence = 0.0; }; +// Construct a canonical prediction from an engine-side predictor. This is +// the same validation boundary used for caller-supplied predictions, minus +// the request-schema check performed by the semantic predictor itself. +bool build_tool_speculation_prediction(const std::string & name, + const json & arguments, + double confidence, + ToolSpeculationPrediction & out, + std::string & error); + // Parse the request extension: // "tool_speculation": { // "call": {"name": "...", "arguments": {...}}, diff --git a/server/test/smoke_qwen3_tool_predictor_ipc.cpp b/server/test/smoke_qwen3_tool_predictor_ipc.cpp new file mode 100644 index 000000000..866bbaf96 --- /dev/null +++ b/server/test/smoke_qwen3_tool_predictor_ipc.cpp @@ -0,0 +1,191 @@ +#include "server/native_semantic_tool_predictor.h" + +#include +#include +#include +#include +#include + +using namespace dflash::common; + +namespace { + +struct Case { + const char * id; + const char * prompt; + const char * expected_name; + ordered_json expected_arguments; +}; + +json production_tools() { + return json::parse(R"json( +[ + {"type":"function","function":{"name":"get_weather","description":"Get current weather for one city.","parameters":{"type":"object","properties":{"city":{"type":"string"},"unit":{"type":"string","enum":["celsius","fahrenheit"]}},"required":["city","unit"],"additionalProperties":false}}}, + {"type":"function","function":{"name":"get_stock_quote","description":"Get the latest market quote for a ticker symbol.","parameters":{"type":"object","properties":{"symbol":{"type":"string"}},"required":["symbol"],"additionalProperties":false}}}, + {"type":"function","function":{"name":"search_documents","description":"Search indexed documents.","parameters":{"type":"object","properties":{"query":{"type":"string"},"limit":{"type":"integer","minimum":1,"maximum":20}},"required":["query","limit"],"additionalProperties":false}}}, + {"type":"function","function":{"name":"calculate","description":"Evaluate one arithmetic expression.","parameters":{"type":"object","properties":{"expression":{"type":"string"}},"required":["expression"],"additionalProperties":false}}}, + {"type":"function","function":{"name":"lookup_order","description":"Look up an order by its identifier.","parameters":{"type":"object","properties":{"order_id":{"type":"string"}},"required":["order_id"],"additionalProperties":false}}}, + {"type":"function","function":{"name":"translate_text","description":"Translate text to a target language.","parameters":{"type":"object","properties":{"text":{"type":"string"},"target_language":{"type":"string"}},"required":["text","target_language"],"additionalProperties":false}}}, + {"type":"function","function":{"name":"plan_route","description":"Plan a route between two places.","parameters":{"type":"object","properties":{"origin":{"type":"string"},"destination":{"type":"string"},"mode":{"type":"string","enum":["car","walk","transit"]}},"required":["origin","destination","mode"],"additionalProperties":false}}}, + {"type":"function","function":{"name":"read_file","description":"Read a UTF-8 text file from the workspace.","parameters":{"type":"object","properties":{"path":{"type":"string"}},"required":["path"],"additionalProperties":false}}} +] +)json"); +} + +json make_request(const std::string & prompt, const json & tools, + int32_t max_tokens) { + return { + {"model", "native-qwen3"}, + {"messages", json::array({{ + {"role", "user"}, + {"content", prompt}, + }})}, + {"tools", tools}, + {"tool_choice", "required"}, + {"temperature", 0}, + {"max_tokens", max_tokens}, + }; +} + +std::vector production_cases() { + return { + {"weather_rome", "What is the weather in Rome? Use Celsius.", + "get_weather", {{"city", "Rome"}, {"unit", "celsius"}}}, + {"weather_boston", "Check Boston weather in Fahrenheit.", + "get_weather", {{"city", "Boston"}, {"unit", "fahrenheit"}}}, + {"stock_nvda", "Get the latest quote for NVDA.", + "get_stock_quote", {{"symbol", "NVDA"}}}, + {"stock_amd", "Look up AMD's current stock quote.", + "get_stock_quote", {{"symbol", "AMD"}}}, + {"search_rocm", + "Search documents for 'ROCm graph replay' and return at most 5 results.", + "search_documents", {{"query", "ROCm graph replay"}, {"limit", 5}}}, + {"search_tool", + "Find the top 3 documents about speculative tool execution.", + "search_documents", + {{"query", "speculative tool execution"}, {"limit", 3}}}, + {"calculate", "Calculate (73.5 * 4) / 7.", + "calculate", {{"expression", "(73.5 * 4) / 7"}}}, + {"order", "Look up order LBX-2048-A.", + "lookup_order", {{"order_id", "LBX-2048-A"}}}, + {"translate", "Translate 'the server is ready' to Italian.", + "translate_text", + {{"text", "the server is ready"}, {"target_language", "Italian"}}}, + {"route", + "Plan a walking route from Termini Station to the Colosseum.", + "plan_route", + {{"origin", "Termini Station"}, + {"destination", "the Colosseum"}, + {"mode", "walk"}}}, + {"read_file", "Read the file docs/production.md.", + "read_file", {{"path", "docs/production.md"}}}, + {"punctuation", + "Search for the exact phrase 'R9700 + Strix: 0731/DS4' with limit 4.", + "search_documents", + {{"query", "R9700 + Strix: 0731/DS4"}, {"limit", 4}}}, + }; +} + +bool arguments_equal(const ordered_json & left, const ordered_json & right) { + // Object key order is irrelevant to tool-call semantics. Convert both + // ordered objects to the canonical map-backed representation first. + return json::parse(left.dump()) == json::parse(right.dump()); +} + +void print_prediction(const char * id, const SemanticToolPrediction & prediction, + const char * expected_name, + const ordered_json & expected_arguments, + const std::string & generated) { + const json output = { + {"id", id}, + {"ok", prediction.ok}, + {"error", prediction.error}, + {"wall_ms", prediction.wall_ms}, + {"name", prediction.call.name}, + {"arguments", prediction.call.arguments}, + {"generated", generated}, + {"name_match", prediction.ok && prediction.call.name == expected_name}, + {"exact_match", prediction.ok && prediction.call.name == expected_name && + arguments_equal(prediction.call.arguments, + expected_arguments)}, + }; + std::printf("%s\n", output.dump().c_str()); + std::fflush(stdout); +} + +} // namespace + +int main(int argc, char ** argv) { + if (argc < 4) { + std::fprintf(stderr, + "usage: %s [prompt]\n", + argv[0]); + return 2; + } + + SemanticToolPredictorConfig config; + config.native_model_path = argv[1]; + config.native_ipc_bin = argv[2]; + config.native_gpu = std::atoi(argv[3]); + config.native_max_ctx = 4096; + config.max_tokens = 96; + + std::string error; + auto predictor = NativeSemanticToolPredictor::create(config, error); + if (!predictor) { + std::fprintf(stderr, "predictor start failed: %s\n", error.c_str()); + return 1; + } + + const json tools = production_tools(); + if (argc > 4) { + const std::string prompt = argv[4]; + std::string generated; + const auto prediction = predictor->predict( + make_request(prompt, tools, config.max_tokens), tools, &generated); + print_prediction("custom", prediction, "", ordered_json::object(), + generated); + return prediction.ok ? 0 : 1; + } + + const std::vector cases = production_cases(); + size_t valid = 0; + size_t name_matches = 0; + size_t exact_matches = 0; + std::vector walls; + for (const Case & test_case : cases) { + std::string generated; + const auto prediction = predictor->predict( + make_request(test_case.prompt, tools, config.max_tokens), tools, + &generated); + print_prediction(test_case.id, prediction, test_case.expected_name, + test_case.expected_arguments, generated); + valid += prediction.ok ? 1 : 0; + name_matches += prediction.ok && + prediction.call.name == test_case.expected_name ? 1 : 0; + exact_matches += prediction.ok && + prediction.call.name == test_case.expected_name && + arguments_equal(prediction.call.arguments, + test_case.expected_arguments) + ? 1 : 0; + walls.push_back(prediction.wall_ms); + } + std::sort(walls.begin(), walls.end()); + const double wall_p50 = walls.empty() + ? 0.0 : 0.5 * (walls[(walls.size() - 1) / 2] + walls[walls.size() / 2]); + const json summary = { + {"requests", cases.size()}, + {"valid", valid}, + {"name_matches", name_matches}, + {"exact_matches", exact_matches}, + {"name_accuracy", cases.empty() ? 0.0 + : static_cast(name_matches) / + static_cast(cases.size())}, + {"exact_accuracy", cases.empty() ? 0.0 + : static_cast(exact_matches) / + static_cast(cases.size())}, + {"wall_p50_ms", wall_p50}, + }; + std::printf("%s\n", summary.dump().c_str()); + return valid == cases.size() && name_matches == cases.size() ? 0 : 1; +} diff --git a/server/test/test_moe_hybrid_storage.cpp b/server/test/test_moe_hybrid_storage.cpp index b57ceb15a..4e7a1d136 100644 --- a/server/test/test_moe_hybrid_storage.cpp +++ b/server/test/test_moe_hybrid_storage.cpp @@ -3,7 +3,6 @@ #include "../src/common/moe_hybrid_storage.h" #include -#include #include using namespace dflash::common; @@ -68,10 +67,7 @@ TEST_CASE(MoeHybridStorageFixture, fractional_route_quota_rounds_over_the_batch) ctx, ids, weights, local_lut, candidate_lut, /*main_slots_x4=*/18, /*main_owner=*/true); REQUIRE(owner_ids != nullptr); - int32_t main_quota = 0; - std::memcpy(&main_quota, - owner_ids->op_params + sizeof(int32_t), - sizeof(main_quota)); + const int32_t main_quota = owner_ids->op_params[1]; REQUIRE(main_quota == 23); ggml_free(ctx); diff --git a/server/test/test_semantic_tool_hint.cpp b/server/test/test_semantic_tool_hint.cpp new file mode 100644 index 000000000..6fa96b792 --- /dev/null +++ b/server/test/test_semantic_tool_hint.cpp @@ -0,0 +1,248 @@ +#include "CppUnitTestFramework.hpp" + +#include "server/semantic_tool_hint.h" + +#include + +namespace { +struct SemanticToolHintFixture {}; +} + +using namespace dflash::common; + +static json weather_tools() { + return json::array({{ + {"type", "function"}, + {"function", { + {"name", "get_weather"}, + {"parameters", { + {"type", "object"}, + {"properties", { + {"city", {{"type", "string"}}}, + {"unit", {{"type", "string"}}}, + }}, + }}, + }}, + }}); +} + +TEST_CASE(SemanticToolHintFixture, parses_qwen_openai_tool_call_semantics) { + const json response = { + {"choices", json::array({{ + {"message", { + {"role", "assistant"}, + {"content", ""}, + {"tool_calls", json::array({{ + {"type", "function"}, + {"function", { + {"name", "get_weather"}, + {"arguments", "{\"city\":\"Rome\",\"unit\":\"celsius\"}"}, + }}, + }})}, + }}, + }})}, + }; + SemanticToolCall call; + std::string error; + CHECK(parse_semantic_tool_prediction( + response, weather_tools(), call, error)); + CHECK(error.empty()); + CHECK(call.name == "get_weather"); + CHECK(call.arguments.dump() == + "{\"city\":\"Rome\",\"unit\":\"celsius\"}"); +} + +TEST_CASE(SemanticToolHintFixture, rejects_unknown_predicted_function) { + const json response = { + {"choices", json::array({{ + {"message", { + {"tool_calls", json::array({{ + {"function", { + {"name", "delete_everything"}, + {"arguments", "{}"}, + }}, + }})}, + }}, + }})}, + }; + SemanticToolCall call; + std::string error; + CHECK(!parse_semantic_tool_prediction( + response, weather_tools(), call, error)); + CHECK(error == "predictor_selected_unknown_function"); +} + +TEST_CASE(SemanticToolHintFixture, materializes_declared_optional_defaults) { + json tools = weather_tools(); + tools[0]["function"]["parameters"]["properties"]["unit"]["default"] = + "celsius"; + SemanticToolCall call; + call.name = "get_weather"; + call.arguments = ordered_json::parse(R"({"city":"Rome"})"); + std::string error; + + CHECK(materialize_declared_tool_defaults(tools, call, error)); + CHECK(error.empty()); + CHECK(call.arguments.dump() == + R"({"city":"Rome","unit":"celsius"})"); +} + +TEST_CASE(SemanticToolHintFixture, explicit_prediction_beats_schema_default) { + json tools = weather_tools(); + tools[0]["function"]["parameters"]["properties"]["unit"]["default"] = + "celsius"; + SemanticToolCall call; + call.name = "get_weather"; + call.arguments = ordered_json::parse( + R"({"city":"Rome","unit":"fahrenheit"})"); + std::string error; + + CHECK(materialize_declared_tool_defaults(tools, call, error)); + CHECK(call.arguments["unit"] == "fahrenheit"); +} + +TEST_CASE(SemanticToolHintFixture, predictor_request_forwards_only_semantics) { + const json target = { + {"model", "deepseek-v4-flash"}, + {"messages", json::array({{{"role", "user"}, {"content", "weather"}}})}, + {"tools", weather_tools()}, + {"tool_choice", "required"}, + {"tool_speculation", {{"name", "unsafe"}}}, + {"prefix_cache", {{"scope", "full"}}}, + }; + const json request = build_semantic_tool_predictor_request( + target, "Qwen3-0.6B", 32); + CHECK(request["model"] == "Qwen3-0.6B"); + CHECK(request["max_tokens"] == 32); + CHECK(request["tool_choice"] == "required"); + CHECK(!request.contains("tool_speculation")); + CHECK(!request.contains("prefix_cache")); +} + +TEST_CASE(SemanticToolHintFixture, native_predictor_config_is_independent_of_http) { + SemanticToolPredictorConfig config; + config.native_model_path = "/models/qwen3-0.6b.gguf"; + config.native_ipc_bin = "/opt/lucebox/backend_ipc_daemon"; + CHECK(config.native_enabled()); + CHECK(!config.http_enabled()); + CHECK(config.enabled()); + CHECK(config.native_runs_before_model()); + CHECK(std::string(native_tool_predictor_schedule_name( + config.native_schedule)) == "before-model"); +} + +TEST_CASE(SemanticToolHintFixture, native_predictor_overlap_is_explicit) { + NativeToolPredictorSchedule schedule = + NativeToolPredictorSchedule::BeforeModel; + CHECK(parse_native_tool_predictor_schedule("overlap", schedule)); + CHECK(schedule == NativeToolPredictorSchedule::Overlap); + CHECK(std::string(native_tool_predictor_schedule_name(schedule)) == + "overlap"); + CHECK(!parse_native_tool_predictor_schedule("automatic", schedule)); +} + +TEST_CASE(SemanticToolHintFixture, native_prompt_uses_qwen_tool_contract) { + const json request = { + {"messages", json::array({{ + {"role", "user"}, + {"content", "What is the weather in Rome?"}, + }})}, + {"tools", weather_tools()}, + {"tool_choice", "required"}, + }; + std::string error; + const std::string prompt = + build_native_semantic_tool_predictor_prompt(request, error); + CHECK(error.empty()); + CHECK(prompt.find("You must call exactly one available function.") != + std::string::npos); + CHECK(prompt.find("get_weather") != std::string::npos); + CHECK(prompt.find("What is the weather in Rome?") != std::string::npos); + CHECK(prompt.find("{\"name\": , \"arguments\":") != + std::string::npos); + CHECK(prompt.find("") == + std::string::npos); + CHECK(prompt.find("\n\n") != std::string::npos); +} + +TEST_CASE(SemanticToolHintFixture, parses_native_qwen_xml_semantics) { + const std::string generated = + "\n" + "\n" + "\nRome\n\n" + "\ncelsius\n\n" + "\n" + ""; + SemanticToolCall call; + std::string error; + CHECK(parse_native_semantic_tool_prediction( + generated, weather_tools(), call, error)); + CHECK(error.empty()); + CHECK(call.name == "get_weather"); + CHECK(call.arguments.dump() == + "{\"city\":\"Rome\",\"unit\":\"celsius\"}"); +} + +TEST_CASE(SemanticToolHintFixture, repairs_qwen_missing_outer_call_brace) { + const json tools = json::array({{ + {"type", "function"}, + {"function", { + {"name", "get_stock_quote"}, + {"parameters", { + {"type", "object"}, + {"properties", {{"symbol", {{"type", "string"}}}}}, + {"required", json::array({"symbol"})}, + }}, + }}, + }}); + const std::string generated = + "\n" + " \"name\": \"get_stock_quote\",\n" + " \"arguments\": {\"symbol\": \"NVDA\"}\n" + "\"}<|im_end|>"; + SemanticToolCall call; + std::string error; + CHECK(parse_native_semantic_tool_prediction( + generated, tools, call, error)); + CHECK(error.empty()); + CHECK(call.name == "get_stock_quote"); + CHECK(call.arguments.dump() == "{\"symbol\":\"NVDA\"}"); +} + +TEST_CASE(SemanticToolHintFixture, maps_bare_arguments_only_for_one_tool) { + const json one_tool = json::array({{ + {"name", "benchmark_cpu_sparse"}, + {"parameters", { + {"type", "object"}, + {"properties", {{"iterations", {{"type", "integer"}}}}}, + {"required", json::array({"iterations"})}, + }}, + }}); + SemanticToolCall call; + std::string error; + CHECK(parse_native_semantic_tool_prediction( + "\n{\"iterations\":172452}\n", + one_tool, call, error)); + CHECK(error.empty()); + CHECK(call.name == "benchmark_cpu_sparse"); + CHECK(call.arguments.dump() == "{\"iterations\":172452}"); + + json two_tools = one_tool; + two_tools.push_back({ + {"name", "other"}, + {"parameters", {{"type", "object"}}}, + }); + CHECK(!parse_native_semantic_tool_prediction( + "{\"iterations\":172452}", two_tools, call, error)); +} + +TEST_CASE(SemanticToolHintFixture, native_parser_rejects_multiple_calls) { + const std::string call = + "Rome" + "celsius"; + SemanticToolCall prediction; + std::string error; + CHECK(!parse_native_semantic_tool_prediction( + call + call, weather_tools(), prediction, error)); + CHECK(error == "native_predictor_response_has_multiple_calls"); +} diff --git a/server/test/test_server_unit.cpp b/server/test/test_server_unit.cpp index 83a7ddc25..f65b67487 100644 --- a/server/test/test_server_unit.cpp +++ b/server/test/test_server_unit.cpp @@ -2295,6 +2295,11 @@ TEST_CASE(ServerUnitFixture, test_pflash_raw_body_preserved) { TEST_ASSERT(req.raw_body["temperature"].get() > 0.6f); } +TEST_CASE(ServerUnitFixture, test_tool_speculation_defaults_to_automatic_prediction) { + ParsedRequest req; + TEST_ASSERT(req.automatic_tool_speculation_enabled); +} + TEST_CASE(ServerUnitFixture, test_parse_request_sampler_applies_defaults_and_overrides) { SamplingDefaults defaults; defaults.has_temperature = true; @@ -2552,6 +2557,42 @@ TEST_CASE(ServerUnitFixture, test_deepseek4_render_empty_chat_gen_prompt) { TEST_ASSERT(out == expected); } +TEST_CASE(ServerUnitFixture, test_deepseek4_render_required_tool_instructions) { + std::vector msgs = { + {"user", "What is the weather?", ""}, + }; + const std::string tools = + R"([{"type":"function","function":{"name":"weather.get","parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}}])"; + const std::string out = render_chat_template( + msgs, ChatFormat::DEEPSEEK4, + /*add_generation_prompt=*/true, + /*enable_thinking=*/false, + tools, + /*tool_call_required=*/true); + + TEST_ASSERT(out.find("\n" + tools + "\n") != + std::string::npos); + TEST_ASSERT(out.find("") != std::string::npos); + TEST_ASSERT(out.find("MUST call exactly one") != std::string::npos); + TEST_ASSERT(out.find("<｜User｜>What is the weather?") != + std::string::npos); + const std::string suffix = "<｜Assistant｜>"; + TEST_ASSERT(out.size() >= suffix.size()); + TEST_ASSERT(out.compare(out.size() - suffix.size(), suffix.size(), suffix) == 0); +} + +TEST_CASE(ServerUnitFixture, test_deepseek4_auto_tool_is_not_forced) { + std::vector msgs = {{"user", "Hello", ""}}; + const std::string tools = + R"([{"type":"function","function":{"name":"weather.get","parameters":{"type":"object","properties":{}}}}])"; + const std::string out = render_chat_template( + msgs, ChatFormat::DEEPSEEK4, true, false, tools, + /*tool_call_required=*/false); + + TEST_ASSERT(out.find("If a function is applicable") != std::string::npos); + TEST_ASSERT(out.find("MUST call exactly one") == std::string::npos); +} + TEST_CASE(ServerUnitFixture, test_jinja_render_basic) { std::vector msgs = { {"system", "you are helpful", ""}, @@ -4629,6 +4670,9 @@ TEST_CASE(ServerUnitFixture, test_props_tool_speculation_shape) { TEST_ASSERT(body.contains("tool_speculation")); const json & disabled = body["tool_speculation"]; TEST_ASSERT(!disabled["enabled"].get()); + TEST_ASSERT(!disabled["automatic_prediction_enabled"].get()); + TEST_ASSERT(disabled["prediction_source"].is_null()); + TEST_ASSERT(disabled["prediction_confidence"].is_null()); TEST_ASSERT(disabled["profile_status"].is_null()); TEST_ASSERT(disabled["executor_contract"].is_null()); TEST_ASSERT(disabled["protocol"].get() == @@ -4673,6 +4717,8 @@ TEST_CASE(ServerUnitFixture, test_props_tool_speculation_shape) { body = build_props_body(cfg, pc, tm); const json & enabled = body["tool_speculation"]; TEST_ASSERT(enabled["enabled"].get()); + TEST_ASSERT(!enabled["automatic_prediction_enabled"].get()); + TEST_ASSERT(!enabled["predictor_decode_isolated"].get()); TEST_ASSERT(enabled["profile_status"].get() == "qualified"); TEST_ASSERT(enabled["allowed_tools"] == json::array({"lookup"})); TEST_ASSERT(enabled["preserves_token_speculation"].get()); @@ -4694,6 +4740,45 @@ TEST_CASE(ServerUnitFixture, test_props_tool_speculation_shape) { TEST_ASSERT(!enabled["profile_lanes"][0] ["requires_unique_expert_ownership"].get()); + cfg.semantic_tool_predictor.native_model_path = "/models/qwen3-0.6b.gguf"; + cfg.semantic_tool_predictor.native_ipc_bin = "/bin/backend-ipc"; + body = build_props_body(cfg, pc, tm); + const json & automatic = body["tool_speculation"]; + TEST_ASSERT(automatic["automatic_prediction_enabled"].get()); + TEST_ASSERT(!automatic["requires_client_support"].get()); + TEST_ASSERT(automatic["prediction_source"].get() == + "native-qwen3"); + TEST_ASSERT(std::fabs( + automatic["prediction_confidence"].get() - 0.75) < 1e-9); + TEST_ASSERT(automatic["predictor_schedule"].get() == + "before-model"); + TEST_ASSERT(automatic["predictor_decode_isolated"].get()); + + cfg.semantic_tool_predictor.native_schedule = + NativeToolPredictorSchedule::Overlap; + body = build_props_body(cfg, pc, tm); + const json & overlapping = body["tool_speculation"]; + TEST_ASSERT(overlapping["predictor_schedule"].get() == + "overlap"); + TEST_ASSERT(!overlapping["predictor_decode_isolated"].get()); + cfg.semantic_tool_predictor.native_schedule = + NativeToolPredictorSchedule::BeforeModel; + + cfg.semantic_tool_predictor.native_model_path.clear(); + cfg.semantic_tool_predictor.native_ipc_bin.clear(); + cfg.semantic_tool_predictor.url = "http://127.0.0.1:9000/v1/chat/completions"; + cfg.semantic_tool_predictor.model = "remote-predictor"; + body = build_props_body(cfg, pc, tm); + const json & remote = body["tool_speculation"]; + TEST_ASSERT(remote["automatic_prediction_enabled"].get()); + TEST_ASSERT(remote["prediction_source"].get() == + "remote-predictor"); + TEST_ASSERT(!remote["predictor_decode_isolated"].get()); + cfg.semantic_tool_predictor.url.clear(); + cfg.semantic_tool_predictor.model.clear(); + cfg.semantic_tool_predictor.native_model_path = "/models/qwen3-0.6b.gguf"; + cfg.semantic_tool_predictor.native_ipc_bin = "/bin/backend-ipc"; + cfg.tool_speculation.cpu_affinity = {14, 30}; cfg.tool_speculation.model_cpu_affinity = {0, 1, 2, 3}; cfg.tool_speculation.cpu_affinity_isolated = true; diff --git a/server/test/test_tool_speculation.cpp b/server/test/test_tool_speculation.cpp index ed377f74c..e9686e611 100644 --- a/server/test/test_tool_speculation.cpp +++ b/server/test/test_tool_speculation.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -28,6 +29,7 @@ using dflash::common::ToolSpeculationExecution; using dflash::common::ToolSpeculationExecutor; using dflash::common::ToolSpeculationPolicy; using dflash::common::ToolSpeculationPrediction; +using dflash::common::build_tool_speculation_prediction; using dflash::common::json; using dflash::common::parse_tool_speculation_prediction; using dflash::common::parse_tool_speculation_cpu_affinity; @@ -215,6 +217,21 @@ TEST_CASE(ToolSpeculationFixture, canonical_identity_ignores_argument_order) { CHECK(first.arguments_json == R"({"a":1,"b":2})"); } +TEST_CASE(ToolSpeculationFixture, engine_prediction_uses_canonical_boundary) { + ToolSpeculationPrediction value; + std::string error; + CHECK(build_tool_speculation_prediction( + "lookup", json{{"b", 2}, {"a", 1}}, 0.75, value, error)); + CHECK(error.empty()); + CHECK(value.call.name == "lookup"); + CHECK(value.call.arguments_json == "{\"a\":1,\"b\":2}"); + CHECK(std::fabs(value.confidence - 0.75) < 1e-9); + CHECK(!build_tool_speculation_prediction( + "lookup", json::array(), 0.75, value, error)); + CHECK(!build_tool_speculation_prediction( + "lookup", json::object(), 1.01, value, error)); +} + TEST_CASE(ToolSpeculationFixture, prediction_requires_declared_tool) { const json tools = json::array({{ {"type", "function"}, From c2720c9f609d2d51cea9fcb897dd22cbfb79d5fc Mon Sep 17 00:00:00 2001 From: mrciffa <49000955+davide221@users.noreply.github.com> Date: Mon, 17 Aug 2026 22:02:36 +0200 Subject: [PATCH 05/11] chore(bench): drop superseded raw tool-spec results --- .../lucebox5-cpu-lane-qualification.json | 1839 ------------- .../results/lucebox5-cpu-native-20pairs.json | 2346 ----------------- 2 files changed, 4185 deletions(-) delete mode 100644 optimizations/ooo_spec_lucebox5_cpu/results/lucebox5-cpu-lane-qualification.json delete mode 100644 optimizations/ooo_spec_lucebox5_cpu/results/lucebox5-cpu-native-20pairs.json diff --git a/optimizations/ooo_spec_lucebox5_cpu/results/lucebox5-cpu-lane-qualification.json b/optimizations/ooo_spec_lucebox5_cpu/results/lucebox5-cpu-lane-qualification.json deleted file mode 100644 index 70c6c50d5..000000000 --- a/optimizations/ooo_spec_lucebox5_cpu/results/lucebox5-cpu-lane-qualification.json +++ /dev/null @@ -1,1839 +0,0 @@ -{ - "calibration": { - "model_samples": [ - { - "accept_rate": 0.625, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 7, - "decode_ms": 728.4, - "decode_tokens_per_sec": 9.6, - "model_compute_ms": 3426.5, - "prefill_ms": 2698.1, - "request_wall_ms": 3444.775058000232, - "speculation": null, - "tool_call": { - "arguments": { - "iterations": 20 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "a97210b345373f933eeb50349895292971f6d1bfd676f63e999ab9f178cdeba2" - }, - { - "accept_rate": 0.625, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 7, - "decode_ms": 346.4, - "decode_tokens_per_sec": 20.2, - "model_compute_ms": 3728.5, - "prefill_ms": 3382.1, - "request_wall_ms": 3730.295108995051, - "speculation": null, - "tool_call": { - "arguments": { - "iterations": 20 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "a97210b345373f933eeb50349895292971f6d1bfd676f63e999ab9f178cdeba2" - }, - { - "accept_rate": 0.625, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 7, - "decode_ms": 342.1, - "decode_tokens_per_sec": 20.5, - "model_compute_ms": 2619.1, - "prefill_ms": 2277.0, - "request_wall_ms": 2620.606353986659, - "speculation": null, - "tool_call": { - "arguments": { - "iterations": 20 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "a97210b345373f933eeb50349895292971f6d1bfd676f63e999ab9f178cdeba2" - }, - { - "accept_rate": 0.625, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 7, - "decode_ms": 342.3, - "decode_tokens_per_sec": 20.4, - "model_compute_ms": 2619.2000000000003, - "prefill_ms": 2276.9, - "request_wall_ms": 2620.5322190071456, - "speculation": null, - "tool_call": { - "arguments": { - "iterations": 20 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "a97210b345373f933eeb50349895292971f6d1bfd676f63e999ab9f178cdeba2" - }, - { - "accept_rate": 0.625, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 7, - "decode_ms": 342.5, - "decode_tokens_per_sec": 20.4, - "model_compute_ms": 2620.6, - "prefill_ms": 2278.1, - "request_wall_ms": 2622.0629360032035, - "speculation": null, - "tool_call": { - "arguments": { - "iterations": 20 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "a97210b345373f933eeb50349895292971f6d1bfd676f63e999ab9f178cdeba2" - } - ], - "selected_iterations": 172452, - "steps": [ - { - "iterations": 20, - "samples": [ - { - "result": { - "checksum": "18180057682806790565", - "compute_ms": 1.4058, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 20, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 4.593224002746865 - }, - { - "result": { - "checksum": "18180057682806790565", - "compute_ms": 1.296406, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 20, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 3.755755999009125 - }, - { - "result": { - "checksum": "18180057682806790565", - "compute_ms": 0.823681, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 20, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 2.570759999798611 - } - ], - "tool_wall_p50_ms": 3.755755999009125 - }, - { - "iterations": 13963, - "samples": [ - { - "result": { - "checksum": "13677167202453426650", - "compute_ms": 249.549945, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 13963, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 251.17609799781349 - }, - { - "result": { - "checksum": "13677167202453426650", - "compute_ms": 210.351586, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 13963, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 212.30194400413893 - }, - { - "result": { - "checksum": "13677167202453426650", - "compute_ms": 210.386741, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 13963, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 212.24752299895044 - } - ], - "tool_wall_p50_ms": 212.30194400413893 - }, - { - "iterations": 172452, - "samples": [ - { - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2552.662603, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 2554.3570239969995 - }, - { - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2546.354615, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 2548.043864997453 - }, - { - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2546.93318, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 2548.6412460013526 - } - ], - "tool_wall_p50_ms": 2548.6412460013526 - } - ], - "target_model_request_p50_ms": 2622.0629360032035 - }, - "config": { - "binary": "/home/lucebox5/tool-spec-cpu-20260813/cpu_sparse_tool_executor", - "fixed_sparse_shape": { - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2 - }, - "max_tokens": 32, - "pairs": 12, - "seed": 814, - "tool_arguments": { - "iterations": 172452 - }, - "tool_cpus": [ - 14, - 15, - 30, - 31 - ], - "url": "http://127.0.0.1:18145/v1/chat/completions", - "warmups": 2 - }, - "host": "lucebox5", - "misses": [ - { - "authoritative_tool": { - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2607.356476, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 2610.2510019991314 - }, - "mode": "miss", - "model": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 475.8, - "decode_tokens_per_sec": 16.8, - "model_compute_ms": 2893.1000000000004, - "prefill_ms": 2417.3, - "request_wall_ms": 2895.4200049920473, - "speculation": null, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" - }, - "private_result_exposed": false, - "task_ms": 5507.685866992688 - }, - { - "authoritative_tool": { - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2634.827473, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 2637.8311429871246 - }, - "mode": "miss", - "model": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 475.1, - "decode_tokens_per_sec": 16.8, - "model_compute_ms": 2913.1, - "prefill_ms": 2438.0, - "request_wall_ms": 2915.902043998358, - "speculation": null, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" - }, - "private_result_exposed": false, - "task_ms": 5555.7190620020265 - }, - { - "authoritative_tool": { - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2596.870639, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 2599.092707008822 - }, - "mode": "miss", - "model": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 475.4, - "decode_tokens_per_sec": 16.8, - "model_compute_ms": 2914.5, - "prefill_ms": 2439.1, - "request_wall_ms": 2917.0129179983633, - "speculation": null, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" - }, - "private_result_exposed": false, - "task_ms": 5517.983499012189 - }, - { - "authoritative_tool": { - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2717.568347, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 2720.183129000361 - }, - "mode": "miss", - "model": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 475.4, - "decode_tokens_per_sec": 16.8, - "model_compute_ms": 2912.8, - "prefill_ms": 2437.4, - "request_wall_ms": 2915.4062980087474, - "speculation": null, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" - }, - "private_result_exposed": false, - "task_ms": 5637.134058008087 - }, - { - "authoritative_tool": { - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2611.866018, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 2614.5963160088286 - }, - "mode": "miss", - "model": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 475.3, - "decode_tokens_per_sec": 16.8, - "model_compute_ms": 2913.0, - "prefill_ms": 2437.7, - "request_wall_ms": 2915.487136997399, - "speculation": null, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" - }, - "private_result_exposed": false, - "task_ms": 5531.995229001041 - } - ], - "model_cpu_affinity": [ - 0, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - 23, - 24, - 25, - 26, - 27, - 28, - 29 - ], - "model_pid": 498874, - "pairs": [ - { - "arm_order": [ - "speculative", - "control" - ], - "control": { - "mode": "control", - "model": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 477.4, - "decode_tokens_per_sec": 16.8, - "model_compute_ms": 2891.3, - "prefill_ms": 2413.9, - "request_wall_ms": 2892.969884997001, - "speculation": null, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" - }, - "task_ms": 5524.237097008154, - "tool": { - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2627.687774, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 2631.0234859993216 - } - }, - "pair_index": 0, - "speculative": { - "mode": "speculative", - "model": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 474.6, - "decode_tokens_per_sec": 16.9, - "model_compute_ms": 2888.0, - "prefill_ms": 2413.4, - "request_wall_ms": 2889.660949993413, - "speculation": null, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" - }, - "task_ms": 2891.6836440039333, - "tool": { - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2734.337179, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 2891.6460639884463 - } - } - }, - { - "arm_order": [ - "speculative", - "control" - ], - "control": { - "mode": "control", - "model": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 477.1, - "decode_tokens_per_sec": 16.8, - "model_compute_ms": 2892.2999999999997, - "prefill_ms": 2415.2, - "request_wall_ms": 2893.8748500077054, - "speculation": null, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" - }, - "task_ms": 5493.723748004413, - "tool": { - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2597.238064, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 2599.693889002083 - } - }, - "pair_index": 1, - "speculative": { - "mode": "speculative", - "model": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 474.7, - "decode_tokens_per_sec": 16.9, - "model_compute_ms": 2909.1, - "prefill_ms": 2434.4, - "request_wall_ms": 2911.0499909875216, - "speculation": null, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" - }, - "task_ms": 2912.997834995622, - "tool": { - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2714.367807, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 2912.963351001963 - } - } - }, - { - "arm_order": [ - "speculative", - "control" - ], - "control": { - "mode": "control", - "model": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 477.2, - "decode_tokens_per_sec": 16.8, - "model_compute_ms": 2892.5, - "prefill_ms": 2415.3, - "request_wall_ms": 2893.6694570002146, - "speculation": null, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" - }, - "task_ms": 5522.214145996259, - "tool": { - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2625.622342, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 2628.3829860039987 - } - }, - "pair_index": 2, - "speculative": { - "mode": "speculative", - "model": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 474.8, - "decode_tokens_per_sec": 16.8, - "model_compute_ms": 2909.8, - "prefill_ms": 2435.0, - "request_wall_ms": 2912.0525879989145, - "speculation": null, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" - }, - "task_ms": 2913.6232059972826, - "tool": { - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2743.90931, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 2913.592108001467 - } - } - }, - { - "arm_order": [ - "speculative", - "control" - ], - "control": { - "mode": "control", - "model": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 477.1, - "decode_tokens_per_sec": 16.8, - "model_compute_ms": 2909.0, - "prefill_ms": 2431.9, - "request_wall_ms": 2911.317157006124, - "speculation": null, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" - }, - "task_ms": 5842.6958240015665, - "tool": { - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2927.840256, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 2931.1076189915184 - } - }, - "pair_index": 3, - "speculative": { - "mode": "speculative", - "model": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 475.2, - "decode_tokens_per_sec": 16.8, - "model_compute_ms": 2921.5, - "prefill_ms": 2446.3, - "request_wall_ms": 2923.7128459935775, - "speculation": null, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" - }, - "task_ms": 2990.422590999515, - "tool": { - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2988.352859, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 2990.3616270021303 - } - } - }, - { - "arm_order": [ - "control", - "speculative" - ], - "control": { - "mode": "control", - "model": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 477.4, - "decode_tokens_per_sec": 16.8, - "model_compute_ms": 2912.7000000000003, - "prefill_ms": 2435.3, - "request_wall_ms": 2915.1544740016107, - "speculation": null, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" - }, - "task_ms": 5531.82867099531, - "tool": { - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2613.478577, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 2616.4246090047527 - } - }, - "pair_index": 4, - "speculative": { - "mode": "speculative", - "model": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 475.0, - "decode_tokens_per_sec": 16.8, - "model_compute_ms": 2910.8, - "prefill_ms": 2435.8, - "request_wall_ms": 2913.0806419998407, - "speculation": null, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" - }, - "task_ms": 2914.699681001366, - "tool": { - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2726.278932, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 2914.6627919981256 - } - } - }, - { - "arm_order": [ - "speculative", - "control" - ], - "control": { - "mode": "control", - "model": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 477.2, - "decode_tokens_per_sec": 16.8, - "model_compute_ms": 2891.5, - "prefill_ms": 2414.3, - "request_wall_ms": 2892.994167006691, - "speculation": null, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" - }, - "task_ms": 5518.224094994366, - "tool": { - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2622.666621, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 2625.0104759965325 - } - }, - "pair_index": 5, - "speculative": { - "mode": "speculative", - "model": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 474.7, - "decode_tokens_per_sec": 16.9, - "model_compute_ms": 2890.1, - "prefill_ms": 2415.4, - "request_wall_ms": 2891.9894639984705, - "speculation": null, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" - }, - "task_ms": 2893.9604710030835, - "tool": { - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2770.130032, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 2893.926097007352 - } - } - }, - { - "arm_order": [ - "control", - "speculative" - ], - "control": { - "mode": "control", - "model": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 477.4, - "decode_tokens_per_sec": 16.8, - "model_compute_ms": 2914.5, - "prefill_ms": 2437.1, - "request_wall_ms": 2917.0066920050886, - "speculation": null, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" - }, - "task_ms": 5918.36711500946, - "tool": { - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2998.242149, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 3001.084837989765 - } - }, - "pair_index": 6, - "speculative": { - "mode": "speculative", - "model": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 475.0, - "decode_tokens_per_sec": 16.8, - "model_compute_ms": 2911.9, - "prefill_ms": 2436.9, - "request_wall_ms": 2914.4634409894934, - "speculation": null, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" - }, - "task_ms": 2916.5130859910278, - "tool": { - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2736.112772, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 2916.4474730059737 - } - } - }, - { - "arm_order": [ - "speculative", - "control" - ], - "control": { - "mode": "control", - "model": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 477.5, - "decode_tokens_per_sec": 16.8, - "model_compute_ms": 2910.4, - "prefill_ms": 2432.9, - "request_wall_ms": 2912.770160997752, - "speculation": null, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" - }, - "task_ms": 5521.725176004111, - "tool": { - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2606.126825, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 2608.781943010399 - } - }, - "pair_index": 7, - "speculative": { - "mode": "speculative", - "model": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 475.2, - "decode_tokens_per_sec": 16.8, - "model_compute_ms": 2892.2, - "prefill_ms": 2417.0, - "request_wall_ms": 2894.1632219939493, - "speculation": null, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" - }, - "task_ms": 3026.5094359929208, - "tool": { - "result": { - "checksum": "9705095564492366076", - "compute_ms": 3023.66315, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 3026.4584609976737 - } - } - }, - { - "arm_order": [ - "control", - "speculative" - ], - "control": { - "mode": "control", - "model": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 476.4, - "decode_tokens_per_sec": 16.8, - "model_compute_ms": 2911.8, - "prefill_ms": 2435.4, - "request_wall_ms": 2914.0317879937356, - "speculation": null, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" - }, - "task_ms": 5525.245836994145, - "tool": { - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2608.567377, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 2611.0557520005386 - } - }, - "pair_index": 8, - "speculative": { - "mode": "speculative", - "model": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 475.0, - "decode_tokens_per_sec": 16.8, - "model_compute_ms": 2911.4, - "prefill_ms": 2436.4, - "request_wall_ms": 2913.8368099957006, - "speculation": null, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" - }, - "task_ms": 2915.164353995351, - "tool": { - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2755.945269, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 2915.1280960068107 - } - } - }, - { - "arm_order": [ - "speculative", - "control" - ], - "control": { - "mode": "control", - "model": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 477.5, - "decode_tokens_per_sec": 16.8, - "model_compute_ms": 2892.5, - "prefill_ms": 2415.0, - "request_wall_ms": 2894.126478000544, - "speculation": null, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" - }, - "task_ms": 5507.012105998001, - "tool": { - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2610.119884, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 2612.722923993715 - } - }, - "pair_index": 9, - "speculative": { - "mode": "speculative", - "model": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 475.1, - "decode_tokens_per_sec": 16.8, - "model_compute_ms": 2890.9, - "prefill_ms": 2415.8, - "request_wall_ms": 2893.0320760118775, - "speculation": null, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" - }, - "task_ms": 2894.950055007939, - "tool": { - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2749.14522, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 2894.9089880043175 - } - } - }, - { - "arm_order": [ - "control", - "speculative" - ], - "control": { - "mode": "control", - "model": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 477.5, - "decode_tokens_per_sec": 16.8, - "model_compute_ms": 2914.7, - "prefill_ms": 2437.2, - "request_wall_ms": 2917.2719510097522, - "speculation": null, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" - }, - "task_ms": 5515.812735000509, - "tool": { - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2595.765081, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 2598.3751849998953 - } - }, - "pair_index": 10, - "speculative": { - "mode": "speculative", - "model": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 475.5, - "decode_tokens_per_sec": 16.8, - "model_compute_ms": 2913.4, - "prefill_ms": 2437.9, - "request_wall_ms": 2915.5481519992463, - "speculation": null, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" - }, - "task_ms": 3491.498399002012, - "tool": { - "result": { - "checksum": "9705095564492366076", - "compute_ms": 3489.411735, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 3491.450679008267 - } - } - }, - { - "arm_order": [ - "control", - "speculative" - ], - "control": { - "mode": "control", - "model": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 477.5, - "decode_tokens_per_sec": 16.8, - "model_compute_ms": 2914.5, - "prefill_ms": 2437.0, - "request_wall_ms": 2917.092024013982, - "speculation": null, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" - }, - "task_ms": 5554.181414001505, - "tool": { - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2634.129552, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 2636.9060069991974 - } - }, - "pair_index": 11, - "speculative": { - "mode": "speculative", - "model": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 475.4, - "decode_tokens_per_sec": 16.8, - "model_compute_ms": 2912.4, - "prefill_ms": 2437.0, - "request_wall_ms": 2914.609836996533, - "speculation": null, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101" - }, - "task_ms": 2916.817977995379, - "tool": { - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2758.995074, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "wall_ms": 2916.75399900123 - } - } - } - ], - "phase": "qualification", - "profile": { - "executor": "child_process_cpu_affinity", - "path_summary": { - "100": { - "accelerator_relation": "non_accelerator", - "decode_interference_qualified": true, - "hit": { - "control_task_mean_ms": 5581.2723303339835, - "model_slowdown_percent": 0.11341375399527287, - "speculative_task_mean_ms": 2973.236727998786 - }, - "miss": { - "control_task_mean_ms": 5581.2723303339835, - "model_slowdown_percent": 0.11341375399527287, - "speculative_task_mean_ms": 5550.103543003206 - } - } - }, - "profile_kind": "disjoint_strix_cpu_sparse_compute", - "profile_status": "qualified", - "qualification": { - "checks": { - "direct_speedup": true, - "disjoint_cpu_affinity": true, - "ds4_active": true, - "identical_model_outputs": true, - "identical_tool_outputs": true, - "model_slowdown": true, - "private_miss_result_hidden": true - }, - "host": "lucebox5", - "model_cpu_affinity": [ - 0, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - 23, - 24, - 25, - 26, - 27, - 28, - 29 - ], - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ] - } - }, - "summary": { - "checks": { - "direct_speedup": true, - "disjoint_cpu_affinity": true, - "ds4_active": true, - "identical_model_outputs": true, - "identical_tool_outputs": true, - "model_slowdown": true, - "private_miss_result_hidden": true - }, - "control_model_compute_p50_ms": 2909.7, - "control_task_p50_ms": 5523.225621502206, - "control_tool_wall_p50_ms": 2620.7175425006426, - "direct_exact_hit_speedup": 1.8948042658786695, - "median_accept_rate": 0.4166666567325592, - "miss_task_p50_ms": 5531.995229001041, - "model_compute_slowdown_percent": 0.11341375399527287, - "overlap_model_compute_p50_ms": 2910.3, - "overlap_task_p50_ms": 2914.9320174983586, - "overlap_tool_wall_p50_ms": 2914.895444002468, - "pairs": 12, - "passed": true - }, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ] -} diff --git a/optimizations/ooo_spec_lucebox5_cpu/results/lucebox5-cpu-native-20pairs.json b/optimizations/ooo_spec_lucebox5_cpu/results/lucebox5-cpu-native-20pairs.json deleted file mode 100644 index 393f209cf..000000000 --- a/optimizations/ooo_spec_lucebox5_cpu/results/lucebox5-cpu-native-20pairs.json +++ /dev/null @@ -1,2346 +0,0 @@ -{ - "config": { - "binary": "/home/lucebox5/tool-spec-cpu-20260813/cpu_sparse_tool_executor", - "fixed_sparse_shape": { - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2 - }, - "max_tokens": 32, - "pairs": 20, - "seed": 814, - "tool_arguments": { - "iterations": 172452 - }, - "tool_cpus": [ - 14, - 15, - 30, - 31 - ], - "url": "http://127.0.0.1:18145/v1/chat/completions", - "warmups": 2 - }, - "correctness_passed": true, - "host": "lucebox5", - "methodology": { - "commit": "result exposed only after exact canonical call match", - "control": "model request followed by the identical CPU-pinned sparse tool", - "pairing": "randomized arm order within every warm pair", - "speculative": "engine starts the identical CPU-pinned sparse tool before DS4 generation" - }, - "miss_check": { - "passed": true, - "private_result_exposed": false, - "reason": "invocation_mismatch", - "status": "miss" - }, - "pairs": [ - { - "arm_order": [ - "speculative", - "control" - ], - "control": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 476.8, - "decode_tokens_per_sec": 16.8, - "mode": "control", - "model_compute_ms": 2886.7000000000003, - "prefill_ms": 2409.9, - "request_wall_ms": 2887.7563690039096, - "speculation": null, - "task_ms": 5497.278031994938, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2606.721652, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2609.3294839956798 - }, - "pair_index": 0, - "speculative": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 473.5, - "decode_tokens_per_sec": 16.9, - "mode": "speculative", - "model_compute_ms": 2884.6, - "prefill_ms": 2411.1, - "request_wall_ms": 2886.079392003012, - "speculation": { - "accelerator_relation": "non_accelerator", - "call_id": "call_de0927ff226ce042cdf4d97b", - "commit_signal_sent": false, - "commit_wait_ms": 0.018996, - "confidence": 1.0, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "cpu_affinity_isolated": true, - "decode_interference_qualified": true, - "executor_wall_ms": 2885.142969, - "expected_speedup": 1.8771705185044594, - "prediction": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "protocol": "dflash.tool-speculation.v1", - "resource_percentage": 100, - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2716.64606, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "status": "hit" - }, - "task_ms": 2886.079392003012, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2716.64606, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2885.142969 - } - }, - { - "arm_order": [ - "speculative", - "control" - ], - "control": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 476.9, - "decode_tokens_per_sec": 16.8, - "mode": "control", - "model_compute_ms": 2888.3, - "prefill_ms": 2411.4, - "request_wall_ms": 2889.4118949974654, - "speculation": null, - "task_ms": 5495.428775990149, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2602.850033, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2605.80943300738 - }, - "pair_index": 1, - "speculative": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 473.8, - "decode_tokens_per_sec": 16.9, - "mode": "speculative", - "model_compute_ms": 2903.6000000000004, - "prefill_ms": 2429.8, - "request_wall_ms": 2936.8787970015546, - "speculation": { - "accelerator_relation": "non_accelerator", - "call_id": "call_a780efe5dceb275f785f0531", - "commit_signal_sent": false, - "commit_wait_ms": 0.015519, - "confidence": 1.0, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "cpu_affinity_isolated": true, - "decode_interference_qualified": true, - "executor_wall_ms": 2904.636247, - "expected_speedup": 1.8771705185044594, - "prediction": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "protocol": "dflash.tool-speculation.v1", - "resource_percentage": 100, - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2715.15693, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "status": "hit" - }, - "task_ms": 2936.8787970015546, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2715.15693, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2904.636247 - } - }, - { - "arm_order": [ - "speculative", - "control" - ], - "control": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 476.7, - "decode_tokens_per_sec": 16.8, - "mode": "control", - "model_compute_ms": 2902.5, - "prefill_ms": 2425.8, - "request_wall_ms": 2904.134411000996, - "speculation": null, - "task_ms": 5490.023413003655, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2582.769682, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2585.659131000284 - }, - "pair_index": 2, - "speculative": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 474.5, - "decode_tokens_per_sec": 16.9, - "mode": "speculative", - "model_compute_ms": 2905.9, - "prefill_ms": 2431.4, - "request_wall_ms": 2953.2264759909594, - "speculation": { - "accelerator_relation": "non_accelerator", - "call_id": "call_22cf7759b3f52086d81a1a87", - "commit_signal_sent": true, - "commit_wait_ms": 44.199584, - "confidence": 1.0, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "cpu_affinity_isolated": true, - "decode_interference_qualified": true, - "executor_wall_ms": 2951.125814, - "expected_speedup": 1.8771705185044594, - "prediction": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "protocol": "dflash.tool-speculation.v1", - "resource_percentage": 100, - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2949.894469, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "status": "hit" - }, - "task_ms": 2953.2264759909594, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2949.894469, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2951.125814 - } - }, - { - "arm_order": [ - "speculative", - "control" - ], - "control": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 476.8, - "decode_tokens_per_sec": 16.8, - "mode": "control", - "model_compute_ms": 2888.2000000000003, - "prefill_ms": 2411.4, - "request_wall_ms": 2889.3942540016724, - "speculation": null, - "task_ms": 5515.519638996921, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2623.90572, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2625.9387350000907 - }, - "pair_index": 3, - "speculative": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 474.5, - "decode_tokens_per_sec": 16.9, - "mode": "speculative", - "model_compute_ms": 2904.5, - "prefill_ms": 2430.0, - "request_wall_ms": 2907.0747140067397, - "speculation": { - "accelerator_relation": "non_accelerator", - "call_id": "call_2fefe53d3cad565ed33dcfc1", - "commit_signal_sent": false, - "commit_wait_ms": 0.014087, - "confidence": 1.0, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "cpu_affinity_isolated": true, - "decode_interference_qualified": true, - "executor_wall_ms": 2905.407734, - "expected_speedup": 1.8771705185044594, - "prediction": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "protocol": "dflash.tool-speculation.v1", - "resource_percentage": 100, - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2757.805857, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "status": "hit" - }, - "task_ms": 2907.0747140067397, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2757.805857, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2905.407734 - } - }, - { - "arm_order": [ - "control", - "speculative" - ], - "control": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 477.1, - "decode_tokens_per_sec": 16.8, - "mode": "control", - "model_compute_ms": 2908.9, - "prefill_ms": 2431.8, - "request_wall_ms": 2910.745012006373, - "speculation": null, - "task_ms": 5511.091899999883, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2597.192803, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2600.07576100179 - }, - "pair_index": 4, - "speculative": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 474.5, - "decode_tokens_per_sec": 16.9, - "mode": "speculative", - "model_compute_ms": 2906.6, - "prefill_ms": 2432.1, - "request_wall_ms": 2909.174690998043, - "speculation": { - "accelerator_relation": "non_accelerator", - "call_id": "call_badafb42708c7fd7b1cb8f88", - "commit_signal_sent": false, - "commit_wait_ms": 0.019236, - "confidence": 1.0, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "cpu_affinity_isolated": true, - "decode_interference_qualified": true, - "executor_wall_ms": 2907.57131, - "expected_speedup": 1.8771705185044594, - "prediction": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "protocol": "dflash.tool-speculation.v1", - "resource_percentage": 100, - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2727.222714, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "status": "hit" - }, - "task_ms": 2909.174690998043, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2727.222714, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2907.57131 - } - }, - { - "arm_order": [ - "speculative", - "control" - ], - "control": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 477.8, - "decode_tokens_per_sec": 16.7, - "mode": "control", - "model_compute_ms": 2934.2000000000003, - "prefill_ms": 2456.4, - "request_wall_ms": 2935.454065009253, - "speculation": null, - "task_ms": 5563.332241988974, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2624.312122, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2627.6193329977104 - }, - "pair_index": 5, - "speculative": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 474.7, - "decode_tokens_per_sec": 16.9, - "mode": "speculative", - "model_compute_ms": 2884.5, - "prefill_ms": 2409.8, - "request_wall_ms": 2886.354169007973, - "speculation": { - "accelerator_relation": "non_accelerator", - "call_id": "call_75f7f3e0c38ee6c0b3928dbc", - "commit_signal_sent": false, - "commit_wait_ms": 0.014156, - "confidence": 1.0, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "cpu_affinity_isolated": true, - "decode_interference_qualified": true, - "executor_wall_ms": 2885.126902, - "expected_speedup": 1.8771705185044594, - "prediction": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "protocol": "dflash.tool-speculation.v1", - "resource_percentage": 100, - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2751.568559, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "status": "hit" - }, - "task_ms": 2886.354169007973, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2751.568559, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2885.126902 - } - }, - { - "arm_order": [ - "control", - "speculative" - ], - "control": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 477.4, - "decode_tokens_per_sec": 16.8, - "mode": "control", - "model_compute_ms": 2910.0, - "prefill_ms": 2432.6, - "request_wall_ms": 2914.380813992466, - "speculation": null, - "task_ms": 5517.575254998519, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2599.810015, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2602.895701988018 - }, - "pair_index": 6, - "speculative": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 474.5, - "decode_tokens_per_sec": 16.9, - "mode": "speculative", - "model_compute_ms": 2904.8, - "prefill_ms": 2430.3, - "request_wall_ms": 2910.256342001958, - "speculation": { - "accelerator_relation": "non_accelerator", - "call_id": "call_787231685d2800c0b1d9a7e1", - "commit_signal_sent": false, - "commit_wait_ms": 0.018144, - "confidence": 1.0, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "cpu_affinity_isolated": true, - "decode_interference_qualified": true, - "executor_wall_ms": 2905.826018, - "expected_speedup": 1.8771705185044594, - "prediction": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "protocol": "dflash.tool-speculation.v1", - "resource_percentage": 100, - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2781.764245, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "status": "hit" - }, - "task_ms": 2910.256342001958, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2781.764245, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2905.826018 - } - }, - { - "arm_order": [ - "speculative", - "control" - ], - "control": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 476.8, - "decode_tokens_per_sec": 16.8, - "mode": "control", - "model_compute_ms": 2887.3, - "prefill_ms": 2410.5, - "request_wall_ms": 2888.609096989967, - "speculation": null, - "task_ms": 5489.209855993977, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2597.462915, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2600.3301620075945 - }, - "pair_index": 7, - "speculative": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 474.6, - "decode_tokens_per_sec": 16.9, - "mode": "speculative", - "model_compute_ms": 2885.2999999999997, - "prefill_ms": 2410.7, - "request_wall_ms": 2887.2296159970574, - "speculation": { - "accelerator_relation": "non_accelerator", - "call_id": "call_ef49973cfda3a48ef0e760b4", - "commit_signal_sent": false, - "commit_wait_ms": 0.029695, - "confidence": 1.0, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "cpu_affinity_isolated": true, - "decode_interference_qualified": true, - "executor_wall_ms": 2885.999673, - "expected_speedup": 1.8771705185044594, - "prediction": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "protocol": "dflash.tool-speculation.v1", - "resource_percentage": 100, - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2777.924106, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "status": "hit" - }, - "task_ms": 2887.2296159970574, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2777.924106, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2885.999673 - } - }, - { - "arm_order": [ - "control", - "speculative" - ], - "control": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 477.4, - "decode_tokens_per_sec": 16.8, - "mode": "control", - "model_compute_ms": 2909.2000000000003, - "prefill_ms": 2431.8, - "request_wall_ms": 2911.187883990351, - "speculation": null, - "task_ms": 5520.180744992103, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2605.88394, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2608.771926999907 - }, - "pair_index": 8, - "speculative": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 474.7, - "decode_tokens_per_sec": 16.9, - "mode": "speculative", - "model_compute_ms": 2906.6, - "prefill_ms": 2431.9, - "request_wall_ms": 2910.375022998778, - "speculation": { - "accelerator_relation": "non_accelerator", - "call_id": "call_49a6db2f32b7708066aad513", - "commit_signal_sent": false, - "commit_wait_ms": 0.017313, - "confidence": 1.0, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "cpu_affinity_isolated": true, - "decode_interference_qualified": true, - "executor_wall_ms": 2908.617383, - "expected_speedup": 1.8771705185044594, - "prediction": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "protocol": "dflash.tool-speculation.v1", - "resource_percentage": 100, - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2727.319056, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "status": "hit" - }, - "task_ms": 2910.375022998778, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2727.319056, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2908.617383 - } - }, - { - "arm_order": [ - "speculative", - "control" - ], - "control": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 477.4, - "decode_tokens_per_sec": 16.8, - "mode": "control", - "model_compute_ms": 2895.0, - "prefill_ms": 2417.6, - "request_wall_ms": 2896.2876409932505, - "speculation": null, - "task_ms": 5509.29852599802, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2610.033131, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2612.8059619950363 - }, - "pair_index": 9, - "speculative": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 475.6, - "decode_tokens_per_sec": 16.8, - "mode": "speculative", - "model_compute_ms": 2920.6, - "prefill_ms": 2445.0, - "request_wall_ms": 2922.3824299988337, - "speculation": { - "accelerator_relation": "non_accelerator", - "call_id": "call_8e769b586f88a80d9a2ecbbb", - "commit_signal_sent": false, - "commit_wait_ms": 0.020619, - "confidence": 1.0, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "cpu_affinity_isolated": true, - "decode_interference_qualified": true, - "executor_wall_ms": 2921.317176, - "expected_speedup": 1.8771705185044594, - "prediction": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "protocol": "dflash.tool-speculation.v1", - "resource_percentage": 100, - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2768.185407, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "status": "hit" - }, - "task_ms": 2922.3824299988337, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2768.185407, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2921.317176 - } - }, - { - "arm_order": [ - "control", - "speculative" - ], - "control": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 478.0, - "decode_tokens_per_sec": 16.7, - "mode": "control", - "model_compute_ms": 2910.0, - "prefill_ms": 2432.0, - "request_wall_ms": 2912.1384430036414, - "speculation": null, - "task_ms": 5512.187327010906, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2596.584479, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2599.779108000803 - }, - "pair_index": 10, - "speculative": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 475.1, - "decode_tokens_per_sec": 16.8, - "mode": "speculative", - "model_compute_ms": 2907.7, - "prefill_ms": 2432.6, - "request_wall_ms": 2910.429274998023, - "speculation": { - "accelerator_relation": "non_accelerator", - "call_id": "call_aeebec7e380f1356a52b1436", - "commit_signal_sent": false, - "commit_wait_ms": 0.017904, - "confidence": 1.0, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "cpu_affinity_isolated": true, - "decode_interference_qualified": true, - "executor_wall_ms": 2908.812999, - "expected_speedup": 1.8771705185044594, - "prediction": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "protocol": "dflash.tool-speculation.v1", - "resource_percentage": 100, - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2752.299532, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "status": "hit" - }, - "task_ms": 2910.429274998023, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2752.299532, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2908.812999 - } - }, - { - "arm_order": [ - "control", - "speculative" - ], - "control": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 478.0, - "decode_tokens_per_sec": 16.7, - "mode": "control", - "model_compute_ms": 2926.6, - "prefill_ms": 2448.6, - "request_wall_ms": 2927.8773260011803, - "speculation": null, - "task_ms": 5546.5061139984755, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2614.751961, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2618.455734991585 - }, - "pair_index": 11, - "speculative": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 475.1, - "decode_tokens_per_sec": 16.8, - "mode": "speculative", - "model_compute_ms": 2909.5, - "prefill_ms": 2434.4, - "request_wall_ms": 2912.3231999983545, - "speculation": { - "accelerator_relation": "non_accelerator", - "call_id": "call_41a313b6d60b1a5ca8220181", - "commit_signal_sent": false, - "commit_wait_ms": 0.014998, - "confidence": 1.0, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "cpu_affinity_isolated": true, - "decode_interference_qualified": true, - "executor_wall_ms": 2910.675286, - "expected_speedup": 1.8771705185044594, - "prediction": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "protocol": "dflash.tool-speculation.v1", - "resource_percentage": 100, - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2754.031487, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "status": "hit" - }, - "task_ms": 2912.3231999983545, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2754.031487, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2910.675286 - } - }, - { - "arm_order": [ - "control", - "speculative" - ], - "control": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 477.4, - "decode_tokens_per_sec": 16.8, - "mode": "control", - "model_compute_ms": 2888.7000000000003, - "prefill_ms": 2411.3, - "request_wall_ms": 2889.886129007209, - "speculation": null, - "task_ms": 5489.258956004051, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2596.611067, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2599.2073560046265 - }, - "pair_index": 12, - "speculative": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 475.1, - "decode_tokens_per_sec": 16.8, - "mode": "speculative", - "model_compute_ms": 2906.5, - "prefill_ms": 2431.4, - "request_wall_ms": 2909.0337640082, - "speculation": { - "accelerator_relation": "non_accelerator", - "call_id": "call_fb33f7142390b5ffa3f1df08", - "commit_signal_sent": false, - "commit_wait_ms": 0.020308, - "confidence": 1.0, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "cpu_affinity_isolated": true, - "decode_interference_qualified": true, - "executor_wall_ms": 2907.440281, - "expected_speedup": 1.8771705185044594, - "prediction": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "protocol": "dflash.tool-speculation.v1", - "resource_percentage": 100, - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2758.809735, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "status": "hit" - }, - "task_ms": 2909.0337640082, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2758.809735, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2907.440281 - } - }, - { - "arm_order": [ - "control", - "speculative" - ], - "control": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 477.0, - "decode_tokens_per_sec": 16.8, - "mode": "control", - "model_compute_ms": 2889.3, - "prefill_ms": 2412.3, - "request_wall_ms": 2890.6500519951805, - "speculation": null, - "task_ms": 5495.200345001649, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2601.708584, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2604.3903139943723 - }, - "pair_index": 13, - "speculative": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 475.3, - "decode_tokens_per_sec": 16.8, - "mode": "speculative", - "model_compute_ms": 2947.5, - "prefill_ms": 2472.2, - "request_wall_ms": 2950.2172590000555, - "speculation": { - "accelerator_relation": "non_accelerator", - "call_id": "call_411fd3326040ca4e30bc7f32", - "commit_signal_sent": false, - "commit_wait_ms": 0.019516, - "confidence": 1.0, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "cpu_affinity_isolated": true, - "decode_interference_qualified": true, - "executor_wall_ms": 2948.600252, - "expected_speedup": 1.8771705185044594, - "prediction": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "protocol": "dflash.tool-speculation.v1", - "resource_percentage": 100, - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2944.545582, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "status": "hit" - }, - "task_ms": 2950.2172590000555, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2944.545582, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2948.600252 - } - }, - { - "arm_order": [ - "speculative", - "control" - ], - "control": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 477.7, - "decode_tokens_per_sec": 16.7, - "mode": "control", - "model_compute_ms": 2891.5, - "prefill_ms": 2413.8, - "request_wall_ms": 2892.8247750009177, - "speculation": null, - "task_ms": 5513.9780350000365, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2618.248832, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2620.997508012806 - }, - "pair_index": 14, - "speculative": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 475.4, - "decode_tokens_per_sec": 16.8, - "mode": "speculative", - "model_compute_ms": 2889.8, - "prefill_ms": 2414.4, - "request_wall_ms": 2891.653121012496, - "speculation": { - "accelerator_relation": "non_accelerator", - "call_id": "call_f7e244b68e2cc9eb95428449", - "commit_signal_sent": false, - "commit_wait_ms": 0.018024, - "confidence": 1.0, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "cpu_affinity_isolated": true, - "decode_interference_qualified": true, - "executor_wall_ms": 2890.557961, - "expected_speedup": 1.8771705185044594, - "prediction": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "protocol": "dflash.tool-speculation.v1", - "resource_percentage": 100, - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2765.571476, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "status": "hit" - }, - "task_ms": 2891.653121012496, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2765.571476, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2890.557961 - } - }, - { - "arm_order": [ - "speculative", - "control" - ], - "control": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 477.6, - "decode_tokens_per_sec": 16.7, - "mode": "control", - "model_compute_ms": 2890.5, - "prefill_ms": 2412.9, - "request_wall_ms": 2891.906609002035, - "speculation": null, - "task_ms": 5502.877983992221, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2607.65675, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2610.64586599241 - }, - "pair_index": 15, - "speculative": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 475.2, - "decode_tokens_per_sec": 16.8, - "mode": "speculative", - "model_compute_ms": 2942.2999999999997, - "prefill_ms": 2467.1, - "request_wall_ms": 2945.138353999937, - "speculation": { - "accelerator_relation": "non_accelerator", - "call_id": "call_0e513da24ccdc47dfbb286f2", - "commit_signal_sent": false, - "commit_wait_ms": 0.009428, - "confidence": 1.0, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "cpu_affinity_isolated": true, - "decode_interference_qualified": true, - "executor_wall_ms": 2943.285245, - "expected_speedup": 1.8771705185044594, - "prediction": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "protocol": "dflash.tool-speculation.v1", - "resource_percentage": 100, - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2751.061218, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "status": "hit" - }, - "task_ms": 2945.138353999937, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2751.061218, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2943.285245 - } - }, - { - "arm_order": [ - "control", - "speculative" - ], - "control": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 478.1, - "decode_tokens_per_sec": 16.7, - "mode": "control", - "model_compute_ms": 2947.2999999999997, - "prefill_ms": 2469.2, - "request_wall_ms": 2949.831121004536, - "speculation": null, - "task_ms": 5568.309862996102, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2614.703825, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2618.221951997839 - }, - "pair_index": 16, - "speculative": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 475.0, - "decode_tokens_per_sec": 16.8, - "mode": "speculative", - "model_compute_ms": 2910.8, - "prefill_ms": 2435.8, - "request_wall_ms": 2914.3550349981524, - "speculation": { - "accelerator_relation": "non_accelerator", - "call_id": "call_e2fabbd7938f66d73229dede", - "commit_signal_sent": false, - "commit_wait_ms": 0.022062, - "confidence": 1.0, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "cpu_affinity_isolated": true, - "decode_interference_qualified": true, - "executor_wall_ms": 2911.896423, - "expected_speedup": 1.8771705185044594, - "prediction": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "protocol": "dflash.tool-speculation.v1", - "resource_percentage": 100, - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2735.067036, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "status": "hit" - }, - "task_ms": 2914.3550349981524, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2735.067036, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2911.896423 - } - }, - { - "arm_order": [ - "control", - "speculative" - ], - "control": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 477.5, - "decode_tokens_per_sec": 16.8, - "mode": "control", - "model_compute_ms": 2889.6, - "prefill_ms": 2412.1, - "request_wall_ms": 2891.2249889981467, - "speculation": null, - "task_ms": 6090.181550011039, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 3195.757001, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 3198.7133949878626 - }, - "pair_index": 17, - "speculative": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 475.0, - "decode_tokens_per_sec": 16.8, - "mode": "speculative", - "model_compute_ms": 2906.6, - "prefill_ms": 2431.6, - "request_wall_ms": 2909.964589009178, - "speculation": { - "accelerator_relation": "non_accelerator", - "call_id": "call_5eba859117cad5dfe23f4821", - "commit_signal_sent": false, - "commit_wait_ms": 0.014517, - "confidence": 1.0, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "cpu_affinity_isolated": true, - "decode_interference_qualified": true, - "executor_wall_ms": 2907.821637, - "expected_speedup": 1.8771705185044594, - "prediction": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "protocol": "dflash.tool-speculation.v1", - "resource_percentage": 100, - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2777.633709, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "status": "hit" - }, - "task_ms": 2909.964589009178, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2777.633709, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2907.821637 - } - }, - { - "arm_order": [ - "control", - "speculative" - ], - "control": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 477.5, - "decode_tokens_per_sec": 16.8, - "mode": "control", - "model_compute_ms": 2890.1, - "prefill_ms": 2412.6, - "request_wall_ms": 2891.499435005244, - "speculation": null, - "task_ms": 5504.3630370055325, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2610.124635, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2612.7046740002697 - }, - "pair_index": 18, - "speculative": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 475.1, - "decode_tokens_per_sec": 16.8, - "mode": "speculative", - "model_compute_ms": 2908.5, - "prefill_ms": 2433.4, - "request_wall_ms": 2911.1464049929054, - "speculation": { - "accelerator_relation": "non_accelerator", - "call_id": "call_d3fd156d25c15849551e318e", - "commit_signal_sent": false, - "commit_wait_ms": 0.019105, - "confidence": 1.0, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "cpu_affinity_isolated": true, - "decode_interference_qualified": true, - "executor_wall_ms": 2909.505163, - "expected_speedup": 1.8771705185044594, - "prediction": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "protocol": "dflash.tool-speculation.v1", - "resource_percentage": 100, - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2744.908547, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "status": "hit" - }, - "task_ms": 2911.1464049929054, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2744.908547, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2909.505163 - } - }, - { - "arm_order": [ - "control", - "speculative" - ], - "control": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 478.6, - "decode_tokens_per_sec": 16.7, - "mode": "control", - "model_compute_ms": 2924.9, - "prefill_ms": 2446.3, - "request_wall_ms": 2926.6352289996576, - "speculation": null, - "task_ms": 5560.942076990614, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2630.064817, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2634.031902998686 - }, - "pair_index": 19, - "speculative": { - "accept_rate": 0.4166666567325592, - "assistant_content_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "completion_tokens": 8, - "decode_ms": 475.5, - "decode_tokens_per_sec": 16.8, - "mode": "speculative", - "model_compute_ms": 2946.7, - "prefill_ms": 2471.2, - "request_wall_ms": 2950.1483669882873, - "speculation": { - "accelerator_relation": "non_accelerator", - "call_id": "call_69bbfcf178f13d17d02fc907", - "commit_signal_sent": false, - "commit_wait_ms": 0.017122, - "confidence": 1.0, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "cpu_affinity_isolated": true, - "decode_interference_qualified": true, - "executor_wall_ms": 2948.194829, - "expected_speedup": 1.8771705185044594, - "prediction": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "protocol": "dflash.tool-speculation.v1", - "resource_percentage": 100, - "result": { - "checksum": "9705095564492366076", - "compute_ms": 2766.327547, - "cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "iterations": 172452, - "nonzeros_per_row": 16, - "rows": 4096, - "seed": 731, - "threads": 2, - "worker_cpus": [ - 14, - 15 - ] - }, - "status": "hit" - }, - "task_ms": 2950.1483669882873, - "tool_call": { - "arguments": { - "iterations": 172452 - }, - "name": "benchmark_cpu_sparse" - }, - "tool_call_sha256": "8236afddf102fbccc42b1d8fd12eaf1f7fc3c5b4ef9d1be12428c3bd05412101", - "tool_checksum": "9705095564492366076", - "tool_compute_ms": 2766.327547, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "tool_wall_ms": 2948.194829 - } - } - ], - "phase": "native_engine", - "production_gate": { - "checks": { - "correctness": true, - "exact_hit_speedup": true, - "model_slowdown": true, - "speedup_ci_low": true, - "strong_sequential_baseline": true - }, - "passed": true, - "thresholds": { - "max_model_slowdown_percent": 5.0, - "min_exact_hit_speedup": 1.8, - "min_speedup_ci_low": 1.7 - } - }, - "server_snapshot": { - "runtime": { - "backend": "hip", - "chunk": 2048, - "draft_device": null, - "draft_residency": "auto", - "fa_window": 0, - "kv_cache_k": "q4_0", - "kv_cache_v": "q4_0", - "lazy_draft": false, - "target_device": "hip:0", - "target_sharding": false - }, - "speculative": { - "ddtree_budget": null, - "enabled": false - }, - "tool_speculation": { - "allowed_tools": [ - "benchmark_cpu_sparse" - ], - "compute_isolation": "disjoint_cpu_affinity", - "cpu_affinity_isolated": true, - "enabled": true, - "execution_mode": "child_process_cpu_affinity", - "executor_contract": "child_process_cpu_affinity", - "hip_reserved_tool_compute_units": 0, - "hip_tool_device": null, - "max_model_slowdown_ratio": 1.05, - "model_cpu_affinity": [ - 0, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - 23, - 24, - 25, - 26, - 27, - 28, - 29 - ], - "model_expert_ownership_unique": false, - "model_routing_static": false, - "preserves_token_speculation": true, - "profile_lanes": [ - { - "accelerator_relation": "non_accelerator", - "decode_interference_qualified": true, - "model_slowdown_ratio": 1.0011341375399527, - "requires_static_model_routing": false, - "requires_unique_expert_ownership": false, - "resource_percentage": 100 - } - ], - "profile_status": "qualified", - "protocol": "dflash.tool-speculation.v1", - "requires_client_support": true, - "tool_cpu_affinity": [ - 14, - 15, - 30, - 31 - ], - "unqualified_lane_policy": "defer" - } - }, - "summary": { - "all_calls_identical": true, - "all_model_outputs_identical": true, - "all_tool_outputs_equivalent": true, - "control_model_compute_p50_ms": 2893.25, - "control_task_p50_ms": 5511.639613505395, - "control_tool_compute_p50_ms": 2607.189201, - "exact_hit_speedup": 1.8937725205440066, - "exact_hit_speedup_bootstrap_95ci": [ - 1.8858550837704051, - 1.8964496590120083 - ], - "ideal_zero_interference_speedup_ceiling": 1.9011282125637259, - "latency_match_ratio": 0.901128212563726, - "median_accept_rate": 0.4166666567325592, - "median_decode_tokens_per_sec": 16.8, - "model_compute_slowdown_percent": 0.46141881966645926, - "native_hits": 20, - "pairs": 20, - "speculative_model_compute_p50_ms": 2906.6, - "speculative_task_p50_ms": 2910.4021489984007, - "speculative_tool_compute_p50_ms": 2755.918672, - "task_latency_reduction_percent": 47.19534742679975, - "tool_compute_slowdown_percent": 5.704590635115925 - } -} From b39e3cbaaa37312b34651bf3907adaea91be1965 Mon Sep 17 00:00:00 2001 From: mrciffa <49000955+davide221@users.noreply.github.com> Date: Tue, 18 Aug 2026 13:23:12 +0200 Subject: [PATCH 06/11] fix(server): harden speculative tool execution --- optimizations/ooo_spec_lucebox5_cpu/README.md | 24 +- ...sh_server_native_tool_predictor_wrapper.sh | 4 +- server/CMakeLists.txt | 8 +- .../llama.cpp/ggml/src/ggml-cuda/common.cuh | 87 ---- server/src/common/backend_ipc.cpp | 61 ++- server/src/common/backend_ipc.h | 4 + server/src/common/model_backend.h | 5 - .../src/common/qwen3_tool_predictor_ipc.cpp | 215 +++++++-- server/src/common/qwen3_tool_predictor_ipc.h | 16 +- server/src/server/http_server.cpp | 266 ++++++---- .../server/native_semantic_tool_predictor.cpp | 13 +- server/src/server/server_main.cpp | 163 +------ server/src/server/tool_speculation.cpp | 209 ++++---- server/src/server/tool_speculation.h | 63 +-- .../src/server/tool_speculation_hip_probe.cpp | 456 ------------------ .../src/server/tool_speculation_hip_probe.h | 21 - server/test/test_semantic_tool_hint.cpp | 49 ++ server/test/test_server_unit.cpp | 77 +-- server/test/test_tool_speculation.cpp | 337 +++++++------ 19 files changed, 826 insertions(+), 1252 deletions(-) delete mode 100644 server/src/server/tool_speculation_hip_probe.cpp delete mode 100644 server/src/server/tool_speculation_hip_probe.h diff --git a/optimizations/ooo_spec_lucebox5_cpu/README.md b/optimizations/ooo_spec_lucebox5_cpu/README.md index 1f18a0e8e..8a2368578 100644 --- a/optimizations/ooo_spec_lucebox5_cpu/README.md +++ b/optimizations/ooo_spec_lucebox5_cpu/README.md @@ -6,9 +6,10 @@ compute window; the predicted read-only tool runs on reserved CPU cores while DeepSeek-V4-0731 decodes with DS4/DSpark on R9700 + Strix. The result stays private unless DeepSeek emits the exact same canonical function and arguments. -This path does not inject tokens, replace DSpark, or retry speculative decoding -with autoregressive decoding. A wrong prediction is discarded and the caller -executes the target model's authoritative call normally. +This path does not inject tokens or replace DSpark. Tool prediction does not +alter the target decoder's selection or normal recovery policy. A wrong +prediction is discarded and the caller executes the target model's +authoritative call normally. ## Production result @@ -52,14 +53,16 @@ hash for provenance. - Only explicitly allowlisted, read-only/idempotent tools are eligible. - The external result is committed only on an exact canonical call match. -- The executor is launched directly without a shell and has a hard timeout. +- The executor is launched directly without a shell and has a hard deadline + measured from launch; inherited server file descriptors are closed. - Lucebox5 reserves CPUs `14-15,30-31`; the model uses `0-13,16-29`. -- Startup fails closed if CPU masks overlap or the measured lane profile fails. +- Startup fails closed if CPU masks overlap, the measured lane profile is not + qualified, or its executor contract differs from the configured lane. - `before-model` is the native predictor default, so shared-GPU prediction cannot reduce target prefill/decode throughput. - The same API works on a single GPU: run the predictor before the target and - overlap only the CPU tool. An HTTP predictor can use the same verification - and executor path on other model families. + overlap only the CPU tool. A local or numeric-IPv4 HTTP predictor can use the + same verification and executor path on other model families. The speedup applies to tool-using request latency, not token throughput. Its real-world value depends on exact predictor hit rate and on how much tool work @@ -81,9 +84,10 @@ Launch the qualified single-call configuration on an otherwise idle Lucebox5: ``` The launcher defaults to Qwen3-0.6B Q8_0 on predictor GPU 1. Override placement -with `PREDICTOR_MODEL`, `PREDICTOR_GPU`, `PREDICTOR_MAX_CTX`, and -`PREDICTOR_MAX_TOKENS`. The adjacent `candidate-build` symlink in the wrapper -selects a build even though the qualified launcher clears ambient variables. +with `PREDICTOR_MODEL`, `PREDICTOR_GPU`, `PREDICTOR_MAX_CTX`, +`PREDICTOR_MAX_TOKENS`, and `PREDICTOR_TIMEOUT_MS`. The adjacent +`candidate-build` symlink in the wrapper selects a build even though the +qualified launcher clears ambient variables. Run the single-call paired gate: diff --git a/optimizations/ooo_spec_lucebox5_cpu/dflash_server_native_tool_predictor_wrapper.sh b/optimizations/ooo_spec_lucebox5_cpu/dflash_server_native_tool_predictor_wrapper.sh index d8e118d85..fd48d27ab 100755 --- a/optimizations/ooo_spec_lucebox5_cpu/dflash_server_native_tool_predictor_wrapper.sh +++ b/optimizations/ooo_spec_lucebox5_cpu/dflash_server_native_tool_predictor_wrapper.sh @@ -16,6 +16,7 @@ PREDICTOR_IPC_BIN="${PREDICTOR_IPC_BIN:-${CANDIDATE_BUILD}/backend_ipc_daemon}" PREDICTOR_GPU="${PREDICTOR_GPU:-1}" PREDICTOR_MAX_CTX="${PREDICTOR_MAX_CTX:-4096}" PREDICTOR_MAX_TOKENS="${PREDICTOR_MAX_TOKENS:-256}" +PREDICTOR_TIMEOUT_MS="${PREDICTOR_TIMEOUT_MS:-2000}" PREDICTOR_CONFIDENCE="${PREDICTOR_CONFIDENCE:-0.75}" PREDICTOR_SCHEDULE="${PREDICTOR_SCHEDULE:-before-model}" # The qualified 0731 launcher disables caches for cold throughput benchmarks. @@ -55,7 +56,8 @@ exec "${CANDIDATE_BUILD}/dflash_server" "$@" \ --tool-hint-native-ipc-bin "${PREDICTOR_IPC_BIN}" \ --tool-hint-native-gpu "${PREDICTOR_GPU}" \ --tool-hint-native-max-ctx "${PREDICTOR_MAX_CTX}" \ - --tool-hint-sidecar-max-tokens "${PREDICTOR_MAX_TOKENS}" \ + --tool-hint-max-tokens "${PREDICTOR_MAX_TOKENS}" \ + --tool-hint-timeout-ms "${PREDICTOR_TIMEOUT_MS}" \ --tool-hint-native-schedule "${PREDICTOR_SCHEDULE}" \ --tool-hint-execution-confidence "${PREDICTOR_CONFIDENCE}" \ "${cache_args[@]}" diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index ef4206105..d8bd8170b 100644 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -1947,8 +1947,6 @@ if(DFLASH27B_SERVER) target_include_directories(dflash_server PRIVATE ${DFLASH27B_SRC_INCLUDE_DIRS}) if(DFLASH27B_GPU_BACKEND STREQUAL "hip") target_compile_definitions(dflash_server PRIVATE DFLASH27B_BACKEND_HIP=1 GGML_USE_HIP) - target_sources(dflash_server PRIVATE - src/server/tool_speculation_hip_probe.cpp) if(DFLASH27B_ENABLE_MIXED_CUDA_HIP) target_compile_definitions(dflash_server PRIVATE DFLASH27B_BACKEND_MIXED=1) @@ -1975,11 +1973,7 @@ if(DFLASH27B_SERVER) find_package(CUDAToolkit REQUIRED) target_link_libraries(dflash_server PRIVATE CUDA::cudart) else() - # ggml-hip finds hipBLAS in a child-directory scope. The trusted - # in-process tool adapter also calls hipBLAS directly, so import - # the target in this scope before linking the server executable. - find_package(hipblas REQUIRED) - target_link_libraries(dflash_server PRIVATE hip::host roc::hipblas) + target_link_libraries(dflash_server PRIVATE hip::host) endif() # Copy share/status.html next to the binary so it can be found at runtime. diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/common.cuh b/server/deps/llama.cpp/ggml/src/ggml-cuda/common.cuh index 0680df55e..31b04cd14 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cuda/common.cuh +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/common.cuh @@ -25,10 +25,8 @@ #include #include #include -#include #include #include -#include #include #include #include @@ -163,60 +161,6 @@ static int ggml_cuda_highest_compiled_arch(const int arch) { #define GGML_CUDA_MAX_STREAMS 8 -#if defined(GGML_USE_HIP) -// Optional Lucebox experiment: reserve the lowest CUs on one HIP device for a -// trusted in-process tool stream. Every lazily-created ggml stream on that -// device receives the complementary CU mask, creating a real disjoint lane -// inside one HIP context. Default behavior is unchanged when the variable is -// absent. Format: DFLASH_HIP_RESERVED_TOOL_LANE=DEVICE:CUS (for example 0:1). -static inline bool dflash_hip_model_stream_mask( - int device, std::vector & mask, int & reserved_cus) { - const char * raw = std::getenv("DFLASH_HIP_RESERVED_TOOL_LANE"); - if (!raw || !*raw) return false; - - errno = 0; - char * separator = nullptr; - const long configured_device = std::strtol(raw, &separator, 10); - if (errno != 0 || separator == raw || !separator || *separator != ':') { - GGML_ABORT( - "DFLASH_HIP_RESERVED_TOOL_LANE must be DEVICE:CUS, got '%s'\n", - raw); - } - char * end = nullptr; - errno = 0; - const long configured_cus = std::strtol(separator + 1, &end, 10); - if (errno != 0 || !end || *end != '\0' || configured_device < 0 || - configured_cus <= 0) { - GGML_ABORT( - "DFLASH_HIP_RESERVED_TOOL_LANE must be DEVICE:CUS with positive " - "integers, got '%s'\n", raw); - } - if (configured_device != device) return false; - - hipDeviceProp_t properties{}; - const hipError_t status = hipGetDeviceProperties(&properties, device); - if (status != hipSuccess) { - GGML_ABORT( - "DFLASH_HIP_RESERVED_TOOL_LANE could not inspect HIP device %d: " - "%s\n", - device, hipGetErrorString(status)); - } - if (configured_cus >= properties.multiProcessorCount) { - GGML_ABORT( - "DFLASH_HIP_RESERVED_TOOL_LANE reserves %ld of %d CUs on device " - "%d; at least one model CU is required\n", - configured_cus, properties.multiProcessorCount, device); - } - reserved_cus = static_cast(configured_cus); - mask.assign( - static_cast((properties.multiProcessorCount + 31) / 32), 0); - for (int cu = reserved_cus; cu < properties.multiProcessorCount; ++cu) { - mask[static_cast(cu / 32)] |= uint32_t{1} << (cu % 32); - } - return true; -} -#endif - [[noreturn]] void ggml_cuda_error(const char * stmt, const char * func, const char * file, int line, const char * msg); @@ -1542,37 +1486,6 @@ struct ggml_backend_cuda_context { cudaStream_t stream(int device, int stream) { if (streams[device][stream] == nullptr) { ggml_cuda_set_device(device); -#if defined(GGML_USE_HIP) - std::vector model_cu_mask; - int reserved_cus = 0; - const bool disjoint_tool_lane = dflash_hip_model_stream_mask( - device, model_cu_mask, reserved_cus); - if (disjoint_tool_lane) { - CUDA_CHECK(hipExtStreamCreateWithCUMask( - &streams[device][stream], - static_cast(model_cu_mask.size()), - model_cu_mask.data())); - if (low_priority_streams) { -#if HIP_VERSION_MAJOR >= 7 - hipStreamAttrValue priority{}; - priority.priority = stream_priority; - CUDA_CHECK(hipStreamSetAttribute( - streams[device][stream], - hipStreamAttributePriority, &priority)); -#else - GGML_ABORT( - "DFLASH_HIP_RESERVED_TOOL_LANE requires ROCm 7+ " - "to preserve low-priority DSpark streams\n"); -#endif - } - if (stream == 0) { - std::fprintf(stderr, - "ggml_hip: device %d model streams exclude %d " - "low CU(s) reserved for in-process tools\n", - device, reserved_cus); - } - } else -#endif if (low_priority_streams) { CUDA_CHECK(cudaStreamCreateWithPriority( &streams[device][stream], cudaStreamNonBlocking, diff --git a/server/src/common/backend_ipc.cpp b/server/src/common/backend_ipc.cpp index 4e72cd86a..e26e47bfa 100644 --- a/server/src/common/backend_ipc.cpp +++ b/server/src/common/backend_ipc.cpp @@ -7,12 +7,15 @@ #include #include #include +#include #include +#include #include #if !defined(_WIN32) # include # include +# include # include # include # include @@ -248,7 +251,19 @@ bool BackendIpcProcess::start(const BackendIpcLaunchConfig & cfg) { } void BackendIpcProcess::close() { + close_impl(false); +} + +void BackendIpcProcess::terminate() { + close_impl(true); +} + +void BackendIpcProcess::close_impl(bool force_terminate) { #if !defined(_WIN32) + const pid_t child = pid_; + if (force_terminate && child > 0) { + (void)::kill(child, SIGTERM); + } if (cmd_) { std::fclose(cmd_); cmd_ = nullptr; @@ -269,14 +284,36 @@ void BackendIpcProcess::close() { ::close(shared_payload_fd_); shared_payload_fd_ = -1; } - if (pid_ > 0) { + if (child > 0) { int status = 0; - ::waitpid(pid_, &status, 0); + if (force_terminate) { + bool reaped = false; + const auto deadline = std::chrono::steady_clock::now() + + std::chrono::milliseconds(100); + while (std::chrono::steady_clock::now() < deadline) { + const pid_t waited = ::waitpid(child, &status, WNOHANG); + if (waited == child || + (waited < 0 && errno == ECHILD)) { + reaped = true; + break; + } + if (waited < 0 && errno != EINTR) break; + std::this_thread::sleep_for(std::chrono::milliseconds(2)); + } + if (!reaped) { + (void)::kill(child, SIGKILL); + while (::waitpid(child, &status, 0) < 0 && errno == EINTR) {} + } + } else { + while (::waitpid(child, &status, 0) < 0 && errno == EINTR) {} + } pid_ = -1; } if (owns_work_dir_ && !work_dir_.empty()) { ::rmdir(work_dir_.c_str()); } +#else + (void)force_terminate; #endif active_ = false; owns_work_dir_ = false; @@ -403,12 +440,20 @@ bool BackendIpcProcess::init_work_dir(const std::string & requested) { work_dir_.c_str(), std::strerror(errno)); return false; } - struct stat st; - if (::stat(work_dir_.c_str(), &st) != 0 || !S_ISDIR(st.st_mode)) { - std::fprintf(stderr, "backend-ipc work_dir is not a directory: %s\n", - work_dir_.c_str()); - return false; - } + } + struct stat st; + if (::lstat(work_dir_.c_str(), &st) != 0 || + !S_ISDIR(st.st_mode)) { + std::fprintf(stderr, + "backend-ipc work_dir is not a real directory: %s\n", + work_dir_.c_str()); + return false; + } + if (st.st_uid != ::geteuid() || (st.st_mode & 0777) != 0700) { + std::fprintf(stderr, + "backend-ipc work_dir must be owned by the server and mode 0700: %s\n", + work_dir_.c_str()); + return false; } return true; } diff --git a/server/src/common/backend_ipc.h b/server/src/common/backend_ipc.h index 8731f3714..e181827ec 100644 --- a/server/src/common/backend_ipc.h +++ b/server/src/common/backend_ipc.h @@ -140,6 +140,9 @@ class BackendIpcProcess { bool start(const BackendIpcLaunchConfig & cfg); void close(); + // Stop a wedged daemon without waiting indefinitely for graceful EOF + // handling. Used by optional sidecar lanes with hard deadlines. + void terminate(); bool active() const { return active_; } FILE * command_stream() const { return cmd_; } @@ -160,6 +163,7 @@ class BackendIpcProcess { bool read_shared_payload(void * data, size_t bytes, uint64_t seq) const; private: + void close_impl(bool force_terminate); #if !defined(_WIN32) bool init_work_dir(const std::string & requested); bool init_shared_payload(size_t bytes); diff --git a/server/src/common/model_backend.h b/server/src/common/model_backend.h index 36970babf..12445e6b2 100644 --- a/server/src/common/model_backend.h +++ b/server/src/common/model_backend.h @@ -197,10 +197,6 @@ struct GenerateRequest { // path returns success but emits no tokens, so each backend can route the // retry through its existing AR path without copying retry policy. bool force_ar_decode = false; - // Opt out of the common speculative-to-AR empty-output retry. Tool - // speculation sets this false so the external optimization can never - // change the request's model decode strategy. - bool allow_decode_mode_retry = true; }; // Stable, backend-independent generation failure categories. Backends should @@ -373,7 +369,6 @@ struct ModelBackend { static bool should_retry_empty_spec_decode(const GenerateRequest & req, const GenerateResult & result) { return req.n_gen > 0 - && req.allow_decode_mode_retry && !req.force_ar_decode && result.ok() && result.spec_decode_ran diff --git a/server/src/common/qwen3_tool_predictor_ipc.cpp b/server/src/common/qwen3_tool_predictor_ipc.cpp index a7bdf038c..cd428ab37 100644 --- a/server/src/common/qwen3_tool_predictor_ipc.cpp +++ b/server/src/common/qwen3_tool_predictor_ipc.cpp @@ -3,9 +3,145 @@ #include "io_utils.h" #include +#include +#include #include +#if !defined(_WIN32) +# include +# include +# include +#endif + namespace dflash::common { +namespace { + +#if !defined(_WIN32) +bool write_private_prompt_file( + const std::string & work_dir, + const std::vector & prompt_ids, + std::string & path) { + std::string pattern = work_dir + "/tool_predictor_prompt_XXXXXX"; + std::vector buffer(pattern.begin(), pattern.end()); + buffer.push_back('\0'); + const int fd = ::mkstemp(buffer.data()); + if (fd < 0) return false; + path = buffer.data(); + const bool written = ::fchmod(fd, S_IRUSR | S_IWUSR) == 0 && + write_exact_fd( + fd, prompt_ids.data(), prompt_ids.size() * sizeof(int32_t)); + const bool closed = ::close(fd) == 0; + if (!written || !closed) { + ::unlink(path.c_str()); + path.clear(); + return false; + } + return true; +} + +bool read_exact_until( + int fd, + void * data, + size_t bytes, + const std::chrono::steady_clock::time_point & deadline, + bool & timed_out) { + auto * cursor = static_cast(data); + size_t received = 0; + timed_out = false; + while (received < bytes) { + const auto now = std::chrono::steady_clock::now(); + if (now >= deadline) { + timed_out = true; + return false; + } + const auto remaining = + std::chrono::duration_cast( + deadline - now).count(); + pollfd descriptor{fd, POLLIN | POLLHUP, 0}; + const int polled = ::poll( + &descriptor, 1, + static_cast((std::max)(int64_t{1}, remaining))); + if (polled == 0) { + timed_out = true; + return false; + } + if (polled < 0) { + if (errno == EINTR) continue; + return false; + } + if (descriptor.revents & (POLLERR | POLLNVAL)) return false; + const ssize_t count = ::read( + fd, cursor + received, bytes - received); + if (count == 0) return false; + if (count < 0) { + if (errno == EINTR) continue; + return false; + } + received += static_cast(count); + } + return true; +} +#endif + +} // namespace + +bool read_qwen3_tool_predictor_response( + int stream_fd, + int max_tokens, + int timeout_ms, + std::vector & output_ids, + std::string & error) { + output_ids.clear(); + error.clear(); +#if defined(_WIN32) + (void)stream_fd; + (void)max_tokens; + (void)timeout_ms; + error = "native_predictor_ipc_unsupported"; + return false; +#else + if (stream_fd < 0 || max_tokens <= 0 || timeout_ms <= 0) { + error = "native_predictor_invalid_request"; + return false; + } + const auto deadline = std::chrono::steady_clock::now() + + std::chrono::milliseconds(timeout_ms); + bool timed_out = false; + int32_t status = -1; + if (!read_exact_until( + stream_fd, &status, sizeof(status), deadline, timed_out)) { + error = timed_out + ? "native_predictor_timeout" + : "native_predictor_invalid_response"; + return false; + } + if (status != 0) { + error = "native_predictor_generation_failed"; + return false; + } + + int32_t count = -1; + if (!read_exact_until( + stream_fd, &count, sizeof(count), deadline, timed_out) || + count <= 0 || count > max_tokens) { + error = timed_out + ? "native_predictor_timeout" + : "native_predictor_invalid_response"; + return false; + } + output_ids.assign(static_cast(count), 0); + if (!read_exact_until( + stream_fd, output_ids.data(), + output_ids.size() * sizeof(int32_t), deadline, timed_out)) { + output_ids.clear(); + error = timed_out + ? "native_predictor_timeout" + : "native_predictor_invalid_response"; + return false; + } + return true; +#endif +} bool Qwen3ToolPredictorIpcClient::start( const std::string & bin, @@ -19,7 +155,7 @@ bool Qwen3ToolPredictorIpcClient::start( "Qwen3 tool-predictor IPC is only implemented on POSIX hosts\n"); return false; #else - std::lock_guard lock(mutex_); + std::lock_guard lock(mutex_); close_locked(); if (bin.empty() || model_path.empty() || max_ctx <= 0) return false; @@ -46,55 +182,75 @@ bool Qwen3ToolPredictorIpcClient::start( bool Qwen3ToolPredictorIpcClient::predict( const std::vector & prompt_ids, int max_tokens, + int timeout_ms, std::vector & output_ids, std::string & error) { output_ids.clear(); error.clear(); #if defined(_WIN32) - (void)prompt_ids; (void)max_tokens; + (void)prompt_ids; (void)max_tokens; (void)timeout_ms; error = "native_predictor_ipc_unsupported"; return false; #else - std::lock_guard lock(mutex_); + if (prompt_ids.empty() || max_tokens <= 0 || timeout_ms <= 0) { + error = "native_predictor_invalid_request"; + return false; + } + const auto deadline = std::chrono::steady_clock::now() + + std::chrono::milliseconds(timeout_ms); + std::unique_lock lock(mutex_, std::defer_lock); + if (!lock.try_lock_until(deadline)) { + error = "native_predictor_timeout"; + return false; + } FILE * command = process_.command_stream(); const int stream_fd = process_.stream_fd(); - if (!active_ || !command || stream_fd < 0) { + if (!active_.load(std::memory_order_acquire) || + !command || stream_fd < 0) { error = "native_predictor_not_active"; return false; } - if (prompt_ids.empty() || max_tokens <= 0) { - error = "native_predictor_invalid_request"; - return false; - } - const std::string path = process_.next_path("tool_predictor_prompt"); - if (!write_int32_file(path, prompt_ids)) { + std::string path; + if (!write_private_prompt_file( + process_.work_dir(), prompt_ids, path)) { error = "native_predictor_prompt_write_failed"; return false; } - std::fprintf(command, "predict %d %s\n", max_tokens, path.c_str()); - std::fflush(command); + if (std::chrono::steady_clock::now() >= deadline) { + std::remove(path.c_str()); + error = "native_predictor_timeout"; + return false; + } - int32_t status = -1; - bool ok = read_exact_fd(stream_fd, &status, sizeof(status)) && status == 0; - if (ok) { - int32_t count = -1; - ok = read_exact_fd(stream_fd, &count, sizeof(count)) && - count > 0 && count <= max_tokens; - if (ok) { - output_ids.assign(static_cast(count), 0); - ok = read_exact_fd(stream_fd, output_ids.data(), - output_ids.size() * sizeof(int32_t)); - } + if (std::fprintf( + command, "predict %d %s\n", max_tokens, path.c_str()) < 0 || + std::fflush(command) != 0) { + std::remove(path.c_str()); + error = "native_predictor_command_write_failed"; + process_.terminate(); + active_ = false; + return false; + } + const auto response_started = std::chrono::steady_clock::now(); + if (response_started >= deadline) { + std::remove(path.c_str()); + error = "native_predictor_timeout"; + process_.terminate(); + active_ = false; + return false; } + const int response_timeout_ms = std::max(1, static_cast( + std::chrono::duration_cast( + deadline - response_started).count())); + const bool ok = read_qwen3_tool_predictor_response( + stream_fd, max_tokens, response_timeout_ms, output_ids, error); std::remove(path.c_str()); if (!ok) { - error = status == 0 - ? "native_predictor_invalid_response" - : "native_predictor_generation_failed"; output_ids.clear(); - close_locked(); + process_.terminate(); + active_ = false; return false; } return true; @@ -102,8 +258,7 @@ bool Qwen3ToolPredictorIpcClient::predict( } bool Qwen3ToolPredictorIpcClient::active() const { - std::lock_guard lock(mutex_); - return active_; + return active_.load(std::memory_order_acquire); } void Qwen3ToolPredictorIpcClient::close_locked() { @@ -112,7 +267,7 @@ void Qwen3ToolPredictorIpcClient::close_locked() { } void Qwen3ToolPredictorIpcClient::close() { - std::lock_guard lock(mutex_); + std::lock_guard lock(mutex_); close_locked(); } diff --git a/server/src/common/qwen3_tool_predictor_ipc.h b/server/src/common/qwen3_tool_predictor_ipc.h index 5cab9513c..7c5dc0095 100644 --- a/server/src/common/qwen3_tool_predictor_ipc.h +++ b/server/src/common/qwen3_tool_predictor_ipc.h @@ -9,6 +9,7 @@ #include "backend_ipc.h" +#include #include #include #include @@ -35,6 +36,7 @@ class Qwen3ToolPredictorIpcClient { // On transport or generation failure the lane closes and fails shut. bool predict(const std::vector & prompt_ids, int max_tokens, + int timeout_ms, std::vector & output_ids, std::string & error); @@ -44,11 +46,21 @@ class Qwen3ToolPredictorIpcClient { private: void close_locked(); - mutable std::mutex mutex_; + mutable std::timed_mutex mutex_; BackendIpcProcess process_; - bool active_ = false; + std::atomic active_{false}; }; +// Read one daemon response under a single wall-clock deadline. Kept outside +// the client so the timeout and partial-response behavior can be unit tested +// without loading a model. +bool read_qwen3_tool_predictor_response( + int stream_fd, + int max_tokens, + int timeout_ms, + std::vector & output_ids, + std::string & error); + int run_qwen3_tool_predictor_ipc_daemon(const char * model_path, int gpu, int max_ctx, diff --git a/server/src/server/http_server.cpp b/server/src/server/http_server.cpp index d079a1934..786f4f057 100644 --- a/server/src/server/http_server.cpp +++ b/server/src/server/http_server.cpp @@ -58,7 +58,7 @@ typedef long ssize_t; #define poll(fds,nfds,timeout) WSAPoll(fds,nfds,timeout) // Replace fcntl(F_GETFL) / fcntl(F_SETFL, O_NONBLOCK) with ioctlsocket static inline int sock_get_flags(SocketHandle fd) { (void)fd; return 0; /* stub */ } -static inline void sock_set_nonblock(SocketHandle fd) { u_long m = 1; ioctlsocket(fd, FIONBIO, &m); } +static inline bool sock_set_nonblock(SocketHandle fd) { u_long m = 1; return ioctlsocket(fd, FIONBIO, &m) == 0; } static inline void sock_set_block(SocketHandle fd) { u_long m = 0; ioctlsocket(fd, FIONBIO, &m); } static inline void socket_close(SocketHandle fd) { closesocket(fd); } #define SETSOCKOPT_CAST (const char *) @@ -76,7 +76,7 @@ static inline bool sock_is_eagain(int e) { return e == WSAEWOULDBLOCK; } #include #include static inline int sock_get_flags(SocketHandle fd) { return fcntl(fd, F_GETFL, 0); } -static inline void sock_set_nonblock(SocketHandle fd) { int f = fcntl(fd, F_GETFL, 0); if (f >= 0) fcntl(fd, F_SETFL, f | O_NONBLOCK); } +static inline bool sock_set_nonblock(SocketHandle fd) { int f = fcntl(fd, F_GETFL, 0); return f >= 0 && fcntl(fd, F_SETFL, f | O_NONBLOCK) == 0; } static inline void sock_set_block(SocketHandle fd) { int f = fcntl(fd, F_GETFL, 0); if (f >= 0) fcntl(fd, F_SETFL, f & ~O_NONBLOCK); } static inline void socket_close(SocketHandle fd) { ::close(fd); } #define SETSOCKOPT_CAST /* empty on POSIX */ @@ -502,19 +502,49 @@ bool parse_semantic_sidecar_url( ? "/" : value.substr(path_begin); if (out.host.empty() || out.port.empty() || out.path.empty()) return false; if (out.host.front() == '[' || out.host.find(':') != std::string::npos) { - // The production bridge is loopback IPv4. Reject ambiguous IPv6 - // authority parsing instead of silently connecting to the wrong host. + // Reject ambiguous IPv6 authority parsing instead of silently + // connecting to the wrong host. return false; } - return std::all_of(out.port.begin(), out.port.end(), [](unsigned char ch) { + if (!std::all_of(out.port.begin(), out.port.end(), [](unsigned char ch) { return std::isdigit(ch) != 0; - }); + })) { + return false; + } + try { + const unsigned long port = std::stoul(out.port); + if (port == 0 || port > 65535) return false; + } catch (...) { + return false; + } + // A numeric address makes the configured wall-clock deadline independent + // of an unbounded platform DNS resolver. Keep localhost as the ergonomic + // production spelling for an on-host predictor. + if (out.host == "localhost") out.host = "127.0.0.1"; + in_addr address{}; + return ::inet_pton(AF_INET, out.host.c_str(), &address) == 1; } bool semantic_sidecar_send_all( - SocketHandle fd, const char * data, size_t size) { + SocketHandle fd, + const char * data, + size_t size, + const std::chrono::steady_clock::time_point & deadline) { size_t sent = 0; while (sent < size) { + const auto now = std::chrono::steady_clock::now(); + if (now >= deadline) return false; + const int remaining_ms = std::max(1, static_cast( + std::chrono::duration_cast( + deadline - now).count())); + struct pollfd descriptor{fd, POLLOUT, 0}; + const int polled = poll(&descriptor, 1, remaining_ms); + if (polled < 0 && sock_is_eintr(sock_errno())) continue; + if (polled <= 0 || + !(descriptor.revents & POLLOUT) || + (descriptor.revents & (POLLERR | POLLHUP | POLLNVAL))) { + return false; + } #if defined(_WIN32) const int n = ::send( fd, data + sent, @@ -523,26 +553,83 @@ bool semantic_sidecar_send_all( const ssize_t n = ::send( fd, data + sent, size - sent, MSG_NOSIGNAL); #endif + if (n < 0 && (sock_is_eintr(sock_errno()) || + sock_is_eagain(sock_errno()))) { + continue; + } if (n <= 0) return false; sent += static_cast(n); } return true; } -void set_semantic_sidecar_socket_timeout(SocketHandle fd, int timeout_ms) { +bool semantic_sidecar_connect( + const SemanticSidecarUrl & url, + const std::chrono::steady_clock::time_point & deadline, + SocketHandle & fd) { + fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + if (!socket_is_valid(fd)) return false; + if (!sock_set_nonblock(fd)) { + socket_close(fd); + fd = kInvalidSocket; + return false; + } + + sockaddr_in address{}; + address.sin_family = AF_INET; + address.sin_port = htons(static_cast(std::stoul(url.port))); + if (::inet_pton(AF_INET, url.host.c_str(), &address.sin_addr) != 1) { + socket_close(fd); + fd = kInvalidSocket; + return false; + } + if (::connect(fd, reinterpret_cast(&address), + static_cast(sizeof(address))) == 0) { + return true; + } + const int connect_error = sock_errno(); #if defined(_WIN32) - const DWORD timeout = static_cast(timeout_ms); - setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, - reinterpret_cast(&timeout), sizeof(timeout)); - setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, - reinterpret_cast(&timeout), sizeof(timeout)); + const bool pending = connect_error == WSAEWOULDBLOCK || + connect_error == WSAEINPROGRESS || + connect_error == WSAEINVAL; #else - struct timeval timeout{}; - timeout.tv_sec = timeout_ms / 1000; - timeout.tv_usec = (timeout_ms % 1000) * 1000; - setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout)); - setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(timeout)); + const bool pending = connect_error == EINPROGRESS || + connect_error == EWOULDBLOCK; #endif + if (!pending) { + socket_close(fd); + fd = kInvalidSocket; + return false; + } + + struct pollfd descriptor{fd, POLLOUT, 0}; + int polled = -1; + while (true) { + const auto now = std::chrono::steady_clock::now(); + if (now >= deadline) { + socket_close(fd); + fd = kInvalidSocket; + return false; + } + const int remaining_ms = std::max(1, static_cast( + std::chrono::duration_cast( + deadline - now).count())); + descriptor.revents = 0; + polled = poll(&descriptor, 1, remaining_ms); + if (polled < 0 && sock_is_eintr(sock_errno())) continue; + break; + } + int socket_error = 0; + socklen_t error_size = static_cast(sizeof(socket_error)); + if (polled <= 0 || !(descriptor.revents & POLLOUT) || + getsockopt(fd, SOL_SOCKET, SO_ERROR, + reinterpret_cast(&socket_error), &error_size) != 0 || + socket_error != 0) { + socket_close(fd); + fd = kInvalidSocket; + return false; + } + return true; } std::string lowercase_ascii(std::string value) { @@ -588,6 +675,8 @@ SemanticToolPrediction request_semantic_tool_prediction( const json & payload, const json & request_tools) { const auto started = std::chrono::steady_clock::now(); + const auto deadline = started + + std::chrono::milliseconds(std::max(1, config.timeout_ms)); SemanticToolPrediction prediction; prediction.source = config.model; auto finish = [&]() { @@ -598,34 +687,12 @@ SemanticToolPrediction request_semantic_tool_prediction( SemanticSidecarUrl url; if (!parse_semantic_sidecar_url(config.url, url)) { - prediction.error = "predictor_url_must_be_http_host_port_path"; - return finish(); - } - - struct addrinfo hints{}; - hints.ai_family = AF_UNSPEC; - hints.ai_socktype = SOCK_STREAM; - struct addrinfo * addresses = nullptr; - if (getaddrinfo(url.host.c_str(), url.port.c_str(), &hints, &addresses) != 0) { - prediction.error = "predictor_host_resolution_failed"; + prediction.error = "predictor_url_must_use_http_numeric_ipv4"; return finish(); } SocketHandle fd = kInvalidSocket; - for (auto * address = addresses; address; address = address->ai_next) { - fd = socket(address->ai_family, address->ai_socktype, - address->ai_protocol); - if (!socket_is_valid(fd)) continue; - set_semantic_sidecar_socket_timeout(fd, config.timeout_ms); - if (connect(fd, address->ai_addr, - static_cast(address->ai_addrlen)) == 0) { - break; - } - socket_close(fd); - fd = kInvalidSocket; - } - freeaddrinfo(addresses); - if (!socket_is_valid(fd)) { + if (!semantic_sidecar_connect(url, deadline, fd)) { prediction.error = "predictor_connect_failed"; return finish(); } @@ -639,9 +706,11 @@ SemanticToolPrediction request_semantic_tool_prediction( "Connection: close\r\n" + "Content-Length: " + std::to_string(body.size()) + "\r\n\r\n" + body; - if (!semantic_sidecar_send_all(fd, request.data(), request.size())) { + if (!semantic_sidecar_send_all( + fd, request.data(), request.size(), deadline)) { socket_close(fd); - prediction.error = "predictor_send_failed"; + prediction.error = std::chrono::steady_clock::now() >= deadline + ? "predictor_timeout" : "predictor_send_failed"; return finish(); } @@ -649,6 +718,29 @@ SemanticToolPrediction request_semantic_tool_prediction( std::string response; std::array buffer{}; while (response.size() < kMaxResponseBytes) { + const auto now = std::chrono::steady_clock::now(); + if (now >= deadline) { + socket_close(fd); + prediction.error = "predictor_timeout"; + return finish(); + } + const int remaining_ms = std::max(1, static_cast( + std::chrono::duration_cast( + deadline - now).count())); + struct pollfd descriptor{fd, POLLIN | POLLHUP, 0}; + const int polled = poll(&descriptor, 1, remaining_ms); + if (polled < 0 && sock_is_eintr(sock_errno())) continue; + if (polled == 0) { + socket_close(fd); + prediction.error = "predictor_timeout"; + return finish(); + } + if (polled < 0 || + (descriptor.revents & (POLLERR | POLLNVAL))) { + socket_close(fd); + prediction.error = "predictor_receive_failed"; + return finish(); + } #if defined(_WIN32) const int received = recv( fd, buffer.data(), static_cast(buffer.size()), 0); @@ -657,8 +749,12 @@ SemanticToolPrediction request_semantic_tool_prediction( #endif if (received == 0) break; if (received < 0) { + if (sock_is_eintr(sock_errno()) || + sock_is_eagain(sock_errno())) { + continue; + } socket_close(fd); - prediction.error = "predictor_receive_failed_or_timed_out"; + prediction.error = "predictor_receive_failed"; return finish(); } response.append(buffer.data(), static_cast(received)); @@ -1025,10 +1121,6 @@ json build_props_body(const ServerConfig & config, {"decode_interference_qualified", lane.decode_interference_qualified}, {"accelerator_relation", lane.accelerator_relation}, - {"requires_static_model_routing", - lane.requires_static_model_routing}, - {"requires_unique_expert_ownership", - lane.requires_unique_expert_ownership}, }); } json body = { @@ -1149,27 +1241,15 @@ json build_props_body(const ServerConfig & config, {"allowed_tools", config.tool_speculation.allowed_tools}, {"max_model_slowdown_ratio", config.tool_speculation.max_model_slowdown_ratio}, - {"model_routing_static", - config.tool_speculation.model_routing_static}, - {"model_expert_ownership_unique", - config.tool_speculation.model_expert_ownership_unique}, {"compute_isolation", config.tool_speculation.cpu_affinity_isolated - ? "disjoint_cpu_affinity" - : config.tool_speculation.hip_reserved_tool_compute_units > 0 - ? "disjoint_hip_cu_masks" : "none"}, + ? "disjoint_cpu_affinity" : "none"}, {"cpu_affinity_isolated", config.tool_speculation.cpu_affinity_isolated}, {"tool_cpu_affinity", config.tool_speculation.cpu_affinity}, {"model_cpu_affinity", config.tool_speculation.model_cpu_affinity}, - {"hip_tool_device", - config.tool_speculation.hip_tool_device >= 0 - ? json(config.tool_speculation.hip_tool_device) - : json(nullptr)}, - {"hip_reserved_tool_compute_units", - config.tool_speculation.hip_reserved_tool_compute_units}, {"profile_lanes", tool_spec_lanes}, }}, {"sampling", { @@ -2387,6 +2467,7 @@ void HttpServer::launch_semantic_tool_prediction(ParsedRequest & req) const { req.semantic_tool_prediction = std::async( std::launch::async, [predictor, payload, tools, native]() { + const auto prediction_started = std::chrono::steady_clock::now(); SemanticToolPrediction native_result; if (native && native->active()) { native_result = native->predict(payload, tools); @@ -2395,12 +2476,35 @@ void HttpServer::launch_semantic_tool_prediction(ParsedRequest & req) const { } } if (predictor.http_enabled()) { + const double elapsed_ms = + std::chrono::duration( + std::chrono::steady_clock::now() - + prediction_started).count(); + const int remaining_ms = predictor.timeout_ms - + static_cast(std::ceil(elapsed_ms)); + if (remaining_ms <= 0) { + native_result.source = predictor.model; + native_result.ok = false; + native_result.error = native_result.error.empty() + ? "predictor_timeout" + : "native=" + native_result.error + + ";http=predictor_timeout"; + native_result.wall_ms = elapsed_ms; + return native_result; + } + SemanticToolPredictorConfig fallback_config = predictor; + fallback_config.timeout_ms = remaining_ms; SemanticToolPrediction fallback = - request_semantic_tool_prediction(predictor, payload, tools); + request_semantic_tool_prediction( + fallback_config, payload, tools); if (!fallback.ok && !native_result.error.empty()) { fallback.error = "native=" + native_result.error + ";http=" + fallback.error; } + fallback.wall_ms = + std::chrono::duration( + std::chrono::steady_clock::now() - + prediction_started).count(); return fallback; } if (native_result.error.empty()) { @@ -3955,16 +4059,9 @@ void HttpServer::prepare_generation_inputs( inputs.request.n_gen = inputs.generation_cap; inputs.request.sampler = req.sampler; inputs.request.do_sample = req.sampler.needs_logit_processing(); - // Tool prediction must never change the target decoder or trigger an - // autoregressive retry. DS4/DSpark remains authoritative on every arm. - const bool semantic_tool_request = - req.automatic_tool_speculation_enabled && - config_.tool_speculation.enabled() && - config_.semantic_tool_predictor.enabled() && - !req.tools.empty(); - inputs.request.allow_decode_mode_retry = - !req.tool_speculation.has_value() && - !semantic_tool_request; + // External tool prediction is deliberately absent from GenerateRequest. + // It must not change the target decoder, including the backend's normal + // empty-speculation recovery policy. // Tokens are delivered through DaemonIO so all API formats share the // same disconnect and streaming state machine. inputs.request.stream = false; @@ -4415,10 +4512,11 @@ void HttpServer::process_job(ServerJob * job) { if (job->client_disconnected.load(std::memory_order_acquire)) { client_disconnected = true; } - auto finish_tool_speculation = [&](bool cancel) -> std::optional { + auto finish_tool_speculation = [ + &](const char * cancel_reason) -> std::optional { if (tool_speculation) { - json metadata = cancel - ? tool_speculation->cancel("client_disconnected") + json metadata = cancel_reason + ? tool_speculation->cancel(cancel_reason) : tool_speculation->resolve(emitter.tool_calls()); metadata["prediction_source"] = "client"; return metadata; @@ -4429,8 +4527,8 @@ void HttpServer::process_job(ServerJob * job) { req.automatic_tool_speculation.get(); json metadata; if (launch.attempt) { - metadata = cancel - ? launch.attempt->cancel("client_disconnected") + metadata = cancel_reason + ? launch.attempt->cancel(cancel_reason) : launch.attempt->resolve(emitter.tool_calls()); } else { metadata = { @@ -4465,9 +4563,14 @@ void HttpServer::process_job(ServerJob * job) { }; } }; + // A partial tool call from a failed generation is not authoritative. + // Keep its speculative result private just as we do on disconnect. + const char * generation_cancel_reason = + result.ok() ? nullptr : "generation_failed"; if (req.stream && !client_disconnected) { auto final_chunks = emitter.emit_finish(completion_tokens, &gen_timings); - if (auto metadata = finish_tool_speculation(false)) { + if (auto metadata = finish_tool_speculation( + generation_cancel_reason)) { const std::string extension = render_tool_speculation_sse( req.format, req.response_id, req.model, *metadata); // Keep the standard terminal event last: [DONE], message_stop, @@ -4487,7 +4590,8 @@ void HttpServer::process_job(ServerJob * job) { } else if (!req.stream && !client_disconnected) { json response = build_non_streaming_response( req, result, n_gen_cap, gen_timings, tokenizer_, emitter); - if (auto metadata = finish_tool_speculation(false)) { + if (auto metadata = finish_tool_speculation( + generation_cancel_reason)) { response["dflash_tool_speculation"] = std::move(*metadata); } // Streaming uses non-blocking sends; restore blocking mode before @@ -4496,7 +4600,7 @@ void HttpServer::process_job(ServerJob * job) { send_response(fd, 200, "application/json", response.dump() + "\n"); } else { - finish_tool_speculation(true); + finish_tool_speculation("client_disconnected"); } if (client_disconnected) { diff --git a/server/src/server/native_semantic_tool_predictor.cpp b/server/src/server/native_semantic_tool_predictor.cpp index 9551d549c..c375e6b3d 100644 --- a/server/src/server/native_semantic_tool_predictor.cpp +++ b/server/src/server/native_semantic_tool_predictor.cpp @@ -1,5 +1,6 @@ #include "native_semantic_tool_predictor.h" +#include #include #include #include @@ -37,6 +38,8 @@ SemanticToolPrediction NativeSemanticToolPredictor::predict( const json & request_tools, std::string * generated_text) { const auto started = std::chrono::steady_clock::now(); + const auto deadline = started + + std::chrono::milliseconds(std::max(1, config_.timeout_ms)); SemanticToolPrediction prediction; prediction.source = "native-qwen3"; auto finish = [&]() { @@ -63,8 +66,16 @@ SemanticToolPrediction NativeSemanticToolPredictor::predict( return finish(); } + const auto now = std::chrono::steady_clock::now(); + if (now >= deadline) { + prediction.error = "native_predictor_timeout"; + return finish(); + } + const int remaining_ms = std::max(1, static_cast( + std::chrono::duration_cast( + deadline - now).count())); std::vector output_ids; - if (!ipc_.predict(prompt_ids, config_.max_tokens, + if (!ipc_.predict(prompt_ids, config_.max_tokens, remaining_ms, output_ids, prediction.error)) { return finish(); } diff --git a/server/src/server/server_main.cpp b/server/src/server/server_main.cpp index 02b915f0f..fe1ec19ec 100644 --- a/server/src/server/server_main.cpp +++ b/server/src/server/server_main.cpp @@ -12,9 +12,6 @@ // [--max-tokens 4096] [--target-device auto:0] #include "http_server.h" -#if defined(DFLASH27B_BACKEND_HIP) -#include "tool_speculation_hip_probe.h" -#endif #include "chat_template.h" #include "model_card.h" #include "common/backend_factory.h" @@ -206,10 +203,9 @@ static void print_usage(const char * prog) { " overlap is an experimental throughput mode.\n" " --tool-hint-native-work-dir \n" " Optional private IPC scratch directory.\n" - " --tool-hint-sidecar-timeout-ms \n" - " Hard sidecar deadline (default: 2000).\n" - " --tool-hint-sidecar-max-tokens \n" - " Predictor completion cap (default: 96).\n" + " --tool-hint-timeout-ms Hard native/HTTP predictor deadline\n" + " (default: 2000).\n" + " --tool-hint-max-tokens Predictor completion cap (default: 96).\n" " --tool-hint-execution-confidence

\n" " Calibrated 0..1 prior for automatic external\n" " tool admission (default: 0.75).\n" @@ -219,11 +215,6 @@ static void print_usage(const char * prog) { " --tool-spec-executor Trusted executor adapter. Receives one\n" " dflash.tool-speculation.v1 JSON request\n" " on stdin; no shell is used.\n" -#if defined(DFLASH27B_BACKEND_HIP) - " --tool-spec-hip-sgemm-probe \n" - " Benchmark-only trusted in-process HIP\n" - " executor with a per-lane CU-masked stream.\n" -#endif " --tool-spec-profile Measured resource-lane frontier JSON.\n" " --tool-spec-allow Allow one read-only/idempotent tool; repeatable.\n" " --tool-spec-cpu-affinity \n" @@ -287,9 +278,6 @@ int main(int argc, char ** argv) { // Parse arguments. BackendArgs bargs; ServerConfig sconfig; - int tool_spec_hip_probe_device = -1; - int tool_spec_hip_probe_matrix = 0; - int tool_spec_hip_probe_total_cus = 0; bargs.model_path = argv[1]; bool spark_autotune = false; // --spark: self-tuning hot/cold MoE residency int spark_slots = -1; // --spark-slots: explicit cache slots/layer (-1=auto) @@ -660,20 +648,22 @@ int main(int argc, char ** argv) { "before-model or overlap\n"); return 2; } - } else if (std::strcmp(argv[i], "--tool-hint-sidecar-timeout-ms") == 0 && + } else if ((std::strcmp(argv[i], "--tool-hint-timeout-ms") == 0 || + std::strcmp(argv[i], "--tool-hint-sidecar-timeout-ms") == 0) && i + 1 < argc) { sconfig.semantic_tool_predictor.timeout_ms = std::atoi(argv[++i]); if (sconfig.semantic_tool_predictor.timeout_ms <= 0) { std::fprintf(stderr, - "[server] --tool-hint-sidecar-timeout-ms must be positive\n"); + "[server] --tool-hint-timeout-ms must be positive\n"); return 2; } - } else if (std::strcmp(argv[i], "--tool-hint-sidecar-max-tokens") == 0 && + } else if ((std::strcmp(argv[i], "--tool-hint-max-tokens") == 0 || + std::strcmp(argv[i], "--tool-hint-sidecar-max-tokens") == 0) && i + 1 < argc) { sconfig.semantic_tool_predictor.max_tokens = std::atoi(argv[++i]); if (sconfig.semantic_tool_predictor.max_tokens <= 0) { std::fprintf(stderr, - "[server] --tool-hint-sidecar-max-tokens must be positive\n"); + "[server] --tool-hint-max-tokens must be positive\n"); return 2; } } else if (std::strcmp( @@ -692,28 +682,6 @@ int main(int argc, char ** argv) { } else if (std::strcmp(argv[i], "--tool-spec-executor") == 0 && i + 1 < argc) { sconfig.tool_speculation.executor_path = argv[++i]; -#if defined(DFLASH27B_BACKEND_HIP) - } else if (std::strcmp(argv[i], "--tool-spec-hip-sgemm-probe") == 0 && - i + 1 < argc) { - const char * value = argv[++i]; - char * separator = nullptr; - const long device = std::strtol(value, &separator, 10); - if (separator == value || !separator || *separator != ':') { - std::fprintf(stderr, - "[server] --tool-spec-hip-sgemm-probe expects DEVICE:MATRIX\n"); - return 2; - } - char * end = nullptr; - const long matrix = std::strtol(separator + 1, &end, 10); - if (!end || *end != '\0' || device < 0 || - matrix <= 0 || matrix > 8192) { - std::fprintf(stderr, - "[server] invalid --tool-spec-hip-sgemm-probe DEVICE:MATRIX\n"); - return 2; - } - tool_spec_hip_probe_device = static_cast(device); - tool_spec_hip_probe_matrix = static_cast(matrix); -#endif } else if (std::strcmp(argv[i], "--tool-spec-profile") == 0 && i + 1 < argc) { sconfig.tool_speculation.profile_path = argv[++i]; @@ -819,21 +787,6 @@ int main(int argc, char ** argv) { } } - if (tool_spec_hip_probe_device >= 0) { -#if defined(DFLASH27B_BACKEND_HIP) - std::string executor_error; - sconfig.tool_speculation.in_process_executor = - create_hip_sgemm_tool_speculation_executor( - tool_spec_hip_probe_device, - tool_spec_hip_probe_matrix, - tool_spec_hip_probe_total_cus, - executor_error); - if (!sconfig.tool_speculation.in_process_executor) { - std::fprintf(stderr, "[server] %s\n", executor_error.c_str()); - return 2; - } -#endif - } const bool semantic_http_predictor_requested = !sconfig.semantic_tool_predictor.url.empty() || !sconfig.semantic_tool_predictor.model.empty(); @@ -858,33 +811,20 @@ int main(int argc, char ** argv) { } const bool tool_speculation_requested = !sconfig.tool_speculation.executor_path.empty() || - static_cast(sconfig.tool_speculation.in_process_executor) || !sconfig.tool_speculation.profile_path.empty() || !sconfig.tool_speculation.allowed_tools.empty() || !sconfig.tool_speculation.cpu_affinity.empty(); if (tool_speculation_requested) { - sconfig.tool_speculation.model_routing_static = - !environment_flag_enabled( - "DFLASH_MOE_TP_DYNAMIC_ROUTE_BALANCE") && - !environment_flag_enabled( - "DFLASH_DS4_TP_DYNAMIC_ROUTE_BALANCE"); - sconfig.tool_speculation.model_expert_ownership_unique = - !environment_flag_enabled("DFLASH_MOE_DUPLICATE_HOT_ON_COLD"); - const bool has_child_executor = - !sconfig.tool_speculation.executor_path.empty(); - const bool has_in_process_executor = - static_cast(sconfig.tool_speculation.in_process_executor); - if (has_child_executor == has_in_process_executor || + if (sconfig.tool_speculation.executor_path.empty() || sconfig.tool_speculation.profile_path.empty() || sconfig.tool_speculation.allowed_tools.empty()) { std::fprintf(stderr, - "[server] tool speculation requires exactly one executor, " + "[server] tool speculation requires --tool-spec-executor, " "--tool-spec-profile, and at least one --tool-spec-allow\n"); return 2; } #if !defined(_WIN32) - if (has_child_executor && - ::access(sconfig.tool_speculation.executor_path.c_str(), X_OK) != 0) { + if (::access(sconfig.tool_speculation.executor_path.c_str(), X_OK) != 0) { std::fprintf(stderr, "[server] tool speculation executor is not executable: %s\n", sconfig.tool_speculation.executor_path.c_str()); @@ -927,79 +867,6 @@ int main(int argc, char ** argv) { sconfig.tool_speculation.execution_mode()); return 2; } - if (sconfig.tool_speculation.policy.benchmark_only() && - !has_in_process_executor) { - std::fprintf(stderr, - "[server] provisional_benchmark_only tool profiles cannot " - "enable an external production executor\n"); - return 2; - } - if (has_in_process_executor) { - int max_same_gpu_percentage = 0; - for (const auto & lane : - sconfig.tool_speculation.policy.lanes()) { - if (lane.decode_interference_qualified && - lane.accelerator_relation == "same_physical_gpu") { - max_same_gpu_percentage = std::max( - max_same_gpu_percentage, - lane.resource_percentage); - } - } - if (max_same_gpu_percentage > 0) { - const int reserved_cus = - (tool_spec_hip_probe_total_cus * - max_same_gpu_percentage + - 99) / - 100; - if (reserved_cus <= 0 || - reserved_cus >= tool_spec_hip_probe_total_cus) { - std::fprintf(stderr, - "[server] same-GPU HIP tool lane would reserve %d " - "of %d CUs; at least one model CU is required\n", - reserved_cus, tool_spec_hip_probe_total_cus); - return 2; - } - const std::string isolation = - std::to_string(tool_spec_hip_probe_device) + ":" + - std::to_string(reserved_cus); - const char * existing = - std::getenv("DFLASH_HIP_RESERVED_TOOL_LANE"); - if (existing && *existing && isolation != existing) { - std::fprintf(stderr, - "[server] DFLASH_HIP_RESERVED_TOOL_LANE=%s " - "conflicts with profile-required %s\n", - existing, isolation.c_str()); - return 2; - } - set_environment_variable( - "DFLASH_HIP_RESERVED_TOOL_LANE", - isolation.c_str(), true); - sconfig.tool_speculation.hip_tool_device = - tool_spec_hip_probe_device; - sconfig.tool_speculation.hip_reserved_tool_compute_units = - reserved_cus; - std::fprintf(stderr, - "[server] disjoint HIP tool lane: device %d reserves " - "%d/%d low CU(s); model streams use the complement\n", - tool_spec_hip_probe_device, reserved_cus, - tool_spec_hip_probe_total_cus); - } - } - if (sconfig.tool_speculation.policy.requires_static_model_routing() && - !sconfig.tool_speculation.model_routing_static) { - std::fprintf(stderr, - "[server] same-physical-GPU tool lanes require static model " - "routing; disable DFLASH_MOE_TP_DYNAMIC_ROUTE_BALANCE and " - "DFLASH_DS4_TP_DYNAMIC_ROUTE_BALANCE\n"); - return 2; - } - if (sconfig.tool_speculation.policy.requires_unique_expert_ownership() && - !sconfig.tool_speculation.model_expert_ownership_unique) { - std::fprintf(stderr, - "[server] same-physical-GPU tool lanes require unique expert " - "ownership; disable DFLASH_MOE_DUPLICATE_HOT_ON_COLD\n"); - return 2; - } std::fprintf(stderr, "[server] tool speculation preserves token speculation; " "unqualified resource lanes are deferred\n"); @@ -1541,12 +1408,6 @@ int main(int argc, char ** argv) { sconfig.tool_speculation.profile_path.c_str()); std::fprintf(stderr, "[server] │ tool_spec_decode = %s\n", "spec preserved (unqualified lanes deferred)"); - std::fprintf(stderr, "[server] │ tool_spec_routing= %s\n", - sconfig.tool_speculation.model_routing_static - ? "static" : "dynamic"); - std::fprintf(stderr, "[server] │ tool_spec_experts= %s\n", - sconfig.tool_speculation.model_expert_ownership_unique - ? "unique ownership" : "duplicated ownership"); std::fprintf(stderr, "[server] │ tool_spec_lanes ="); for (const auto & lane : sconfig.tool_speculation.policy.lanes()) { std::fprintf(stderr, " %d%%:%s", diff --git a/server/src/server/tool_speculation.cpp b/server/src/server/tool_speculation.cpp index c3d679a3c..85f6b44fa 100644 --- a/server/src/server/tool_speculation.cpp +++ b/server/src/server/tool_speculation.cpp @@ -1,3 +1,7 @@ +#if defined(__linux__) && !defined(_GNU_SOURCE) +# define _GNU_SOURCE +#endif + #include "tool_speculation.h" #include @@ -20,6 +24,7 @@ # include # include # if defined(__linux__) +# include # include # endif extern char ** environ; @@ -84,6 +89,16 @@ bool send_all_socket(int fd, const void * data, size_t bytes) { return true; } +void signal_executor_process_group( + pid_t leader, int signal, bool leader_fallback = true) { + if (leader <= 0) return; + if (::kill(-leader, signal) != 0 && errno == ESRCH && leader_fallback) { + // Defensive fallback for older platforms that ignored the requested + // spawn process group. + (void)::kill(leader, signal); + } +} + std::vector executor_environment( int resource_percentage, const std::string & accelerator_relation, @@ -101,9 +116,14 @@ std::vector executor_environment( "DFLASH_TOOL_SPECULATION_CPU_AFFINITY="; static constexpr size_t kCpuAffinityKeyLen = sizeof("DFLASH_TOOL_SPECULATION_CPU_AFFINITY=") - 1; + static constexpr const char * kEnabledKey = + "DFLASH_TOOL_SPECULATION="; + static constexpr size_t kEnabledKeyLen = + sizeof("DFLASH_TOOL_SPECULATION=") - 1; bool resource_replaced = false; bool relation_replaced = false; bool cpu_affinity_replaced = false; + bool enabled_replaced = false; for (char ** item = environ; item && *item; ++item) { const std::string value(*item); if (value.compare(0, kResourceKeyLen, kResourceKey) == 0) { @@ -125,6 +145,9 @@ std::vector executor_environment( format_cpu_affinity(cpu_affinity)); cpu_affinity_replaced = true; } + } else if (value.compare(0, kEnabledKeyLen, kEnabledKey) == 0) { + values.push_back(std::string(kEnabledKey) + "1"); + enabled_replaced = true; } else { values.push_back(value); } @@ -142,7 +165,9 @@ std::vector executor_environment( std::string(kCpuAffinityKey) + format_cpu_affinity(cpu_affinity)); } - values.push_back("DFLASH_TOOL_SPECULATION=1"); + if (!enabled_replaced) { + values.push_back(std::string(kEnabledKey) + "1"); + } return values; } @@ -380,10 +405,6 @@ bool qualify_tool_speculation_cpu_affinity( return true; } #if defined(__linux__) - if (config.executor_path.empty() || config.in_process_executor) { - error = "tool CPU affinity requires a child-process executor"; - return false; - } const long configured_cpus = ::sysconf(_SC_NPROCESSORS_CONF); if (configured_cpus <= 0) { error = "cannot determine configured CPU count"; @@ -465,29 +486,24 @@ bool ToolSpeculationPolicy::load_json( return false; } - if (report.contains("profile_status")) { - if (!report["profile_status"].is_string()) { - error = "tool-speculation profile_status must be a string"; - return false; - } - profile_status_ = report["profile_status"].get(); - if (profile_status_ != "qualified" && - profile_status_ != "provisional_benchmark_only") { - error = "tool-speculation profile_status must be qualified or " - "provisional_benchmark_only"; - return false; - } + if (!report.contains("profile_status") || + !report["profile_status"].is_string()) { + error = "tool-speculation profile needs profile_status=qualified"; + return false; } - if (report.contains("executor")) { - if (!report["executor"].is_string()) { - error = "tool-speculation executor contract must be a string"; - return false; - } - executor_contract_ = report["executor"].get(); - if (executor_contract_.empty()) { - error = "tool-speculation executor contract cannot be empty"; - return false; - } + profile_status_ = report["profile_status"].get(); + if (profile_status_ != "qualified") { + error = "tool-speculation profile_status must be qualified"; + return false; + } + if (!report.contains("executor") || !report["executor"].is_string()) { + error = "tool-speculation profile needs an executor contract"; + return false; + } + executor_contract_ = report["executor"].get(); + if (executor_contract_.empty()) { + error = "tool-speculation executor contract cannot be empty"; + return false; } std::vector controls; @@ -520,34 +536,18 @@ bool ToolSpeculationPolicy::load_json( decode_interference_qualified = paths["decode_interference_qualified"].get(); } - const std::string accelerator_relation = - paths.value("accelerator_relation", "unspecified"); - if (accelerator_relation != "unspecified" && - accelerator_relation != "non_accelerator" && - accelerator_relation != "separate_physical_gpu" && - accelerator_relation != "same_physical_gpu") { + if (!paths.contains("accelerator_relation") || + !paths["accelerator_relation"].is_string()) { throw std::runtime_error( - "accelerator_relation must be unspecified, " - "non_accelerator, separate_physical_gpu, or " - "same_physical_gpu"); - } - bool requires_static_model_routing = false; - if (paths.contains("requires_static_model_routing")) { - if (!paths["requires_static_model_routing"].is_boolean()) { - throw std::runtime_error( - "requires_static_model_routing must be boolean"); - } - requires_static_model_routing = - paths["requires_static_model_routing"].get(); + "accelerator_relation must be explicit"); } - bool requires_unique_expert_ownership = false; - if (paths.contains("requires_unique_expert_ownership")) { - if (!paths["requires_unique_expert_ownership"].is_boolean()) { - throw std::runtime_error( - "requires_unique_expert_ownership must be boolean"); - } - requires_unique_expert_ownership = - paths["requires_unique_expert_ownership"].get(); + const std::string accelerator_relation = + paths["accelerator_relation"].get(); + if (accelerator_relation != "non_accelerator" && + accelerator_relation != "separate_physical_gpu") { + throw std::runtime_error( + "accelerator_relation must be non_accelerator or " + "separate_physical_gpu"); } if (!finite_positive(hit_control) || !finite_positive(miss_control) || @@ -565,8 +565,6 @@ bool ToolSpeculationPolicy::load_json( 1.0 + slowdown_percent / 100.0, decode_interference_qualified, accelerator_relation, - requires_static_model_routing, - requires_unique_expert_ownership, }); controls.push_back(control); } @@ -598,22 +596,6 @@ bool ToolSpeculationPolicy::load_json( return true; } -bool ToolSpeculationPolicy::requires_static_model_routing() const { - return std::any_of( - lanes_.begin(), lanes_.end(), - [](const ToolSpeculationLane & lane) { - return lane.requires_static_model_routing; - }); -} - -bool ToolSpeculationPolicy::requires_unique_expert_ownership() const { - return std::any_of( - lanes_.begin(), lanes_.end(), - [](const ToolSpeculationLane & lane) { - return lane.requires_unique_expert_ownership; - }); -} - ToolSpeculationAdmission ToolSpeculationPolicy::choose( double confidence, double max_model_slowdown_ratio) const { @@ -722,20 +704,6 @@ void ToolSpeculationAttempt::start() { }}, }; started_at_ = std::chrono::steady_clock::now(); - if (config_.in_process_executor) { - in_process_execution_ = - config_.in_process_executor->start(request, launch_error_); - running_ = static_cast(in_process_execution_); - if (running_) { - std::fprintf(stderr, - "[tool-spec] launched request=%s tool=%s confidence=%.3f " - "resource=%d%% mode=%s\n", - request_id_.c_str(), prediction_.call.name.c_str(), - prediction_.confidence, admission_.resource_percentage, - config_.execution_mode()); - } - return; - } #if defined(_WIN32) launch_error_ = "tool speculation child executors are not implemented on Windows"; return; @@ -769,6 +737,19 @@ void ToolSpeculationAttempt::start() { posix_spawn_file_actions_t actions; int spawn_status = posix_spawn_file_actions_init(&actions); const bool actions_initialized = spawn_status == 0; + posix_spawnattr_t attributes; + const int attributes_status = posix_spawnattr_init(&attributes); + const bool attributes_initialized = attributes_status == 0; + if (spawn_status == 0 && attributes_status != 0) { + spawn_status = attributes_status; + } + if (spawn_status == 0) { + spawn_status = posix_spawnattr_setpgroup(&attributes, 0); + } + if (spawn_status == 0) { + spawn_status = posix_spawnattr_setflags( + &attributes, POSIX_SPAWN_SETPGROUP); + } if (spawn_status == 0) { spawn_status = posix_spawn_file_actions_adddup2( &actions, input_socket[1], STDIN_FILENO); @@ -789,6 +770,17 @@ void ToolSpeculationAttempt::start() { if (spawn_status == 0 && output_pipe[1] != STDOUT_FILENO) { spawn_status = posix_spawn_file_actions_addclose(&actions, output_pipe[1]); } +# if defined(__GLIBC__) && defined(__GLIBC_PREREQ) +# if __GLIBC_PREREQ(2, 34) + // The executor receives only stdin/stdout/stderr. In particular, it must + // not inherit listening sockets, live client connections, model IPC pipes, + // or accelerator descriptors from the long-running server. + if (spawn_status == 0) { + spawn_status = posix_spawn_file_actions_addclosefrom_np( + &actions, STDERR_FILENO + 1); + } +# endif +# endif std::vector env_storage = executor_environment( admission_.resource_percentage, admission_.accelerator_relation, @@ -808,11 +800,15 @@ void ToolSpeculationAttempt::start() { pid_t child = -1; if (spawn_status == 0) { spawn_status = ::posix_spawn( - &child, executable.c_str(), &actions, nullptr, argv, env.data()); + &child, executable.c_str(), &actions, &attributes, argv, + env.data()); } if (actions_initialized) { posix_spawn_file_actions_destroy(&actions); } + if (attributes_initialized) { + posix_spawnattr_destroy(&attributes); + } ::close(input_socket[1]); ::close(output_pipe[1]); if (spawn_status != 0) { @@ -828,7 +824,7 @@ void ToolSpeculationAttempt::start() { std::string affinity_error; if (!pin_and_verify_child_cpu_affinity( child, config_.cpu_affinity, affinity_error)) { - ::kill(child, SIGKILL); + signal_executor_process_group(child, SIGKILL); int child_status = 0; while (::waitpid(child, &child_status, 0) < 0 && errno == EINTR) {} ::close(input_socket[0]); @@ -839,7 +835,7 @@ void ToolSpeculationAttempt::start() { } # else if (!config_.cpu_affinity.empty()) { - ::kill(child, SIGKILL); + signal_executor_process_group(child, SIGKILL); int child_status = 0; while (::waitpid(child, &child_status, 0) < 0 && errno == EINTR) {} ::close(input_socket[0]); @@ -900,10 +896,6 @@ json ToolSpeculationAttempt::base_metadata() const { } bool ToolSpeculationAttempt::send_control(const char * operation) { - if (in_process_execution_) { - return operation && *operation && - in_process_execution_->send_control(operation); - } #if defined(_WIN32) (void)operation; return false; @@ -921,12 +913,6 @@ bool ToolSpeculationAttempt::send_control(const char * operation) { } void ToolSpeculationAttempt::terminate_executor(bool allow_control_grace) { - if (in_process_execution_) { - in_process_execution_->terminate(allow_control_grace); - in_process_execution_.reset(); - running_ = false; - return; - } #if !defined(_WIN32) if (child_stdin_fd_ >= 0) { ::close(child_stdin_fd_); @@ -956,17 +942,21 @@ void ToolSpeculationAttempt::terminate_executor(bool allow_control_grace) { if (allow_control_grace && wait_until( std::chrono::steady_clock::now() + std::chrono::milliseconds(grace_ms))) { + // The executor contract does not permit detached descendants. + // Clean up any process that outlived its group leader. + signal_executor_process_group(pid, SIGKILL, false); return; } - ::kill(pid, SIGTERM); + signal_executor_process_group(pid, SIGTERM); const int term_grace_ms = allow_control_grace ? std::min(20, grace_ms) : grace_ms; if (wait_until(std::chrono::steady_clock::now() + std::chrono::milliseconds(term_grace_ms))) { + signal_executor_process_group(pid, SIGKILL, false); return; } int status = 0; - ::kill(pid, SIGKILL); + signal_executor_process_group(pid, SIGKILL); while (::waitpid(pid, &status, 0) < 0 && errno == EINTR) {} child_pid_ = -1; } @@ -978,14 +968,6 @@ bool ToolSpeculationAttempt::collect_executor_result( json & result, double & wait_ms, std::string & error) { - if (in_process_execution_) { - const bool ok = in_process_execution_->collect_result( - config_.timeout_ms, config_.max_result_bytes, - result, wait_ms, error); - in_process_execution_.reset(); - running_ = false; - return ok; - } #if defined(_WIN32) (void)result; wait_ms = 0.0; @@ -993,8 +975,14 @@ bool ToolSpeculationAttempt::collect_executor_result( return false; #else const auto wait_started = std::chrono::steady_clock::now(); - const auto deadline = wait_started + + const auto deadline = started_at_ + std::chrono::milliseconds(std::max(1, config_.timeout_ms)); + if (wait_started >= deadline) { + error = "executor_timeout"; + terminate_executor(); + wait_ms = 0.0; + return false; + } std::string output; bool eof = false; while (!eof) { @@ -1074,6 +1062,9 @@ bool ToolSpeculationAttempt::collect_executor_result( } std::this_thread::sleep_for(std::chrono::milliseconds(1)); } + // A successful adapter must not leave detached subprocesses behind. + signal_executor_process_group( + static_cast(child_pid_), SIGKILL, false); child_pid_ = -1; running_ = false; wait_ms = std::chrono::duration( diff --git a/server/src/server/tool_speculation.h b/server/src/server/tool_speculation.h index 2e38fadb9..07c8de813 100644 --- a/server/src/server/tool_speculation.h +++ b/server/src/server/tool_speculation.h @@ -84,11 +84,9 @@ struct ToolSpeculationLane { // token speculation is never disabled or replaced with AR decode. bool decode_interference_qualified = false; // Physical relationship between the tool accelerator and the model's - // primary accelerator. Same-GPU lanes have stricter runtime requirements - // because stream priority and CU masks do not isolate shared kernels. + // primary accelerator. Production child executors may use CPU/I/O or a + // separate physical accelerator, but never the model's accelerator. std::string accelerator_relation = "unspecified"; - bool requires_static_model_routing = false; - bool requires_unique_expert_ownership = false; }; struct ToolSpeculationAdmission { @@ -101,32 +99,6 @@ struct ToolSpeculationAdmission { std::string reason; }; -// Optional trusted in-process executor. This avoids a second accelerator -// process/context on runtimes where process-level time-slicing defeats CU or -// stream isolation. Implementations remain behind the same allowlist, -// empirical admission policy, exact-call commit, and private-result boundary -// as the child-process adapter. -class ToolSpeculationExecution { -public: - virtual ~ToolSpeculationExecution() = default; - virtual bool send_control(const std::string & operation) = 0; - virtual bool collect_result(int timeout_ms, - size_t max_result_bytes, - json & result, - double & wait_ms, - std::string & error) = 0; - virtual void terminate(bool allow_control_grace) = 0; -}; - -class ToolSpeculationExecutor { -public: - virtual ~ToolSpeculationExecutor() = default; - virtual std::unique_ptr start( - const json & request, - std::string & error) = 0; - virtual const char * mode_name() const = 0; -}; - // Runtime policy loaded from a qualification report's `path_summary`. This // keeps backend-specific interference measurements out of hard-coded engine // heuristics. @@ -144,11 +116,6 @@ class ToolSpeculationPolicy { const std::vector & lanes() const { return lanes_; } const std::string & profile_status() const { return profile_status_; } const std::string & executor_contract() const { return executor_contract_; } - bool benchmark_only() const { - return profile_status_ == "provisional_benchmark_only"; - } - bool requires_static_model_routing() const; - bool requires_unique_expert_ownership() const; private: std::vector lanes_; @@ -159,7 +126,6 @@ class ToolSpeculationPolicy { struct ToolSpeculationConfig { std::string executor_path; - std::shared_ptr in_process_executor; std::string profile_path; std::vector allowed_tools; ToolSpeculationPolicy policy; @@ -167,14 +133,6 @@ struct ToolSpeculationConfig { int cancel_grace_ms = 100; size_t max_result_bytes = 1024 * 1024; double max_model_slowdown_ratio = 1.20; - // Snapshot of the model routing mode used to validate profile/runtime - // compatibility at startup and expose it through /props. - bool model_routing_static = true; - bool model_expert_ownership_unique = true; - // Runtime evidence that the model and an in-process HIP tool use - // complementary CU masks. Zero means no model-side CU reservation. - int hip_tool_device = -1; - int hip_reserved_tool_compute_units = 0; // Optional child-process CPU lane. Startup verifies that these logical // CPUs are disjoint from the model process affinity; every child is pinned // and re-read before its request payload is released. @@ -182,18 +140,15 @@ struct ToolSpeculationConfig { std::vector model_cpu_affinity; bool cpu_affinity_isolated = false; bool enabled() const { - return (!executor_path.empty() || in_process_executor) && - !allowed_tools.empty() && + return !executor_path.empty() && !allowed_tools.empty() && !policy.empty(); } const char * execution_mode() const { - return in_process_executor - ? in_process_executor->mode_name() - : executor_path.empty() - ? "disabled" - : cpu_affinity.empty() - ? "child_process" - : "child_process_cpu_affinity"; + return executor_path.empty() + ? "disabled" + : cpu_affinity.empty() + ? "child_process" + : "child_process_cpu_affinity"; } bool allows(const std::string & name) const; }; @@ -257,8 +212,6 @@ class ToolSpeculationAttempt { bool running_ = false; bool resolved_ = false; std::string launch_error_; - std::unique_ptr in_process_execution_; - #if !defined(_WIN32) int child_stdin_fd_ = -1; int child_stdout_fd_ = -1; diff --git a/server/src/server/tool_speculation_hip_probe.cpp b/server/src/server/tool_speculation_hip_probe.cpp deleted file mode 100644 index 3c6155617..000000000 --- a/server/src/server/tool_speculation_hip_probe.cpp +++ /dev/null @@ -1,456 +0,0 @@ -#include "tool_speculation_hip_probe.h" - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace dflash::common { -namespace { - -std::string hip_error(const char * operation, hipError_t status) { - return std::string(operation) + ": " + hipGetErrorString(status); -} - -std::string hipblas_error(const char * operation, hipblasStatus_t status) { - return std::string(operation) + " failed with status " + - std::to_string(static_cast(status)); -} - -// HIP's current device is thread-local process state. The HTTP worker that -// launches a tool may immediately continue into model inference, so a trusted -// executor must leave that state exactly as it found it. -class ScopedHipDevice final { -public: - explicit ScopedHipDevice(int device) { - status_ = hipGetDevice(&previous_device_); - if (status_ != hipSuccess) return; - if (previous_device_ == device) return; - status_ = hipSetDevice(device); - switched_ = status_ == hipSuccess; - } - - ~ScopedHipDevice() { - if (switched_) (void) hipSetDevice(previous_device_); - } - - bool ok() const { return status_ == hipSuccess; } - hipError_t status() const { return status_; } - -private: - int previous_device_ = 0; - hipError_t status_ = hipSuccess; - bool switched_ = false; -}; - -class HipSgemmState; - -class HipSgemmExecution final : public ToolSpeculationExecution { -public: - explicit HipSgemmExecution(std::shared_ptr state) - : state_(std::move(state)) {} - ~HipSgemmExecution() override; - - bool send_control(const std::string & operation) override; - bool collect_result(int timeout_ms, - size_t max_result_bytes, - json & result, - double & wait_ms, - std::string & error) override; - void terminate(bool allow_control_grace) override; - -private: - std::shared_ptr state_; - bool committed_ = false; - bool finished_ = false; -}; - -class HipSgemmState final { -public: - HipSgemmState(int device, int matrix_size, int total_cus) - : device_(device), matrix_size_(matrix_size), total_cus_(total_cus) {} - - ~HipSgemmState() { - std::lock_guard lock(mutex_); - ScopedHipDevice device(device_); - if (!device.ok()) return; - if (stream_) (void) hipStreamSynchronize(stream_); - if (finished_) (void) hipEventDestroy(finished_); - if (started_) (void) hipEventDestroy(started_); - if (handle_) (void) hipblasDestroy(handle_); - if (c_) (void) hipFree(c_); - if (b_) (void) hipFree(b_); - if (a_) (void) hipFree(a_); - if (stream_) (void) hipStreamDestroy(stream_); - } - - bool start(const json & request, std::string & error) { - ScopedHipDevice device(device_); - if (!device.ok()) { - error = hip_error("hipSetDevice(tool)", device.status()); - return false; - } - std::lock_guard lock(mutex_); - if (active_) { - error = "HIP probe already has active work"; - return false; - } - try { - const json & call = request.at("call"); - if (call.at("name").get() != "benchmark_hip_sgemm") { - error = "HIP probe only supports benchmark_hip_sgemm"; - return false; - } - const json & arguments = call.at("arguments"); - if (!arguments.is_object() || - !arguments.contains("iterations") || - !arguments["iterations"].is_number_integer()) { - error = "benchmark_hip_sgemm.iterations must be an integer"; - return false; - } - iterations_ = arguments["iterations"].get(); - if (iterations_ <= 0 || iterations_ > 1'000'000) { - error = "benchmark_hip_sgemm.iterations must be 1..1000000"; - return false; - } - const int resource_percentage = - request.at("resource_percentage").get(); - if (resource_percentage <= 0 || resource_percentage > 100) { - error = "resource_percentage must be 1..100"; - return false; - } - const int cu_count = std::clamp( - (total_cus_ * resource_percentage + 99) / 100, - 1, total_cus_); - if (!ensure_resources(cu_count, error)) return false; - - const float alpha = 1.0F; - const float beta = 0.0F; - hipError_t status = hipEventRecord(started_, stream_); - if (status != hipSuccess) { - error = hip_error("hipEventRecord(started)", status); - return false; - } - for (int iteration = 0; iteration < iterations_; ++iteration) { - const hipblasStatus_t blas_status = hipblasSgemm( - handle_, HIPBLAS_OP_N, HIPBLAS_OP_N, - matrix_size_, matrix_size_, matrix_size_, - &alpha, a_, matrix_size_, b_, matrix_size_, - &beta, c_, matrix_size_); - if (blas_status != HIPBLAS_STATUS_SUCCESS) { - error = hipblas_error("hipblasSgemm", blas_status); - (void) hipStreamSynchronize(stream_); - return false; - } - } - status = hipEventRecord(finished_, stream_); - if (status != hipSuccess) { - error = hip_error("hipEventRecord(finished)", status); - (void) hipStreamSynchronize(stream_); - return false; - } - active_ = true; - error.clear(); - return true; - } catch (const std::exception & exception) { - error = std::string("invalid HIP probe request: ") + exception.what(); - return false; - } - } - - bool collect(int timeout_ms, - size_t max_result_bytes, - json & result, - double & wait_ms, - std::string & error) { - ScopedHipDevice device(device_); - if (!device.ok()) { - error = hip_error("hipSetDevice(tool)", device.status()); - release_active(); - return false; - } - const auto wait_started = std::chrono::steady_clock::now(); - const auto deadline = wait_started + - std::chrono::milliseconds(std::max(1, timeout_ms)); - while (true) { - const hipError_t status = hipEventQuery(finished_); - if (status == hipSuccess) break; - if (status != hipErrorNotReady) { - error = hip_error("hipEventQuery", status); - release_active(); - return false; - } - if (std::chrono::steady_clock::now() >= deadline) { - error = "executor_timeout"; - synchronize_and_release(); - wait_ms = std::chrono::duration( - std::chrono::steady_clock::now() - wait_started).count(); - return false; - } - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } - - float gpu_ms = 0.0F; - hipError_t status = hipEventElapsedTime(&gpu_ms, started_, finished_); - if (status != hipSuccess) { - error = hip_error("hipEventElapsedTime", status); - release_active(); - return false; - } - float sample = 0.0F; - status = hipMemcpyAsync( - &sample, c_, sizeof(sample), hipMemcpyDeviceToHost, stream_); - if (status == hipSuccess) status = hipStreamSynchronize(stream_); - if (status != hipSuccess || !std::isfinite(sample)) { - error = status == hipSuccess - ? "HIP probe produced a non-finite sample" - : hip_error("HIP probe result copy", status); - release_active(); - return false; - } - result = { - {"sample", sample}, - {"gpu_ms", gpu_ms}, - {"iterations", iterations_}, - {"matrix_size", matrix_size_}, - {"cu_count", current_cu_count_}, - }; - if (result.dump().size() > max_result_bytes) { - error = "executor_result_too_large"; - release_active(); - return false; - } - wait_ms = std::chrono::duration( - std::chrono::steady_clock::now() - wait_started).count(); - release_active(); - error.clear(); - return true; - } - - void synchronize_and_release() { - ScopedHipDevice device(device_); - std::lock_guard lock(mutex_); - if (device.ok() && active_ && stream_) { - (void) hipStreamSynchronize(stream_); - } - active_ = false; - } - -private: - bool ensure_resources(int cu_count, std::string & error) { - hipError_t status = hipSuccess; - if (!stream_ || !handle_ || current_cu_count_ != cu_count) { - if (stream_) { - (void) hipStreamSynchronize(stream_); - if (handle_) { - (void) hipblasDestroy(handle_); - handle_ = nullptr; - } - (void) hipStreamDestroy(stream_); - stream_ = nullptr; - } - const size_t mask_words = - static_cast((total_cus_ + 31) / 32); - std::vector mask(mask_words, 0); - for (int cu = 0; cu < cu_count; ++cu) { - mask[static_cast(cu / 32)] |= - uint32_t{1} << (cu % 32); - } - status = hipExtStreamCreateWithCUMask( - &stream_, static_cast(mask.size()), mask.data()); - if (status != hipSuccess) { - error = hip_error("hipExtStreamCreateWithCUMask", status); - return false; - } - const hipblasStatus_t create_status = hipblasCreate(&handle_); - if (create_status != HIPBLAS_STATUS_SUCCESS) { - error = hipblas_error("hipblasCreate", create_status); - return false; - } - const hipblasStatus_t stream_status = - hipblasSetStream(handle_, stream_); - if (stream_status != HIPBLAS_STATUS_SUCCESS) { - error = hipblas_error("hipblasSetStream", stream_status); - return false; - } - current_cu_count_ = cu_count; - } - if (!a_ || !b_ || !c_) { - if (c_) (void) hipFree(c_); - if (b_) (void) hipFree(b_); - if (a_) (void) hipFree(a_); - a_ = nullptr; - b_ = nullptr; - c_ = nullptr; - const size_t elements = - static_cast(matrix_size_) * matrix_size_; - const size_t bytes = elements * sizeof(float); - if ((status = hipMalloc(&a_, bytes)) != hipSuccess || - (status = hipMalloc(&b_, bytes)) != hipSuccess || - (status = hipMalloc(&c_, bytes)) != hipSuccess) { - error = hip_error("hipMalloc", status); - if (c_) (void) hipFree(c_); - if (b_) (void) hipFree(b_); - if (a_) (void) hipFree(a_); - a_ = nullptr; - b_ = nullptr; - c_ = nullptr; - return false; - } - if ((status = hipMemsetAsync(a_, 0x01, bytes, stream_)) != hipSuccess || - (status = hipMemsetAsync(b_, 0x02, bytes, stream_)) != hipSuccess || - (status = hipMemsetAsync(c_, 0, bytes, stream_)) != hipSuccess) { - error = hip_error("hipMemsetAsync", status); - return false; - } - const float alpha = 1.0F; - const float beta = 0.0F; - const hipblasStatus_t warm_status = hipblasSgemm( - handle_, HIPBLAS_OP_N, HIPBLAS_OP_N, - matrix_size_, matrix_size_, matrix_size_, - &alpha, a_, matrix_size_, b_, matrix_size_, - &beta, c_, matrix_size_); - if (warm_status != HIPBLAS_STATUS_SUCCESS) { - error = hipblas_error("hipblasSgemm(warmup)", warm_status); - return false; - } - if ((status = hipStreamSynchronize(stream_)) != hipSuccess) { - error = hip_error("hipStreamSynchronize(warmup)", status); - return false; - } - } - if (!started_ && - (status = hipEventCreate(&started_)) != hipSuccess) { - error = hip_error("hipEventCreate(started)", status); - return false; - } - if (!finished_ && - (status = hipEventCreate(&finished_)) != hipSuccess) { - error = hip_error("hipEventCreate(finished)", status); - return false; - } - return true; - } - - void release_active() { - std::lock_guard lock(mutex_); - active_ = false; - } - - std::mutex mutex_; - int device_ = 0; - int matrix_size_ = 0; - int total_cus_ = 0; - int current_cu_count_ = 0; - int iterations_ = 0; - bool active_ = false; - hipStream_t stream_ = nullptr; - hipblasHandle_t handle_ = nullptr; - hipEvent_t started_ = nullptr; - hipEvent_t finished_ = nullptr; - float * a_ = nullptr; - float * b_ = nullptr; - float * c_ = nullptr; -}; - -HipSgemmExecution::~HipSgemmExecution() { - if (!finished_) terminate(false); -} - -bool HipSgemmExecution::send_control(const std::string & operation) { - if (operation == "commit") { - committed_ = true; - return true; - } - if (operation == "cancel") { - committed_ = false; - return true; - } - return false; -} - -bool HipSgemmExecution::collect_result( - int timeout_ms, - size_t max_result_bytes, - json & result, - double & wait_ms, - std::string & error) { - if (finished_) { - error = "executor already collected"; - return false; - } - if (!committed_) { - error = "executor result requested before commit"; - terminate(false); - return false; - } - finished_ = true; - return state_->collect( - timeout_ms, max_result_bytes, result, wait_ms, error); -} - -void HipSgemmExecution::terminate(bool allow_control_grace) { - (void) allow_control_grace; - if (finished_) return; - finished_ = true; - state_->synchronize_and_release(); -} - -class HipSgemmExecutor final : public ToolSpeculationExecutor { -public: - explicit HipSgemmExecutor(std::shared_ptr state) - : state_(std::move(state)) {} - - std::unique_ptr start( - const json & request, std::string & error) override { - if (!state_->start(request, error)) return nullptr; - return std::make_unique(state_); - } - - const char * mode_name() const override { - return "in_process_hip_cu_mask"; - } - -private: - std::shared_ptr state_; -}; - -} // namespace - -std::shared_ptr -create_hip_sgemm_tool_speculation_executor( - int device, - int matrix_size, - int & total_compute_units, - std::string & error) { - total_compute_units = 0; - if (device < 0 || matrix_size <= 0 || matrix_size > 8192) { - error = "HIP probe needs DEVICE >= 0 and MATRIX_SIZE in 1..8192"; - return nullptr; - } - hipDeviceProp_t properties{}; - const hipError_t status = hipGetDeviceProperties(&properties, device); - if (status != hipSuccess) { - error = hip_error("hipGetDeviceProperties", status); - return nullptr; - } - if (properties.multiProcessorCount <= 0) { - error = "HIP device reports no compute units"; - return nullptr; - } - total_compute_units = properties.multiProcessorCount; - error.clear(); - auto state = std::make_shared( - device, matrix_size, properties.multiProcessorCount); - return std::make_shared(std::move(state)); -} - -} // namespace dflash::common diff --git a/server/src/server/tool_speculation_hip_probe.h b/server/src/server/tool_speculation_hip_probe.h deleted file mode 100644 index 673376521..000000000 --- a/server/src/server/tool_speculation_hip_probe.h +++ /dev/null @@ -1,21 +0,0 @@ -#pragma once - -#include "tool_speculation.h" - -#include -#include - -namespace dflash::common { - -// Benchmark-only trusted executor used to qualify same-process HIP sharing. -// It accepts the allowlisted `benchmark_hip_sgemm` tool with -// {"iterations": N}. The matrix size is fixed at startup so allocation and -// warmup stay outside measured requests. -std::shared_ptr -create_hip_sgemm_tool_speculation_executor( - int device, - int matrix_size, - int & total_compute_units, - std::string & error); - -} // namespace dflash::common diff --git a/server/test/test_semantic_tool_hint.cpp b/server/test/test_semantic_tool_hint.cpp index 6fa96b792..7860de3a0 100644 --- a/server/test/test_semantic_tool_hint.cpp +++ b/server/test/test_semantic_tool_hint.cpp @@ -1,8 +1,16 @@ #include "CppUnitTestFramework.hpp" +#include "common/qwen3_tool_predictor_ipc.h" #include "server/semantic_tool_hint.h" +#include +#include #include +#include + +#if !defined(_WIN32) +#include +#endif namespace { struct SemanticToolHintFixture {}; @@ -246,3 +254,44 @@ TEST_CASE(SemanticToolHintFixture, native_parser_rejects_multiple_calls) { call + call, weather_tools(), prediction, error)); CHECK(error == "native_predictor_response_has_multiple_calls"); } + +TEST_CASE(SemanticToolHintFixture, native_ipc_response_obeys_hard_deadline) { +#if !defined(_WIN32) + int descriptors[2] = {-1, -1}; + CHECK(::pipe(descriptors) == 0); + const auto started = std::chrono::steady_clock::now(); + std::vector output; + std::string error; + CHECK(!read_qwen3_tool_predictor_response( + descriptors[0], 8, 25, output, error)); + const double elapsed_ms = std::chrono::duration( + std::chrono::steady_clock::now() - started).count(); + ::close(descriptors[0]); + ::close(descriptors[1]); + CHECK(error == "native_predictor_timeout"); + CHECK(output.empty()); + CHECK(elapsed_ms < 500.0); +#else + CHECK(true); +#endif +} + +TEST_CASE(SemanticToolHintFixture, native_ipc_response_reads_complete_payload) { +#if !defined(_WIN32) + int descriptors[2] = {-1, -1}; + CHECK(::pipe(descriptors) == 0); + const int32_t response[] = {0, 3, 17, 18, 19}; + CHECK(::write(descriptors[1], response, sizeof(response)) == + static_cast(sizeof(response))); + ::close(descriptors[1]); + std::vector output; + std::string error; + CHECK(read_qwen3_tool_predictor_response( + descriptors[0], 8, 100, output, error)); + ::close(descriptors[0]); + CHECK(error.empty()); + CHECK(output == std::vector({17, 18, 19})); +#else + CHECK(true); +#endif +} diff --git a/server/test/test_server_unit.cpp b/server/test/test_server_unit.cpp index b80833c6e..1730512b3 100644 --- a/server/test/test_server_unit.cpp +++ b/server/test/test_server_unit.cpp @@ -4126,6 +4126,24 @@ TEST_CASE(ServerUnitFixture, test_backend_ipc_rejects_file_work_dir) { unlink(file_path.c_str()); } +TEST_CASE(ServerUnitFixture, test_backend_ipc_rejects_public_work_dir) { + const std::string dir_path = + "/tmp/dflash_test_backend_ipc_public_work_dir"; + rmdir(dir_path.c_str()); + TEST_ASSERT(mkdir(dir_path.c_str(), 0755) == 0); + TEST_ASSERT(chmod(dir_path.c_str(), 0755) == 0); + + BackendIpcLaunchConfig cfg; + cfg.bin = "/bin/true"; + cfg.payload_path = "/tmp/dflash_test_backend_ipc_payload"; + cfg.work_dir = dir_path; + + BackendIpcProcess proc; + TEST_ASSERT(!proc.start(cfg)); + TEST_ASSERT(!proc.active()); + rmdir(dir_path.c_str()); +} + TEST_CASE(ServerUnitFixture, test_backend_ipc_payload_pipe_round_trip) { int payload_pipe[2] = {-1, -1}; int status_pipe[2] = {-1, -1}; @@ -4900,14 +4918,10 @@ TEST_CASE(ServerUnitFixture, test_props_tool_speculation_shape) { TEST_ASSERT(disabled["unqualified_lane_policy"].get() == "defer"); TEST_ASSERT(disabled["allowed_tools"].empty()); - TEST_ASSERT(disabled["model_routing_static"].get()); - TEST_ASSERT(disabled["model_expert_ownership_unique"].get()); TEST_ASSERT(disabled["compute_isolation"].get() == "none"); TEST_ASSERT(!disabled["cpu_affinity_isolated"].get()); TEST_ASSERT(disabled["tool_cpu_affinity"].empty()); TEST_ASSERT(disabled["model_cpu_affinity"].empty()); - TEST_ASSERT(disabled["hip_tool_device"].is_null()); - TEST_ASSERT(disabled["hip_reserved_tool_compute_units"].get() == 0); TEST_ASSERT(disabled["profile_lanes"].empty()); cfg.tool_speculation.executor_path = "/trusted/tool-adapter"; @@ -4915,8 +4929,11 @@ TEST_CASE(ServerUnitFixture, test_props_tool_speculation_shape) { cfg.tool_speculation.allowed_tools = {"lookup"}; std::string profile_error; TEST_ASSERT(cfg.tool_speculation.policy.load_json(json{ + {"profile_status", "qualified"}, + {"executor", "child_process"}, {"path_summary", { {"25", { + {"accelerator_relation", "non_accelerator"}, {"decode_interference_qualified", true}, {"hit", { {"control_task_mean_ms", 100.0}, @@ -4938,6 +4955,8 @@ TEST_CASE(ServerUnitFixture, test_props_tool_speculation_shape) { TEST_ASSERT(!enabled["automatic_prediction_enabled"].get()); TEST_ASSERT(!enabled["predictor_decode_isolated"].get()); TEST_ASSERT(enabled["profile_status"].get() == "qualified"); + TEST_ASSERT(enabled["executor_contract"].get() == + "child_process"); TEST_ASSERT(enabled["allowed_tools"] == json::array({"lookup"})); TEST_ASSERT(enabled["preserves_token_speculation"].get()); TEST_ASSERT(enabled["unqualified_lane_policy"].get() == @@ -4952,11 +4971,7 @@ TEST_CASE(ServerUnitFixture, test_props_tool_speculation_shape) { ["decode_interference_qualified"].get()); TEST_ASSERT(enabled["profile_lanes"][0] ["accelerator_relation"].get() == - "unspecified"); - TEST_ASSERT(!enabled["profile_lanes"][0] - ["requires_static_model_routing"].get()); - TEST_ASSERT(!enabled["profile_lanes"][0] - ["requires_unique_expert_ownership"].get()); + "non_accelerator"); cfg.semantic_tool_predictor.native_model_path = "/models/qwen3-0.6b.gguf"; cfg.semantic_tool_predictor.native_ipc_bin = "/bin/backend-ipc"; @@ -5009,16 +5024,6 @@ TEST_CASE(ServerUnitFixture, test_props_tool_speculation_shape) { json::array({14, 30})); TEST_ASSERT(cpu_isolated["model_cpu_affinity"] == json::array({0, 1, 2, 3})); - - cfg.tool_speculation.cpu_affinity_isolated = false; - cfg.tool_speculation.hip_tool_device = 1; - cfg.tool_speculation.hip_reserved_tool_compute_units = 1; - body = build_props_body(cfg, pc, tm); - const json & isolated = body["tool_speculation"]; - TEST_ASSERT(isolated["compute_isolation"].get() == - "disjoint_hip_cu_masks"); - TEST_ASSERT(isolated["hip_tool_device"].get() == 1); - TEST_ASSERT(isolated["hip_reserved_tool_compute_units"].get() == 1); } // ─── /props.runtime captures full config (§4.16) ────────────────────── @@ -5259,40 +5264,6 @@ TEST_CASE(ServerUnitFixture, test_model_backend_retries_empty_spec_restore_once_ TEST_ASSERT(backend.restore_saw_force_ar); } -TEST_CASE(ServerUnitFixture, test_model_backend_can_forbid_ar_retry) { - EmptySpecRetryBackend backend; - GenerateRequest req; - req.prompt = {1, 2, 3}; - req.n_gen = 4; - req.allow_decode_mode_retry = false; - DaemonIO io; - - GenerateResult result = backend.generate(req, io); - - TEST_ASSERT(result.ok()); - TEST_ASSERT(result.tokens.empty()); - TEST_ASSERT(result.spec_decode_ran); - TEST_ASSERT(backend.generate_calls == 1); - TEST_ASSERT(!backend.generate_saw_force_ar); -} - -TEST_CASE(ServerUnitFixture, test_model_backend_restore_can_forbid_ar_retry) { - EmptySpecRetryBackend backend; - GenerateRequest req; - req.prompt = {1, 2, 3}; - req.n_gen = 4; - req.allow_decode_mode_retry = false; - DaemonIO io; - - GenerateResult result = backend.restore_and_generate(7, req, io); - - TEST_ASSERT(result.ok()); - TEST_ASSERT(result.tokens.empty()); - TEST_ASSERT(result.spec_decode_ran); - TEST_ASSERT(backend.restore_calls == 1); - TEST_ASSERT(!backend.restore_saw_force_ar); -} - TEST_CASE(ServerUnitFixture, test_model_backend_retries_empty_visible_spec_generate_once_with_ar) { EmptySpecRetryBackend backend; backend.generate_first_empty_visible = true; diff --git a/server/test/test_tool_speculation.cpp b/server/test/test_tool_speculation.cpp index e9686e611..b2acb3148 100644 --- a/server/test/test_tool_speculation.cpp +++ b/server/test/test_tool_speculation.cpp @@ -4,18 +4,24 @@ #include #include +#include #include #include +#include #include #include #include +#include #include #include #if !defined(_WIN32) +# include +# include # include # include # if defined(__linux__) +# include # include # endif #endif @@ -25,8 +31,6 @@ using dflash::common::CanonicalToolInvocation; using dflash::common::ToolCall; using dflash::common::ToolSpeculationAttempt; using dflash::common::ToolSpeculationConfig; -using dflash::common::ToolSpeculationExecution; -using dflash::common::ToolSpeculationExecutor; using dflash::common::ToolSpeculationPolicy; using dflash::common::ToolSpeculationPrediction; using dflash::common::build_tool_speculation_prediction; @@ -39,69 +43,6 @@ using dflash::common::render_tool_speculation_sse; namespace { struct ToolSpeculationFixture {}; -struct FakeExecutionState { - json request; - std::vector controls; - bool terminated = false; - bool collected = false; -}; - -class FakeExecution final : public ToolSpeculationExecution { -public: - explicit FakeExecution(std::shared_ptr state) - : state_(std::move(state)) {} - - bool send_control(const std::string & operation) override { - state_->controls.push_back(operation); - return operation == "commit" || operation == "cancel"; - } - - bool collect_result(int timeout_ms, - size_t max_result_bytes, - json & result, - double & wait_ms, - std::string & error) override { - (void) timeout_ms; - state_->collected = true; - result = {{"value", 42}}; - wait_ms = 0.0; - if (result.dump().size() > max_result_bytes) { - error = "executor_result_too_large"; - return false; - } - error.clear(); - return true; - } - - void terminate(bool allow_control_grace) override { - (void) allow_control_grace; - state_->terminated = true; - } - -private: - std::shared_ptr state_; -}; - -class FakeExecutor final : public ToolSpeculationExecutor { -public: - explicit FakeExecutor(std::shared_ptr state) - : state_(std::move(state)) {} - - std::unique_ptr start( - const json & request, std::string & error) override { - state_->request = request; - error.clear(); - return std::make_unique(state_); - } - - const char * mode_name() const override { - return "fake_in_process"; - } - -private: - std::shared_ptr state_; -}; - json policy_fixture(bool decode_interference_qualified = true) { auto path = [decode_interference_qualified]( double hit_task, double miss_task, @@ -109,6 +50,7 @@ json policy_fixture(bool decode_interference_qualified = true) { return json{ {"decode_interference_qualified", decode_interference_qualified}, + {"accelerator_relation", "non_accelerator"}, {"hit", { {"control_task_mean_ms", 100.0}, {"speculative_task_mean_ms", hit_task}, @@ -122,6 +64,8 @@ json policy_fixture(bool decode_interference_qualified = true) { }; }; json fixture = { + {"profile_status", "qualified"}, + {"executor", "child_process"}, {"path_summary", { {"25", path(80.0, 101.0, 2.0)}, {"50", path(60.0, 110.0, 7.0)}, @@ -309,81 +253,44 @@ TEST_CASE(ToolSpeculationFixture, profile_metadata_is_fail_closed) { ToolSpeculationPolicy policy; std::string error; json fixture = policy_fixture(); - fixture["profile_status"] = "provisional_benchmark_only"; - fixture["executor"] = "in_process_hip_cu_mask"; CHECK(policy.load_json(fixture, error)); - CHECK(policy.benchmark_only()); - CHECK(policy.executor_contract() == "in_process_hip_cu_mask"); + CHECK(policy.profile_status() == "qualified"); + CHECK(policy.executor_contract() == "child_process"); + + fixture["profile_status"] = "provisional_benchmark_only"; + CHECK(!policy.load_json(fixture, error)); + CHECK(policy.empty()); fixture["profile_status"] = "unknown"; CHECK(!policy.load_json(fixture, error)); CHECK(policy.empty()); + + fixture = policy_fixture(); + fixture.erase("executor"); + CHECK(!policy.load_json(fixture, error)); + CHECK(policy.empty()); + + fixture = policy_fixture(); + fixture.erase("profile_status"); + CHECK(!policy.load_json(fixture, error)); + CHECK(policy.empty()); + + fixture = policy_fixture(); + fixture["path_summary"]["25"].erase("accelerator_relation"); + CHECK(!policy.load_json(fixture, error)); + CHECK(policy.empty()); } -TEST_CASE(ToolSpeculationFixture, same_gpu_profile_declares_routing_requirements) { +TEST_CASE(ToolSpeculationFixture, same_gpu_profile_is_rejected) { ToolSpeculationPolicy policy; std::string error; json fixture = policy_fixture(); for (auto & lane : fixture["path_summary"]) { lane["accelerator_relation"] = "same_physical_gpu"; } - CHECK(policy.load_json(fixture, error)); - CHECK(!policy.requires_static_model_routing()); - CHECK(!policy.requires_unique_expert_ownership()); - - for (auto & lane : fixture["path_summary"]) { - lane["requires_static_model_routing"] = true; - } - CHECK(policy.load_json(fixture, error)); - CHECK(policy.requires_static_model_routing()); - CHECK(!policy.requires_unique_expert_ownership()); - - for (auto & lane : fixture["path_summary"]) { - lane["requires_unique_expert_ownership"] = true; - } - CHECK(policy.load_json(fixture, error)); - CHECK(policy.requires_static_model_routing()); - CHECK(policy.requires_unique_expert_ownership()); - const auto decision = policy.choose(1.0, 2.0); - CHECK(decision.admitted); - CHECK(decision.accelerator_relation == "same_physical_gpu"); -} - -TEST_CASE(ToolSpeculationFixture, same_gpu_profile_obeys_measured_break_even) { - ToolSpeculationPolicy policy; - std::string error; - CHECK(policy.load_json(json{ - {"path_summary", { - {"100", { - {"accelerator_relation", "same_physical_gpu"}, - {"requires_static_model_routing", true}, - {"requires_unique_expert_ownership", true}, - {"decode_interference_qualified", true}, - {"hit", { - {"control_task_mean_ms", 2670.178}, - {"speculative_task_mean_ms", 2389.530}, - {"model_slowdown_percent", 169.109}, - }}, - {"miss", { - {"control_task_mean_ms", 2670.178}, - {"speculative_task_mean_ms", 4171.884}, - {"model_slowdown_percent", 169.109}, - }}, - }}, - }}, - }, error)); - - const auto below = policy.choose(0.84, 3.0); - CHECK(!below.admitted); - CHECK(below.reason == "below_profile_break_even"); - - const auto above = policy.choose(0.85, 3.0); - CHECK(above.admitted); - CHECK(above.resource_percentage == 100); - - const auto guarded = policy.choose(1.0, 1.20); - CHECK(!guarded.admitted); - CHECK(guarded.reason == "model_slowdown_guardrail"); + CHECK(!policy.load_json(fixture, error)); + CHECK(error.find("accelerator_relation") != std::string::npos); + CHECK(policy.empty()); } TEST_CASE(ToolSpeculationFixture, non_allowlisted_tool_is_deferred) { @@ -400,53 +307,6 @@ TEST_CASE(ToolSpeculationFixture, non_allowlisted_tool_is_deferred) { CHECK(!metadata.contains("result")); } -TEST_CASE(ToolSpeculationFixture, in_process_exact_match_commits_private_result) { - auto state = std::make_shared(); - ToolSpeculationConfig config = test_config(); - config.executor_path.clear(); - config.in_process_executor = std::make_shared(state); - CHECK(config.enabled()); - CHECK(std::string(config.execution_mode()) == "fake_in_process"); - - auto attempt = ToolSpeculationAttempt::create( - config, prediction(), "request_in_process_hit"); - attempt->start(); - CHECK(attempt->running()); - CHECK(state->request["call"]["name"] == "lookup"); - CHECK(state->request["resource_percentage"] == 100); - - const json metadata = attempt->resolve({ - ToolCall{"call_1", "lookup", R"({"b":2,"a":1})"}, - }); - CHECK(metadata["status"] == "hit"); - CHECK(metadata["result"]["value"] == 42); - CHECK(state->collected); - CHECK(state->controls.size() == 1); - CHECK(state->controls[0] == "commit"); - CHECK(!state->terminated); -} - -TEST_CASE(ToolSpeculationFixture, in_process_mismatch_cancels_private_result) { - auto state = std::make_shared(); - ToolSpeculationConfig config = test_config(); - config.executor_path.clear(); - config.in_process_executor = std::make_shared(state); - - auto attempt = ToolSpeculationAttempt::create( - config, prediction(), "request_in_process_miss"); - attempt->start(); - const json metadata = attempt->resolve({ - ToolCall{"call_1", "lookup", R"({"a":999})"}, - }); - CHECK(metadata["status"] == "miss"); - CHECK(metadata["reason"] == "invocation_mismatch"); - CHECK(!metadata.contains("result")); - CHECK(!state->collected); - CHECK(state->terminated); - CHECK(state->controls.size() == 1); - CHECK(state->controls[0] == "cancel"); -} - #if !defined(_WIN32) TEST_CASE(ToolSpeculationFixture, cpu_affinity_reaches_child_executor) { #if defined(__linux__) @@ -535,7 +395,7 @@ TEST_CASE(ToolSpeculationFixture, exact_match_exposes_result_and_resource_share) CHECK(metadata["status"] == "hit"); CHECK(metadata["resource_percentage"] == 100); CHECK(metadata["result"]["resource"] == "100"); - CHECK(metadata["result"]["relation"] == "unspecified"); + CHECK(metadata["result"]["relation"] == "non_accelerator"); CHECK(metadata["result"]["value"] == 42); CHECK(control.find("\"op\":\"commit\"") != std::string::npos); CHECK(control.find("\"authoritative_resource_percentage\":100") != @@ -575,6 +435,133 @@ TEST_CASE(ToolSpeculationFixture, executor_failure_is_private) { CHECK(!metadata.contains("result")); } +TEST_CASE(ToolSpeculationFixture, executor_timeout_starts_at_launch) { + const std::string path = make_executor_script( + "sleep 1\n" + "printf '{\"ok\":true,\"result\":{\"value\":42}}\\n'\n"); + ToolSpeculationConfig config = test_config(path); + config.timeout_ms = 25; + auto attempt = ToolSpeculationAttempt::create( + config, prediction(), "request_launch_deadline"); + attempt->start(); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + const auto resolve_started = std::chrono::steady_clock::now(); + const json metadata = attempt->resolve({ + ToolCall{"call_1", "lookup", R"({"a":1,"b":2})"}, + }); + const double resolve_ms = std::chrono::duration( + std::chrono::steady_clock::now() - resolve_started).count(); + ::unlink(path.c_str()); + + CHECK(metadata["status"] == "failed"); + CHECK(metadata["reason"] == "speculative_executor_failure"); + CHECK(metadata["detail"] == "executor_timeout"); + CHECK(!metadata.contains("result")); + CHECK(resolve_ms < 500.0); +} + +TEST_CASE(ToolSpeculationFixture, executor_timeout_terminates_process_group) { +#if defined(__linux__) + const std::string marker_path = make_temp_path(); + const std::string path = make_executor_script( + "sleep 10 &\n" + "printf '%s' \"$!\" > \"" + marker_path + "\"\n" + "IFS= read -r control\n" + "wait\n"); + ToolSpeculationConfig config = test_config(path); + config.timeout_ms = 100; + auto attempt = ToolSpeculationAttempt::create( + config, prediction(), "request_process_group_deadline"); + attempt->start(); + const json metadata = attempt->resolve({ + ToolCall{"call_1", "lookup", R"({"a":1,"b":2})"}, + }); + const std::string child_text = read_text_file(marker_path); + ::unlink(path.c_str()); + ::unlink(marker_path.c_str()); + + CHECK(metadata["status"] == "failed"); + CHECK(metadata["detail"] == "executor_timeout"); + CHECK(!child_text.empty()); + if (child_text.empty()) return; + const pid_t child = static_cast(std::stol(child_text)); + bool gone = false; + for (int retry = 0; retry < 50; ++retry) { + if (::kill(child, 0) != 0 && errno == ESRCH) { + gone = true; + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + CHECK(gone); +#else + CHECK(true); +#endif +} + +TEST_CASE(ToolSpeculationFixture, executor_does_not_inherit_server_fds) { +#if defined(__GLIBC__) && defined(__GLIBC_PREREQ) +# if __GLIBC_PREREQ(2, 34) + const std::string marker_path = make_temp_path(); + const int marker_fd = ::open(marker_path.c_str(), O_RDONLY); + CHECK(marker_fd >= 0); + const std::string path = make_executor_script( + "IFS= read -r control\n" + "leaked=false\n" + "for descriptor in /proc/self/fd/*; do\n" + " target=$(readlink \"$descriptor\" 2>/dev/null || true)\n" + " if [ \"$target\" = \"" + marker_path + "\" ]; then leaked=true; fi\n" + "done\n" + "printf '{\"ok\":true,\"result\":{\"leaked\":%s}}\\n' \"$leaked\"\n"); + ToolSpeculationConfig config = test_config(path); + auto attempt = ToolSpeculationAttempt::create( + config, prediction(), "request_fd_isolation"); + attempt->start(); + const json metadata = attempt->resolve({ + ToolCall{"call_1", "lookup", R"({"a":1,"b":2})"}, + }); + ::close(marker_fd); + ::unlink(path.c_str()); + ::unlink(marker_path.c_str()); + + CHECK(metadata["status"] == "hit"); + CHECK(!metadata["result"]["leaked"].get()); +# else + CHECK(true); +# endif +#else + CHECK(true); +#endif +} + +TEST_CASE(ToolSpeculationFixture, executor_environment_overrides_stale_enable_flag) { + const char * previous = std::getenv("DFLASH_TOOL_SPECULATION"); + const bool had_previous = previous != nullptr; + const std::string previous_value = previous ? previous : ""; + CHECK(::setenv("DFLASH_TOOL_SPECULATION", "0", 1) == 0); + const std::string path = make_executor_script( + "IFS= read -r control\n" + "printf '{\"ok\":true,\"result\":{\"enabled\":\"%s\"}}\\n' " + "\"$DFLASH_TOOL_SPECULATION\"\n"); + ToolSpeculationConfig config = test_config(path); + auto attempt = ToolSpeculationAttempt::create( + config, prediction(), "request_clean_environment"); + attempt->start(); + const json metadata = attempt->resolve({ + ToolCall{"call_1", "lookup", R"({"a":1,"b":2})"}, + }); + ::unlink(path.c_str()); + if (had_previous) { + CHECK(::setenv( + "DFLASH_TOOL_SPECULATION", previous_value.c_str(), 1) == 0); + } else { + CHECK(::unsetenv("DFLASH_TOOL_SPECULATION") == 0); + } + + CHECK(metadata["status"] == "hit"); + CHECK(metadata["result"]["enabled"] == "1"); +} + TEST_CASE(ToolSpeculationFixture, qualified_lane_keeps_speculative_decode) { const std::string path = make_executor_script( "IFS= read -r control\n" From bf463ecb8f2e6aeedc0d8c4665bc9da157a457f2 Mon Sep 17 00:00:00 2001 From: mrciffa <49000955+davide221@users.noreply.github.com> Date: Tue, 18 Aug 2026 18:11:00 +0200 Subject: [PATCH 07/11] fix(server): production-harden tool speculation --- optimizations/ooo_spec_lucebox5_cpu/README.md | 183 +- .../benchmark_cpu_tool_speculation.py | 57 +- .../benchmark_trace_compiled_workflows.py | 472 +- .../bfcl_replay_tool_executor.py | 2 +- .../cpu_sparse_tool_executor.cpp | 54 +- ...sh_server_native_tool_predictor_wrapper.sh | 18 +- .../profiles/lucebox5-cpu-lane-qualified.json | 1 + ...turn-cached-wordref-production-6tasks.json | 5595 +++++++++++++++++ ...engine-qwen-production-6pairs-compact.json | 930 ++- .../trace-compiled-training-traces.json | 46 - .../results/trace-workflow-registry.json | 121 + .../run_native_cpu_server_lucebox5.sh | 41 +- .../test_benchmark_cpu_tool_speculation.py | 32 + ...test_benchmark_trace_compiled_workflows.py | 74 +- .../test_trace_compiled_tool_executor.py | 26 + .../trace_compiled_tool_executor.py | 11 +- server/src/common/backend_ipc.cpp | 127 +- server/src/common/backend_ipc.h | 11 +- .../src/common/qwen3_tool_predictor_ipc.cpp | 10 +- server/src/common/qwen3_tool_predictor_ipc.h | 3 +- server/src/qwen3/qwen3_backend.cpp | 4 +- server/src/qwen3/qwen3_loader.cpp | 91 +- server/src/server/chat_template.cpp | 13 +- server/src/server/chat_template.h | 9 +- server/src/server/http_server.cpp | 328 +- server/src/server/http_server.h | 19 +- .../server/native_semantic_tool_predictor.cpp | 18 +- server/src/server/semantic_tool_hint.cpp | 129 +- server/src/server/semantic_tool_hint.h | 22 +- server/src/server/server_main.cpp | 87 +- server/src/server/tokenizer.cpp | 118 +- server/src/server/tokenizer.h | 21 +- server/src/server/tool_speculation.cpp | 177 +- server/src/server/tool_speculation.h | 1 + .../test/smoke_qwen3_tool_predictor_ipc.cpp | 7 +- server/test/test_semantic_tool_hint.cpp | 93 +- server/test/test_server_unit.cpp | 99 +- server/test/test_tool_speculation.cpp | 21 +- 38 files changed, 8094 insertions(+), 977 deletions(-) create mode 100644 optimizations/ooo_spec_lucebox5_cpu/results/multiturn-cached-wordref-production-6tasks.json delete mode 100644 optimizations/ooo_spec_lucebox5_cpu/results/trace-compiled-training-traces.json create mode 100644 optimizations/ooo_spec_lucebox5_cpu/results/trace-workflow-registry.json diff --git a/optimizations/ooo_spec_lucebox5_cpu/README.md b/optimizations/ooo_spec_lucebox5_cpu/README.md index 8a2368578..75857b1f4 100644 --- a/optimizations/ooo_spec_lucebox5_cpu/README.md +++ b/optimizations/ooo_spec_lucebox5_cpu/README.md @@ -1,95 +1,127 @@ # CPU-isolated tool speculation on Lucebox5 -The engine asks a small predictor for one concrete tool call before the target -model runs. On Lucebox5, Qwen3-0.6B Q8_0 predicts on Strix, then exits its GPU -compute window; the predicted read-only tool runs on reserved CPU cores while -DeepSeek-V4-0731 decodes with DS4/DSpark on R9700 + Strix. The result stays -private unless DeepSeek emits the exact same canonical function and arguments. +The engine can ask a small model for one concrete tool call before the target +model runs. If the call is allowlisted and its measured execution lane is +qualified, the engine starts the read-only tool privately. The target model +remains authoritative: the result is released only when the emitted function +name and canonical arguments match exactly. -This path does not inject tokens or replace DSpark. Tool prediction does not -alter the target decoder's selection or normal recovery policy. A wrong -prediction is discarded and the caller executes the target model's -authoritative call normally. +On Lucebox5, Qwen3-0.6B Q8_0 predicts on Strix and then leaves its compute +window. The CPU tool runs on logical CPUs `14-15,30-31` while +DeepSeek-V4-0731 + DS4/DSpark runs on R9700 + Strix and CPUs +`0-13,16-29`. This path neither injects tokens nor replaces DSpark. -## Production result +## Measured result -The 2026-08-17 paired run compiled a recurring, side-effect-free five-step -trace into one typed workflow tool. Independent branches ran concurrently on -the isolated CPU lane. The six measured tasks covered 10, 15, and 20 leaf calls -twice each, with randomized arm order and one warmup task. +The 2026-08-18 production run used six paired tasks: two each with 10, 15, +and 20 leaf calls. Each branch contained five serial, deterministic read-only +calls; independent branches ran concurrently. Arm order was randomized. +Every arm included model turns, tool time, and a final answer produced from +the actual assistant/tool conversation. | Metric | Result | | --- | ---: | -| Normal stage-batched workflow, p50 | 81.030 s | -| Trace-compiled + speculative workflow, p50 | **14.597 s** | -| End-to-end speedup, paired p50 | **5.5961x** | -| End-to-end bootstrap 95% CI | **5.4577x–5.6599x** | -| Trace compilation alone | **3.2806x** | -| Early launch on top of compilation | **1.6954x** | -| Early-launch bootstrap 95% CI | **1.6760x–1.7210x** | -| Exposed tool wait, compiled / speculative p50 | 10.143 s / **0.027 ms** | -| Qwen prediction latency, p50 | 203.5 ms | -| Target model-compute slowdown, p50 / p95 | -0.458% / -0.332% | -| Target decode slowdown, p50 / p95 | -0.101% / 0.219% | -| Exact predictor-to-target hits | 6 / 6 | - -All 20 production gates passed: identical leaf calls, tool-result hashes, -macro calls, and final outputs; positive DS4 acceptance on every call turn; -correct CPU isolation; and no measurable target slowdown. The 5.60x result -combines two independent gains: four fewer model/tool synchronization barriers -from trace compilation, plus the 1.70x gained by starting the compiled graph -before target authorization completes. - -Artifact: +| Stage-batched workflow, p50 | 87.998 s | +| Trace-compiled workflow, p50 | 32.877 s | +| Trace-compiled + speculative workflow, p50 | **20.474 s** | +| End-to-end paired speedup, p50 | **4.3597x** | +| End-to-end bootstrap 95% CI | **3.7696x–4.8252x** | +| End-to-end paired speedup, p05 | **3.6680x** | +| Trace compilation alone, paired p50 | **2.6876x** | +| Early launch on top of compilation, paired p50 | **1.7676x** | +| Early-launch bootstrap 95% CI | **1.2590x–1.8002x** | +| Exposed tool wait, compiled / speculative p50 | 10.149 s / **0.030 ms** | +| Qwen prediction latency, p50 | 201.0 ms | +| Target model-compute change, p50 / p95 | -0.027% / +0.090% | +| Target decode change, p50 / p95 | -0.658% / +0.374% | +| Exact Qwen predictor hits | 6 / 6 | + +The slowdown figures come from a separate controlled A/B probe. Each task had +three alternating repetitions per arm, and every observation was preceded by +the same warm request. The gate compares the per-task median ratios and +requires matching cache state, completion tokens, call digest, and active +DS4 decoding. + +All 22 gates passed, including stable leaf-call and result digests, exact +final answers, exact macro calls, CPU isolation, controlled p50/p95 model +slowdown, and a wrong-call probe in which no private result crossed the +exact-match gate. + +These numbers apply to recognized, side-effect-free recurring workflows. The +4.36x combines fewer model/tool synchronization barriers from trace +compilation with the 1.77x gain from early tool launch. It is not a claim that +arbitrary single tool calls become 4.36x faster. The broader Qwen smoke suite +currently protects a 9/12 exact-argument baseline; a predictor miss falls back +to the authoritative call without changing the target output. + +Evidence: - `results/trace-compiled-engine-qwen-production-6pairs-compact.json` - (`sha256:0807cca1d22453728b069a0150800fcfa9a513db6a9f25815663fc03b99285b9`) - -The compact training fixture below reproduces the artifact's compiled pattern -fingerprint (`06d95882…0645`); the artifact retains the original full-report -hash for provenance. - -## Safety and portability - -- Only explicitly allowlisted, read-only/idempotent tools are eligible. -- The external result is committed only on an exact canonical call match. -- The executor is launched directly without a shell and has a hard deadline - measured from launch; inherited server file descriptors are closed. -- Lucebox5 reserves CPUs `14-15,30-31`; the model uses `0-13,16-29`. -- Startup fails closed if CPU masks overlap, the measured lane profile is not - qualified, or its executor contract differs from the configured lane. -- `before-model` is the native predictor default, so shared-GPU prediction - cannot reduce target prefill/decode throughput. -- The same API works on a single GPU: run the predictor before the target and - overlap only the CPU tool. A local or numeric-IPv4 HTTP predictor can use the - same verification and executor path on other model families. - -The speedup applies to tool-using request latency, not token throughput. Its -real-world value depends on exact predictor hit rate and on how much tool work -can overlap target generation. - -## Reproduce - -Build the deterministic sparse tool used by the single-call qualification: + (`sha256:b1194bd1447f772dc9c90e6e801e99646bdce121e1b300398c45b54540b87d20`) +- `results/multiturn-cached-wordref-production-6tasks.json` + (`sha256:2475697d418bffed0e9668da26ce6c88a85a952ce97d99749f440f97f9ac5bf9`) +- `results/trace-workflow-registry.json` + (the benchmark artifact records its exact hash) + +The report and registry paths inside the result are repository-relative, and +their recorded hashes match the committed files. + +The full source report is retained deliberately. Its original per-call +speculation performance gate did not pass; the compiler reads only the two +executions whose calls, results, dataflow, and side-effect flags are valid. +That failed baseline is the reason the workflow was compiled. No model was +trained on this report, and the passing 4.36x result is recorded separately. + +## Runtime contract + +- Tools must be explicitly allowlisted and read-only or idempotent. +- A result is committed only on one exact canonical call match. Multiple, + malformed, failed, timed-out, cancelled, or different calls are discarded. +- The child receives a minimal environment and only standard streams. Tool + speculation fails closed unless Linux glibc 2.34+ descriptor isolation is + available. +- CPU affinity is applied before the executor starts and is re-read before + the request payload is released. The model and tool masks must be disjoint. +- Executors run in their own process group with a launch-based deadline and a + bounded output size. Cancellation removes descendants. +- Native predictor IPC has a bounded startup, a private `0700` work directory, + an inherited-descriptor allowlist, serialized requests, and deadline-aware + prompt construction, tokenization, and generation. +- A production profile must declare a qualified non-accelerator or separate + physical-GPU lane and match the configured executor contract. + +Automatic prediction means clients do not need to send a prediction hint. +Clients do need to consume `dflash_tool_speculation.result` on a `hit` and use +it as the tool result; ignoring the extension remains correct but forfeits the +latency gain. + +The schedule also supports a single GPU because predictor compute finishes +before target compute begins; only the CPU tool overlaps target generation. +Both models still have to fit in memory. The production numbers above were +measured on Lucebox5's dual-GPU placement, not on a single-GPU machine. + +## Reproduce on Lucebox5 + +Build the deterministic sparse adapter used to qualify the CPU lane: ```bash JSON_INCLUDE=/path/to/server/deps/json/include \ ./build_cpu_sparse_executor.sh ./cpu_sparse_tool_executor ``` -Launch the qualified single-call configuration on an otherwise idle Lucebox5: +Launch the qualified server: ```bash ./run_native_cpu_server_lucebox5.sh ``` -The launcher defaults to Qwen3-0.6B Q8_0 on predictor GPU 1. Override placement -with `PREDICTOR_MODEL`, `PREDICTOR_GPU`, `PREDICTOR_MAX_CTX`, -`PREDICTOR_MAX_TOKENS`, and `PREDICTOR_TIMEOUT_MS`. The adjacent -`candidate-build` symlink in the wrapper selects a build even though the -qualified launcher clears ambient variables. +The launcher uses Qwen3-0.6B Q8_0 on predictor GPU 1 and the build selected by +the wrapper's adjacent `candidate-build` symlink. It clears ambient variables. +`PREDICTOR_MODEL`, `PREDICTOR_GPU`, `PREDICTOR_MAX_CTX`, +`PREDICTOR_MAX_TOKENS`, and `PREDICTOR_TIMEOUT_MS` apply only when invoking +the wrapper directly. -Run the single-call paired gate: +Run the single-call gate: ```bash python3 benchmark_cpu_tool_speculation.py native-qwen \ @@ -109,8 +141,7 @@ python3 benchmark_cpu_tool_speculation.py native-qwen \ --output results/qwen-auto-production-20pairs.json ``` -For the 10–20-call workflow gate, launch with the trace executor and macro -allowlist: +For the workflow gate, launch with the trace executor and macro allowlist: ```bash TOOL_SPEC_EXECUTOR=./trace_compiled_tool_executor.py \ @@ -123,15 +154,17 @@ Then run: ```bash python3 benchmark_trace_compiled_workflows.py \ --binary ./bfcl_replay_tool_executor.py \ - --training-report results/trace-compiled-training-traces.json \ + --training-report results/multiturn-cached-wordref-production-6tasks.json \ + --workflow-registry results/trace-workflow-registry.json \ --pairs 6 \ --warmup-tasks 1 \ --min-branches 2 \ --max-branches 4 \ + --interference-repetitions 3 \ --seed 814 \ --bootstrap-resamples 20000 \ --output results/trace-compiled-engine-qwen-production-6pairs-compact.json ``` -The harness exits nonzero on any correctness, isolation, DS4-activity, -slowdown, hit-rate, or speed threshold failure. +The harness exits nonzero on any correctness, privacy, isolation, DS4, +slowdown, hit-rate, or speed failure. diff --git a/optimizations/ooo_spec_lucebox5_cpu/benchmark_cpu_tool_speculation.py b/optimizations/ooo_spec_lucebox5_cpu/benchmark_cpu_tool_speculation.py index b2238c97d..7b52bb0e7 100755 --- a/optimizations/ooo_spec_lucebox5_cpu/benchmark_cpu_tool_speculation.py +++ b/optimizations/ooo_spec_lucebox5_cpu/benchmark_cpu_tool_speculation.py @@ -21,6 +21,7 @@ import os import random import re +import signal import statistics import subprocess import time @@ -314,6 +315,7 @@ def pin_child() -> None: text=True, env=environment, preexec_fn=pin_child, + start_new_session=True, ) assert process.stdin is not None process.stdin.write( @@ -330,11 +332,15 @@ def pin_child() -> None: def finish_executor(handle: dict[str, Any], timeout: float) -> dict[str, Any]: process: subprocess.Popen[str] = handle["process"] + elapsed = time.perf_counter() - float(handle["started"]) + remaining = timeout - elapsed + if remaining <= 0 and process.poll() is None: + stop_executor(handle) + raise RuntimeError("CPU executor timed out") try: - stdout, stderr = process.communicate(timeout=timeout) + stdout, stderr = process.communicate(timeout=max(0.001, remaining)) except subprocess.TimeoutExpired: - process.kill() - stdout, stderr = process.communicate(timeout=5) + stop_executor(handle) raise RuntimeError("CPU executor timed out") wall_ms = (time.perf_counter() - float(handle["started"])) * 1000.0 if process.returncode != 0: @@ -356,11 +362,17 @@ def finish_executor(handle: dict[str, Any], timeout: float) -> dict[str, Any]: def stop_executor(handle: dict[str, Any]) -> None: process: subprocess.Popen[str] = handle["process"] if process.poll() is None: - process.terminate() + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass try: process.wait(timeout=1.0) except subprocess.TimeoutExpired: - process.kill() + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass process.wait(timeout=5.0) @@ -547,8 +559,10 @@ def run_direct_miss( wrong["iterations"] += 1 started = time.perf_counter() private = start_executor(args.binary, wrong, args.tool_cpus, f"{label}-wrong") - model = post_model(args.url, arguments, args.max_tokens, args.timeout) - stop_executor(private) + try: + model = post_model(args.url, arguments, args.max_tokens, args.timeout) + finally: + stop_executor(private) authoritative = run_executor( args.binary, arguments, @@ -562,7 +576,6 @@ def run_direct_miss( "task_ms": (time.perf_counter() - started) * 1000.0, "model": model, "authoritative_tool": authoritative, - "private_result_exposed": False, } @@ -665,9 +678,6 @@ def qualify(args: argparse.Namespace) -> None: "ds4_active": ds4_active, "model_slowdown": slowdown_percent <= args.max_model_slowdown_percent, "direct_speedup": speedup >= args.min_qualification_speedup, - "private_miss_result_hidden": all( - not row["private_result_exposed"] for row in misses - ), } passed = all(checks.values()) profile = { @@ -979,11 +989,14 @@ def summarize_native( } -def native(args: argparse.Namespace) -> None: - props = get_json(props_url(args.url), args.timeout) +def require_qualified_cpu_tool_props( + props: dict[str, Any], args: argparse.Namespace, *, automatic: bool = False +) -> dict[str, Any]: tool_props = props.get("tool_speculation") if not isinstance(tool_props, dict) or not tool_props.get("enabled"): raise SystemExit("server tool speculation is not enabled") + if automatic and not tool_props.get("automatic_prediction_enabled"): + raise SystemExit("server automatic Qwen prediction is not enabled") expected_props = { "execution_mode": "child_process_cpu_affinity", "profile_status": "qualified", @@ -1004,6 +1017,12 @@ def native(args: argparse.Namespace) -> None: args.tool_cpus ): raise SystemExit("server model/tool CPU affinity is not disjoint") + return tool_props + + +def native(args: argparse.Namespace) -> None: + props = get_json(props_url(args.url), args.timeout) + tool_props = require_qualified_cpu_tool_props(props, args) arguments = expected_arguments( args.rows, @@ -1149,15 +1168,9 @@ def native(args: argparse.Namespace) -> None: def native_qwen(args: argparse.Namespace) -> None: props = get_json(props_url(args.url), args.timeout) - tool_props = props.get("tool_speculation") - if not isinstance(tool_props, dict) or not tool_props.get("enabled"): - raise SystemExit("server tool speculation is not enabled") - if not tool_props.get("automatic_prediction_enabled"): - raise SystemExit("server automatic Qwen prediction is not enabled") - if tool_props.get("execution_mode") != "child_process_cpu_affinity": - raise SystemExit("automatic benchmark requires the isolated CPU executor") - if tool_props.get("tool_cpu_affinity") != args.tool_cpus: - raise SystemExit("server tool CPU affinity differs from benchmark") + tool_props = require_qualified_cpu_tool_props( + props, args, automatic=True + ) arguments = expected_arguments( args.rows, diff --git a/optimizations/ooo_spec_lucebox5_cpu/benchmark_trace_compiled_workflows.py b/optimizations/ooo_spec_lucebox5_cpu/benchmark_trace_compiled_workflows.py index 208bb7861..e0f5fa3a4 100644 --- a/optimizations/ooo_spec_lucebox5_cpu/benchmark_trace_compiled_workflows.py +++ b/optimizations/ooo_spec_lucebox5_cpu/benchmark_trace_compiled_workflows.py @@ -53,6 +53,12 @@ canonical_call, ) +RESULTS_DIR = Path(__file__).with_name("results") +CANONICAL_TRAINING_REPORT = ( + RESULTS_DIR / "multiturn-cached-wordref-production-6tasks.json" +) +CANONICAL_WORKFLOW_REGISTRY = RESULTS_DIR / "trace-workflow-registry.json" + @dataclass(frozen=True) class ArgumentBinding: @@ -580,20 +586,24 @@ def post_turn( max_tokens: int, *, automatic_tool_speculation: bool = False, + tool_speculation: dict[str, Any] | None = None, ) -> dict[str, Any]: + request = { + "model": "dflash", + "messages": messages, + "tools": tools, + "tool_choice": tool_choice, + "temperature": 0, + "seed": args.seed, + "max_tokens": max_tokens, + "stream": False, + "automatic_tool_speculation": automatic_tool_speculation, + } + if tool_speculation is not None: + request["tool_speculation"] = tool_speculation result, wall_ms = post_json( args.url, - { - "model": "dflash", - "messages": messages, - "tools": tools, - "tool_choice": tool_choice, - "temperature": 0, - "seed": args.seed, - "max_tokens": max_tokens, - "stream": False, - "automatic_tool_speculation": automatic_tool_speculation, - }, + request, args.timeout, ) observation = model_observation(result, wall_ms) @@ -728,20 +738,22 @@ def final_answer_correct(content: str, expected: str) -> bool: def post_final( args: argparse.Namespace, - expected: str, + messages: list[dict[str, Any]], ) -> dict[str, Any]: - """Measure the same minimal final-response turn for every benchmark arm.""" + """Ask the model to derive the receipt from the actual tool conversation.""" return post_turn( args, [ + *messages, { - "role": "system", + "role": "user", "content": ( - "Return the opaque workflow receipt from the user message exactly. " - "Do not explain, reformat, or add any characters." + "Copy the final_ref strings from the completed tool result in " + "their original item order. Return exactly one receipt in the " + "form workflow_complete:ref_one,ref_two. Replace the example " + "references with the actual values. Use no spaces or prose." ), }, - {"role": "user", "content": expected}, ], [], "none", @@ -749,6 +761,33 @@ def post_final( ) +def stage_result_message( + pattern: CompiledPattern, + step_index: int, + branches: list[dict[str, Any]], + tool_call_id: str, +) -> dict[str, Any]: + step = pattern.steps[step_index] + content = { + "stage": step_index + 1, + "complete": step_index + 1 == len(pattern.steps), + "items": [ + { + **branch["root"], + "call_ref": branch["steps"][-1]["tool_result"]["call_ref"], + } + for branch in branches + ], + "side_effects": False, + } + return { + "role": "tool", + "tool_call_id": tool_call_id, + "name": f"batch_{step.tool}", + "content": json.dumps(content, sort_keys=True, separators=(",", ":")), + } + + def flatten_graph_calls(graph: dict[str, Any]) -> list[str]: return [ canonical_call(step["call"]) @@ -774,6 +813,7 @@ def run_stage_batched( """Run a strong non-speculative baseline with parallel calls per stage.""" started = time.perf_counter() stage_messages = stage_batched_messages(task, pattern) + final_messages = list(stage_messages) current_tools: list[dict[str, Any]] = [] branches = [ {"root": root, "steps": [], "final_ref": ""} for root in task["items"] @@ -839,12 +879,21 @@ def run_stage_batched( "tool_wall_ms": tool["wall_ms"], } ) + if step_index + 1 == len(pattern.steps): + final_messages.extend( + [ + model["assistant_message"], + stage_result_message( + pattern, step_index, branches, model["calls"][0]["id"] + ), + ] + ) turns.append(model) for branch in branches: branch["final_ref"] = branch["steps"][-1]["tool_result"]["call_ref"] expected = expected_final(branches) - final = post_final(args, expected) + final = post_final(args, final_messages) all_turns = [*turns, final] graph = {"branches": branches} return { @@ -931,12 +980,15 @@ def run_macro( args, messages, tools, - "required", + {"type": "function", "function": {"name": pattern.macro_name}}, args.macro_max_tokens, automatic_tool_speculation=speculative, ) if len(model["calls"]) != 1: - raise RuntimeError(f"{task['id']}: macro turn emitted {len(model['calls'])} calls") + raise RuntimeError( + f"{task['id']}: macro turn emitted {len(model['calls'])} calls; " + f"content={model['content']!r}" + ) emitted = model["calls"][0] macro_correct = emitted["call"] == expected_call if not macro_correct: @@ -981,7 +1033,7 @@ def run_macro( ] ) expected = expected_final(graph["branches"]) - final = post_final(args, expected) + final = post_final(args, messages) all_turns = [model, final] return { "task_ms": (time.perf_counter() - started) * 1_000.0, @@ -1011,6 +1063,199 @@ def run_macro( } +def interference_observation_qualified(observation: dict[str, Any]) -> bool: + compiled = observation["compiled"] + speculative = observation["speculative"] + return ( + compiled["cache_hit"] == speculative["cache_hit"] + and compiled["cached_prefix_tokens"] + == speculative["cached_prefix_tokens"] + and compiled["completion_tokens"] + == speculative["completion_tokens"] + and compiled["call_sha256"] == speculative["call_sha256"] + and compiled["accept_rate"] > 0.0 + and speculative["accept_rate"] > 0.0 + ) + + +def measure_private_miss( + args: argparse.Namespace, + target_task: dict[str, Any], + predicted_task: dict[str, Any], + pattern: CompiledPattern, +) -> dict[str, Any]: + """Verify that a wrong workflow never crosses the exact-match gate.""" + target_ref = workflow_reference(target_task, pattern) + predicted_ref = workflow_reference(predicted_task, pattern) + if target_ref == predicted_ref: + raise ValueError("privacy miss requires distinct workflow references") + expected_call = { + "name": pattern.macro_name, + "arguments": {"workflow_ref": target_ref}, + } + predicted_call = { + "name": pattern.macro_name, + "arguments": {"workflow_ref": predicted_ref}, + } + observation = post_turn( + args, + macro_messages(target_task, pattern), + [pattern.macro_tool(args.max_branches, target_ref)], + {"type": "function", "function": {"name": pattern.macro_name}}, + args.macro_max_tokens, + tool_speculation={"call": predicted_call, "confidence": 1.0}, + ) + if ( + len(observation["calls"]) != 1 + or observation["calls"][0]["call"] != expected_call + ): + raise RuntimeError("privacy miss did not emit the expected authoritative call") + metadata = observation.get("speculation") + passed = ( + isinstance(metadata, dict) + and metadata.get("status") == "miss" + and metadata.get("reason") == "invocation_mismatch" + and metadata.get("prediction") == predicted_call + and "result" not in metadata + ) + return { + "passed": passed, + "status": metadata.get("status") if isinstance(metadata, dict) else None, + "reason": metadata.get("reason") if isinstance(metadata, dict) else None, + "private_result_exposed": ( + "result" in metadata if isinstance(metadata, dict) else None + ), + "prediction_sha256": hashlib.sha256( + canonical_call(predicted_call).encode() + ).hexdigest(), + "authoritative_call_sha256": hashlib.sha256( + canonical_call(expected_call).encode() + ).hexdigest(), + } + + +def interference_probe_qualified(probe: dict[str, Any]) -> bool: + """Return whether every repeated probe used matched measured conditions.""" + observations = probe.get("observations") + if isinstance(observations, list): + return bool(observations) and all( + interference_observation_qualified(observation) + for observation in observations + ) + return interference_observation_qualified(probe) + + +def measure_interference_probe( + args: argparse.Namespace, + task: dict[str, Any], + pattern: CompiledPattern, + pair_index: int, + previous: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Compare target compute with repeated, immediately warmed prompts.""" + messages = macro_messages(task, pattern) + workflow_ref = workflow_reference(task, pattern) + tools = [pattern.macro_tool(args.max_branches, workflow_ref)] + expected_call = { + "name": pattern.macro_name, + "arguments": {"workflow_ref": workflow_ref}, + } + tool_choice = { + "type": "function", + "function": {"name": pattern.macro_name}, + } + observations: list[dict[str, Any]] = [] + if isinstance(previous, dict): + prior_observations = previous.get("observations") + if isinstance(prior_observations, list): + observations.extend(prior_observations) + elif isinstance(previous.get("compiled"), dict) and isinstance( + previous.get("speculative"), dict + ): + observations.append( + { + "order": previous.get("order", ["compiled", "speculative"]), + "compiled": previous["compiled"], + "speculative": previous["speculative"], + } + ) + + while len(observations) < args.interference_repetitions: + repetition = len(observations) + order = ( + [False, True] + if (pair_index + repetition) % 2 == 0 + else [True, False] + ) + measured: dict[str, dict[str, Any]] = {} + for automatic in order: + # Prefix-cache order dominated the old six-sample p95. Warm the + # exact prompt immediately before each observation so the probe + # measures predictor/tool interference, not arm ordering. + post_turn( + args, + messages, + tools, + tool_choice, + args.macro_max_tokens, + automatic_tool_speculation=False, + ) + observation = post_turn( + args, + messages, + tools, + tool_choice, + args.macro_max_tokens, + automatic_tool_speculation=automatic, + ) + if ( + len(observation["calls"]) != 1 + or observation["calls"][0]["call"] != expected_call + ): + raise RuntimeError( + f"{task['id']}: interference probe emitted an invalid macro call" + ) + metadata = observation.get("speculation") + if automatic and ( + not isinstance(metadata, dict) + or metadata.get("status") != "hit" + or metadata.get("prediction_source") != NATIVE_PREDICTION_SOURCE + or metadata.get("prediction") != expected_call + ): + raise RuntimeError( + f"{task['id']}: interference probe did not produce an exact Qwen hit" + ) + measured["speculative" if automatic else "compiled"] = { + "model_compute_ms": observation["model_compute_ms"], + "prefill_ms": observation["prefill_ms"], + "decode_ms": observation["decode_ms"], + "completion_tokens": observation["completion_tokens"], + "accept_rate": observation["accept_rate"], + "cache_hit": observation["cache_hit"], + "cached_prefix_tokens": observation["cached_prefix_tokens"], + "call_sha256": hashlib.sha256( + canonical_call(expected_call).encode() + ).hexdigest(), + } + observations.append( + { + "order": [ + "speculative" if value else "compiled" for value in order + ], + **measured, + } + ) + + probe = { + "repetitions": len(observations), + "observations": observations, + } + # A cache miss is valid when both observations miss identically. Requiring + # a hit made an otherwise matched, zero-cache comparison fail closed. + probe["qualified"] = interference_probe_qualified(probe) + return probe + + def macro_signature(arm: dict[str, Any]) -> dict[str, Any]: turn = arm["call_turns"][0] return { @@ -1038,14 +1283,23 @@ def bootstrap_speedup_ci( return [percentile(values, 0.025), percentile(values, 0.975)] -def paired_slowdown( +def paired_probe_slowdown( pairs: list[dict[str, Any]], metric: str, quantile: float ) -> float: - ratios = [ - pair["speculative"][metric] / pair["compiled"][metric] - for pair in pairs - if pair["compiled"][metric] > 0.0 - ] + ratios = [] + for pair in pairs: + probe = pair["interference_probe"] + observations = probe.get("observations") + if not isinstance(observations, list): + observations = [probe] + paired_ratios = [ + observation["speculative"][metric] + / observation["compiled"][metric] + for observation in observations + if observation["compiled"][metric] > 0.0 + ] + if paired_ratios: + ratios.append(statistics.median(paired_ratios)) return 100.0 * (percentile(ratios, quantile) - 1.0) @@ -1150,14 +1404,21 @@ def summarize( "predictor_p50_ms": statistics.median( pair["speculative"]["predictor_ms"] for pair in pairs ), - "model_compute_slowdown_p50_percent": paired_slowdown( + "model_compute_slowdown_p50_percent": paired_probe_slowdown( pairs, "model_compute_ms", 0.50 ), - "model_compute_slowdown_p95_percent": paired_slowdown( + "model_compute_slowdown_p95_percent": paired_probe_slowdown( pairs, "model_compute_ms", 0.95 ), - "decode_slowdown_p50_percent": paired_slowdown(pairs, "decode_ms", 0.50), - "decode_slowdown_p95_percent": paired_slowdown(pairs, "decode_ms", 0.95), + "decode_slowdown_p50_percent": paired_probe_slowdown( + pairs, "decode_ms", 0.50 + ), + "decode_slowdown_p95_percent": paired_probe_slowdown( + pairs, "decode_ms", 0.95 + ), + "all_interference_probes_qualified": all( + pair["interference_probe"]["qualified"] for pair in pairs + ), "continuation_cache_hit_rate": sum( turn["cache_hit"] and turn["cached_prefix_tokens"] > 0 for turn in continuation_turns @@ -1224,6 +1485,8 @@ def production_checks(summary: dict[str, Any], args: argparse.Namespace) -> dict > 1.0, "prediction_hit_rate": summary["pattern_prediction_hit_rate"] == 1.0, "prediction_source": summary["all_predictions_from_qwen"], + "private_miss_result_hidden": summary["private_miss_result_hidden"], + "interference_probes": summary["all_interference_probes_qualified"], "model_slowdown_p50": summary["model_compute_slowdown_p50_percent"] <= args.max_model_slowdown_percent, "model_slowdown_p95": summary["model_compute_slowdown_p95_percent"] @@ -1299,6 +1562,7 @@ def compact_pair(pair: dict[str, Any]) -> dict[str, Any]: arm: compact_arm(pair[arm]) for arm in ("stage_batched", "compiled", "speculative") }, + "interference_probe": pair["interference_probe"], } @@ -1354,6 +1618,16 @@ def validate_args(parser: argparse.ArgumentParser, args: argparse.Namespace) -> parser.error("--binary must be an executable tool adapter") if not args.training_report.is_file(): parser.error("--training-report must be an existing trace file or report") + if args.training_report != CANONICAL_TRAINING_REPORT.resolve(): + parser.error( + "--training-report must be the committed canonical report used by " + "the deployed trace executor" + ) + if args.workflow_registry != CANONICAL_WORKFLOW_REGISTRY.resolve(): + parser.error( + "--workflow-registry must be the committed canonical registry used " + "by the deployed trace executor" + ) if ( args.pairs <= 0 or args.warmup_tasks < 0 @@ -1361,6 +1635,8 @@ def validate_args(parser: argparse.ArgumentParser, args: argparse.Namespace) -> or args.timeout <= 0 or min(args.call_max_tokens, args.macro_max_tokens, args.final_max_tokens) <= 0 or args.bootstrap_resamples <= 0 + or args.interference_repetitions < 3 + or args.interference_repetitions % 2 == 0 or args.min_production_pairs < 2 or args.min_e2e_speedup <= 1.0 or args.min_e2e_speedup_p05 <= 1.0 @@ -1369,6 +1645,69 @@ def validate_args(parser: argparse.ArgumentParser, args: argparse.Namespace) -> parser.error("benchmark counts and thresholds are invalid") +def refresh_existing_report( + args: argparse.Namespace, + pattern: CompiledPattern, + measured_tasks: list[dict[str, Any]], + prefix_cache: dict[str, Any], + tool_speculation: dict[str, Any], +) -> bool: + """Add newly introduced safety gates without repeating long timing arms.""" + report = json.loads(args.output.read_text(encoding="utf-8")) + if not isinstance(report, dict): + raise ValueError("existing report is not a JSON object") + recorded_pattern = report.get("pattern") + if ( + report.get("schema_version") != 1 + or not isinstance(recorded_pattern, dict) + or recorded_pattern.get("fingerprint") != pattern.fingerprint + or recorded_pattern.get("training_report_sha256") + != file_sha256(args.training_report) + or recorded_pattern.get("workflow_registry_sha256") + != file_sha256(args.workflow_registry) + or not report.get("production_gate", {}).get("passed") + or len(report.get("pairs", [])) != args.pairs + ): + raise ValueError("existing report does not match this qualified run") + + privacy_miss = measure_private_miss( + args, measured_tasks[0], measured_tasks[1], pattern + ) + summary = report.get("summary") + if not isinstance(summary, dict) or summary.get("tasks") != args.pairs: + raise ValueError("existing report summary does not match --pairs") + summary["private_miss_result_hidden"] = privacy_miss["passed"] + checks = production_checks(summary, args) + report["privacy_miss"] = privacy_miss + report["methodology"]["additive_gate_refresh"] = ( + "the wrong-call privacy probe and server snapshot were refreshed after " + "the timing arms; no recorded timing was recomputed" + ) + report["production_gate"]["checks"] = checks + report["production_gate"]["passed"] = all(checks.values()) + ending_props = get_json(props_url(args.url), args.timeout) + report["server_snapshot"]["prefix_cache_before"] = prefix_cache + report["server_snapshot"]["prefix_cache_after"] = ending_props.get( + "prefix_cache" + ) + report["server_snapshot"]["tool_speculation"] = tool_speculation + args.output.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + print( + json.dumps( + { + "production_gate": report["production_gate"], + "privacy_miss": privacy_miss, + }, + indent=2, + sort_keys=True, + ), + flush=True, + ) + return report["production_gate"]["passed"] + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--url", default="http://127.0.0.1:18145/v1/chat/completions") @@ -1377,7 +1716,7 @@ def main() -> int: parser.add_argument( "--workflow-registry", type=Path, - default=Path(__file__).with_name("results") / "trace-workflow-registry.json", + default=CANONICAL_WORKFLOW_REGISTRY, ) parser.add_argument("--tool-cpus", type=parse_cpu_list, default="14-15,30-31") parser.add_argument("--pairs", type=int, default=6) @@ -1390,6 +1729,7 @@ def main() -> int: parser.add_argument("--timeout", type=float, default=180.0) parser.add_argument("--seed", type=int, default=814) parser.add_argument("--bootstrap-resamples", type=int, default=20_000) + parser.add_argument("--interference-repetitions", type=int, default=3) parser.add_argument("--min-production-pairs", type=int, default=6) parser.add_argument("--min-e2e-speedup", type=float, default=2.0) parser.add_argument("--min-e2e-speedup-p05", type=float, default=1.5) @@ -1403,6 +1743,11 @@ def main() -> int: action="store_true", help="resume the strictly matching .partial checkpoint", ) + parser.add_argument( + "--refresh-report", + action="store_true", + help="run additive safety gates against an existing passing report", + ) parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() args.binary = args.binary.resolve() @@ -1434,6 +1779,11 @@ def main() -> int: f"engine tool_speculation.{key}={tool_speculation.get(key)!r}, " f"expected {expected!r}" ) + if tool_speculation.get("tool_cpu_affinity") != args.tool_cpus: + raise SystemExit( + "engine tool CPU affinity does not match --tool-cpus: " + f"{tool_speculation.get('tool_cpu_affinity')!r} != {args.tool_cpus!r}" + ) if pattern.macro_name not in tool_speculation.get("allowed_tools", []): raise SystemExit(f"engine does not allow compiled macro {pattern.macro_name!r}") @@ -1453,6 +1803,14 @@ def main() -> int: write_workflow_registry( args.workflow_registry, pattern, [*warmup_tasks, *measured_tasks] ) + if args.refresh_report: + if not args.output.is_file(): + parser.error("--refresh-report requires an existing --output") + if not refresh_existing_report( + args, pattern, measured_tasks, prefix_cache, tool_speculation + ): + raise SystemExit("refreshed workflow production gate failed") + return 0 generator = random.Random(args.seed) arm_orders = [] for _ in measured_tasks: @@ -1527,7 +1885,48 @@ def main() -> int: flush=True, ) + for pair_index, pair in enumerate(pairs): + previous_probe = pair.get("interference_probe") + if ( + isinstance(previous_probe, dict) + and isinstance(previous_probe.get("observations"), list) + and len(previous_probe["observations"]) + >= args.interference_repetitions + ): + pair["interference_probe"]["qualified"] = ( + interference_probe_qualified(pair["interference_probe"]) + ) + continue + pair["interference_probe"] = measure_interference_probe( + args, + measured_tasks[pair_index], + pattern, + pair_index, + previous_probe if isinstance(previous_probe, dict) else None, + ) + partial_output.write_text( + json.dumps( + {"schema_version": 1, "complete": False, "pairs": pairs}, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + print( + json.dumps( + { + "interference_probe": pair_index + 1, + "qualified": pair["interference_probe"]["qualified"], + } + ), + flush=True, + ) + + privacy_miss = measure_private_miss( + args, measured_tasks[0], measured_tasks[1], pattern + ) summary = summarize(pairs, args.bootstrap_resamples, args.seed) + summary["private_miss_result_hidden"] = privacy_miss["passed"] ending_props = get_json(props_url(args.url), args.timeout) ending_prefix_cache = ending_props.get("prefix_cache") if not isinstance(ending_prefix_cache, dict): @@ -1545,9 +1944,13 @@ def main() -> int: "macro_name": pattern.macro_name, "fingerprint": pattern.fingerprint, "training_traces": pattern.training_traces, - "training_report": str(args.training_report), + "training_report": str( + args.training_report.relative_to(Path(__file__).resolve().parent) + ), "training_report_sha256": file_sha256(args.training_report), - "workflow_registry": str(args.workflow_registry), + "workflow_registry": str( + args.workflow_registry.relative_to(Path(__file__).resolve().parent) + ), "workflow_registry_sha256": file_sha256(args.workflow_registry), "root_fields": list(pattern.root_fields), "steps": [ @@ -1615,6 +2018,7 @@ def main() -> int: "max_decode_slowdown_p95_percent": args.max_decode_slowdown_p95_percent, }, }, + "privacy_miss": privacy_miss, "summary": summary, "pairs": [compact_pair(pair) for pair in pairs], } diff --git a/optimizations/ooo_spec_lucebox5_cpu/bfcl_replay_tool_executor.py b/optimizations/ooo_spec_lucebox5_cpu/bfcl_replay_tool_executor.py index e1d79a101..9c3416e95 100755 --- a/optimizations/ooo_spec_lucebox5_cpu/bfcl_replay_tool_executor.py +++ b/optimizations/ooo_spec_lucebox5_cpu/bfcl_replay_tool_executor.py @@ -50,7 +50,7 @@ def execute(request: dict[str, Any]) -> dict[str, Any]: if not isinstance(arguments, dict): raise ValueError("tool arguments must be an object") - expected = request.get("cpu_affinity") or [] + expected = request.get("cpu_affinity", []) if not isinstance(expected, list) or not all( isinstance(cpu, int) and cpu >= 0 for cpu in expected ): diff --git a/optimizations/ooo_spec_lucebox5_cpu/cpu_sparse_tool_executor.cpp b/optimizations/ooo_spec_lucebox5_cpu/cpu_sparse_tool_executor.cpp index 95be7dd7b..4a52f47ee 100644 --- a/optimizations/ooo_spec_lucebox5_cpu/cpu_sparse_tool_executor.cpp +++ b/optimizations/ooo_spec_lucebox5_cpu/cpu_sparse_tool_executor.cpp @@ -135,48 +135,30 @@ json execute(const json & request) { throw std::runtime_error("only benchmark_cpu_sparse is allowed"); } const json & arguments = request["call"].at("arguments"); - if (!arguments.is_object()) { - throw std::runtime_error("arguments must be an object"); + if (!arguments.is_object() || arguments.size() != 1 || + !arguments.contains("iterations")) { + throw std::runtime_error("arguments must contain only iterations"); } - const int rows = arguments.contains("rows") - ? integer_argument(arguments, "rows", 64, 1 << 20) : kRows; - const int nonzeros = arguments.contains("nonzeros_per_row") - ? integer_argument(arguments, "nonzeros_per_row", 1, 256) - : kNonzerosPerRow; + const int rows = kRows; + const int nonzeros = kNonzerosPerRow; const int iterations = integer_argument( arguments, "iterations", 1, 1'000'000); - const int threads = arguments.contains("threads") - ? integer_argument(arguments, "threads", 1, 64) : kThreads; - if (static_cast(rows) * static_cast(nonzeros) > - 16ULL * 1024ULL * 1024ULL) { - throw std::runtime_error("sparse matrix exceeds the 16M-entry limit"); - } - uint64_t seed = kSeed; - if (arguments.contains("seed")) { - if (!arguments["seed"].is_number_integer()) { - throw std::runtime_error("seed must be an unsigned integer"); - } - if (arguments["seed"].is_number_unsigned()) { - seed = arguments["seed"].get(); - } else { - const int64_t signed_seed = arguments["seed"].get(); - if (signed_seed < 0) { - throw std::runtime_error("seed must be an unsigned integer"); - } - seed = static_cast(signed_seed); - } - } + const int threads = kThreads; + const uint64_t seed = kSeed; std::vector expected_affinity; - if (request.contains("cpu_affinity")) { - expected_affinity = request["cpu_affinity"].get>(); - std::sort(expected_affinity.begin(), expected_affinity.end()); - expected_affinity.erase( - std::unique(expected_affinity.begin(), expected_affinity.end()), - expected_affinity.end()); - } + const auto affinity_value = request.find("cpu_affinity"); + if (affinity_value == request.end() || !affinity_value->is_array() || + affinity_value->empty()) { + throw std::runtime_error("cpu_affinity must be a non-empty array"); + } + expected_affinity = affinity_value->get>(); + std::sort(expected_affinity.begin(), expected_affinity.end()); + expected_affinity.erase( + std::unique(expected_affinity.begin(), expected_affinity.end()), + expected_affinity.end()); const std::vector affinity = observed_affinity(); - if (!expected_affinity.empty() && affinity != expected_affinity) { + if (affinity != expected_affinity) { throw std::runtime_error("observed CPU affinity does not match request"); } if (!affinity.empty() && threads > static_cast(affinity.size())) { diff --git a/optimizations/ooo_spec_lucebox5_cpu/dflash_server_native_tool_predictor_wrapper.sh b/optimizations/ooo_spec_lucebox5_cpu/dflash_server_native_tool_predictor_wrapper.sh index fd48d27ab..07e2a399e 100755 --- a/optimizations/ooo_spec_lucebox5_cpu/dflash_server_native_tool_predictor_wrapper.sh +++ b/optimizations/ooo_spec_lucebox5_cpu/dflash_server_native_tool_predictor_wrapper.sh @@ -18,7 +18,6 @@ PREDICTOR_MAX_CTX="${PREDICTOR_MAX_CTX:-4096}" PREDICTOR_MAX_TOKENS="${PREDICTOR_MAX_TOKENS:-256}" PREDICTOR_TIMEOUT_MS="${PREDICTOR_TIMEOUT_MS:-2000}" PREDICTOR_CONFIDENCE="${PREDICTOR_CONFIDENCE:-0.75}" -PREDICTOR_SCHEDULE="${PREDICTOR_SCHEDULE:-before-model}" # The qualified 0731 launcher disables caches for cold throughput benchmarks. # Tool-using agent loops need turn-boundary reuse; this later CLI flag wins # without modifying the qualified model/DSpark arguments. @@ -26,7 +25,7 @@ PREFIX_CACHE_SLOTS_OVERRIDE="${PREFIX_CACHE_SLOTS_OVERRIDE:-32}" cache_args=() if [[ -n "${PREFIX_CACHE_SLOTS_OVERRIDE}" ]]; then - [[ "${PREFIX_CACHE_SLOTS_OVERRIDE}" =~ ^[1-9][0-9]*$ ]] || { + [[ "${PREFIX_CACHE_SLOTS_OVERRIDE}" =~ ^(0|[1-9][0-9]*)$ ]] || { printf 'invalid PREFIX_CACHE_SLOTS_OVERRIDE: %s\n' \ "${PREFIX_CACHE_SLOTS_OVERRIDE}" >&2 exit 2 @@ -38,15 +37,17 @@ if [[ -n "${PREFIX_CACHE_SLOTS_OVERRIDE}" ]]; then cache_args+=(--prefix-cache-slots "${PREFIX_CACHE_SLOTS_OVERRIDE}") fi -for required in \ - "${CANDIDATE_BUILD}/dflash_server" \ - "${PREDICTOR_IPC_BIN}" \ - "${PREDICTOR_MODEL}"; do - [[ -e "${required}" ]] || { - printf 'missing Qwen tool-predictor path: %s\n' "${required}" >&2 +for binary in "${CANDIDATE_BUILD}/dflash_server" "${PREDICTOR_IPC_BIN}"; do + [[ -f "${binary}" && -x "${binary}" ]] || { + printf 'Qwen tool-predictor binary is not executable: %s\n' "${binary}" >&2 exit 2 } done +[[ -f "${PREDICTOR_MODEL}" ]] || { + printf 'Qwen tool-predictor model is not a regular file: %s\n' \ + "${PREDICTOR_MODEL}" >&2 + exit 2 +} export LD_LIBRARY_PATH="${CANDIDATE_BUILD}/deps/llama.cpp/ggml/src:${CANDIDATE_BUILD}/deps/llama.cpp/ggml/src/ggml-hip:${LD_LIBRARY_PATH:-}" export LUCE_MMVQ_MAX_NCOLS=5 @@ -58,6 +59,5 @@ exec "${CANDIDATE_BUILD}/dflash_server" "$@" \ --tool-hint-native-max-ctx "${PREDICTOR_MAX_CTX}" \ --tool-hint-max-tokens "${PREDICTOR_MAX_TOKENS}" \ --tool-hint-timeout-ms "${PREDICTOR_TIMEOUT_MS}" \ - --tool-hint-native-schedule "${PREDICTOR_SCHEDULE}" \ --tool-hint-execution-confidence "${PREDICTOR_CONFIDENCE}" \ "${cache_args[@]}" diff --git a/optimizations/ooo_spec_lucebox5_cpu/profiles/lucebox5-cpu-lane-qualified.json b/optimizations/ooo_spec_lucebox5_cpu/profiles/lucebox5-cpu-lane-qualified.json index 84b0ad107..64be189a5 100644 --- a/optimizations/ooo_spec_lucebox5_cpu/profiles/lucebox5-cpu-lane-qualified.json +++ b/optimizations/ooo_spec_lucebox5_cpu/profiles/lucebox5-cpu-lane-qualified.json @@ -28,6 +28,7 @@ "model_slowdown": true, "private_miss_result_hidden": true }, + "private_miss_evidence": "results/trace-compiled-engine-qwen-production-6pairs-compact.json", "host": "lucebox5", "model_cpu_affinity": [ 0, diff --git a/optimizations/ooo_spec_lucebox5_cpu/results/multiturn-cached-wordref-production-6tasks.json b/optimizations/ooo_spec_lucebox5_cpu/results/multiturn-cached-wordref-production-6tasks.json new file mode 100644 index 000000000..5bcaa9d9f --- /dev/null +++ b/optimizations/ooo_spec_lucebox5_cpu/results/multiturn-cached-wordref-production-6tasks.json @@ -0,0 +1,5595 @@ +{ + "host": "lucebox5", + "methodology": { + "arm_order": "randomized per task", + "control": "DS4+DSpark generates each call, then the authoritative tool runs", + "measured_wall_time": "initial request through every dependent model/tool turn and final answer", + "model_seed": 814, + "oracle_prediction": false, + "per_task_cache_warmups": 1, + "semantic_token_injection": false, + "speculative": "Qwen3-0.6B predicts each call, launches the private CPU tool before DS4+DSpark, and commits only after exact target verification", + "warmup_pairs": 0 + }, + "pairs": [ + { + "arm_order": [ + "speculative", + "control" + ], + "control": { + "all_calls_correct": true, + "all_ds4_active": true, + "all_private_miss_results_hidden": true, + "call_count": 3, + "completion_tokens": 153, + "decode_ms": 8322.7, + "expected_final": "workflow_complete:coral", + "exposed_tool_wait_ms": 6080.452936002985, + "final": { + "accept_rate": 0.625, + "assistant_message": { + "content": "workflow_complete:coral", + "role": "assistant" + }, + "cache_hit": true, + "cached_prefix_tokens": 804, + "call": null, + "canonical_call": null, + "completion_tokens": 7, + "content": "workflow_complete:coral", + "content_sha256": "1b68642fe5b149f128d3b8b3eff6391869b30634dc6b45bad25d878cb644f588", + "decode_ms": 434.6, + "decode_tokens_per_sec": 16.1, + "model_compute_ms": 10982.9, + "prefill_ms": 10548.3, + "prefilled_tokens": 245, + "request_wall_ms": 11004.08883899945, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": null, + "tool_call_id": "" + }, + "final_correct": true, + "model_compute_ms": 32930.0, + "prediction_hits": 0, + "predictor_wall_ms": 0.0, + "steps": [ + { + "accept_rate": 0.59375, + "assistant_message": { + "content": "I need to process the customer for destination Rome. Let me start with the first step.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"customer_email\":\"agent-benchmark-0@example.test\"}", + "name": "resolve_customer" + }, + "id": "call_d8a0080f5f9a77cfc6750bd5", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 602, + "call": { + "arguments": { + "customer_email": "agent-benchmark-0@example.test" + }, + "name": "resolve_customer" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"customer_email\":\"agent-benchmark-0@example.test\"},\"name\":\"resolve_customer\"}", + "completion_tokens": 55, + "content": "I need to process the customer for destination Rome. Let me start with the first step.\n\n", + "content_sha256": "e16dd4db257f04920bcbb955a31511f0a7f9a9e1190fa46726d89b9a1907b2a1", + "decode_ms": 3256.7, + "decode_tokens_per_sec": 16.9, + "expected_call": { + "arguments": { + "customer_email": "agent-benchmark-0@example.test" + }, + "name": "resolve_customer" + }, + "exposed_tool_wait_ms": 2026.3470799982315, + "model_compute_ms": 3256.7, + "prediction": null, + "prediction_hit": false, + "prediction_reason": null, + "prediction_status": null, + "predictor_wall_ms": 0.0, + "prefill_ms": 0.0, + "prefilled_tokens": 0, + "private_result_exposed_on_miss": false, + "request_wall_ms": 3269.99559700198, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": null, + "tool_call_id": "call_d8a0080f5f9a77cfc6750bd5", + "tool_result": { + "call_ref": "jade", + "call_sha256": "fa84028f950e9e7f0c3c6265db076d178d96fdb6a5b0d34ac8e14f2982d3e3e4", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.1487330009695, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "resolve_customer" + }, + "tool_wall_ms": 2026.3470799982315 + }, + { + "accept_rate": 0.6346153616905212, + "assistant_message": { + "content": "Now I need to list open orders using the customer_ref from the previous step.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"customer_ref\":\"jade\"}", + "name": "list_open_orders" + }, + "id": "call_fc905ce6914aa15ba13e127f", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 658, + "call": { + "arguments": { + "customer_ref": "jade" + }, + "name": "list_open_orders" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"customer_ref\":\"jade\"},\"name\":\"list_open_orders\"}", + "completion_tokens": 46, + "content": "Now I need to list open orders using the customer_ref from the previous step.\n\n", + "content_sha256": "7de2bb2ccde5cf6b6fc9c84c9ba2744046323df3d0313b5d3cebc00f881a5dac", + "decode_ms": 2718.8, + "decode_tokens_per_sec": 16.9, + "expected_call": { + "arguments": { + "customer_ref": "jade" + }, + "name": "list_open_orders" + }, + "exposed_tool_wait_ms": 2026.6102390014566, + "model_compute_ms": 6878.0, + "prediction": null, + "prediction_hit": false, + "prediction_reason": null, + "prediction_status": null, + "predictor_wall_ms": 0.0, + "prefill_ms": 4159.2, + "prefilled_tokens": 98, + "private_result_exposed_on_miss": false, + "request_wall_ms": 6897.866800001793, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": null, + "tool_call_id": "call_fc905ce6914aa15ba13e127f", + "tool_result": { + "call_ref": "silver", + "call_sha256": "a2696e6bea709259ca1771ea3d90facbabda20e73fed8d3c3b33efe2d39719d9", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.1448610018997, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "list_open_orders" + }, + "tool_wall_ms": 2026.6102390014566 + }, + { + "accept_rate": 0.7727272510528564, + "assistant_message": { + "content": "Now I need to get order details using the orders_ref from the previous step.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"orders_ref\":\"silver\"}", + "name": "get_order_details" + }, + "id": "call_ef21ac0bd404a2e1e7ac2012", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 658, + "call": { + "arguments": { + "orders_ref": "silver" + }, + "name": "get_order_details" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"orders_ref\":\"silver\"},\"name\":\"get_order_details\"}", + "completion_tokens": 45, + "content": "Now I need to get order details using the orders_ref from the previous step.\n\n", + "content_sha256": "53d3157b38fb7ba68290dce87e61a6c6a519446016f2db0acb7f5041229612de", + "decode_ms": 1912.6, + "decode_tokens_per_sec": 23.5, + "expected_call": { + "arguments": { + "orders_ref": "silver" + }, + "name": "get_order_details" + }, + "exposed_tool_wait_ms": 2027.4956170032965, + "model_compute_ms": 11812.4, + "prediction": null, + "prediction_hit": false, + "prediction_reason": null, + "prediction_status": null, + "predictor_wall_ms": 0.0, + "prefill_ms": 9899.8, + "prefilled_tokens": 236, + "private_result_exposed_on_miss": false, + "request_wall_ms": 11832.999707999988, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": null, + "tool_call_id": "call_ef21ac0bd404a2e1e7ac2012", + "tool_result": { + "call_ref": "coral", + "call_sha256": "9acecd135ddfeab105553c2cb4a4fb60b47aeb0a24152723e4e319f35fe55c46", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.4064589993504, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "get_order_details" + }, + "tool_wall_ms": 2027.4956170032965 + } + ], + "task_id": "dependent_workflow_000", + "task_ms": 39086.805807997735 + }, + "pair_index": 0, + "speculative": { + "all_calls_correct": true, + "all_ds4_active": true, + "all_private_miss_results_hidden": true, + "call_count": 3, + "completion_tokens": 153, + "decode_ms": 8348.0, + "expected_final": "workflow_complete:coral", + "exposed_tool_wait_ms": 0.062267, + "final": { + "accept_rate": 0.625, + "assistant_message": { + "content": "workflow_complete:coral", + "role": "assistant" + }, + "cache_hit": true, + "cached_prefix_tokens": 804, + "call": null, + "canonical_call": null, + "completion_tokens": 7, + "content": "workflow_complete:coral", + "content_sha256": "1b68642fe5b149f128d3b8b3eff6391869b30634dc6b45bad25d878cb644f588", + "decode_ms": 437.1, + "decode_tokens_per_sec": 16.0, + "model_compute_ms": 10953.800000000001, + "prefill_ms": 10516.7, + "prefilled_tokens": 245, + "request_wall_ms": 10969.198104998213, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": null, + "tool_call_id": "" + }, + "final_correct": true, + "model_compute_ms": 32847.1, + "prediction_hits": 3, + "predictor_wall_ms": 860.884941, + "steps": [ + { + "accept_rate": 0.59375, + "assistant_message": { + "content": "I need to process the customer for destination Rome. Let me start with the first step.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"customer_email\":\"agent-benchmark-0@example.test\"}", + "name": "resolve_customer" + }, + "id": "call_2e555e8ccc5b1932edd87ac2", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 602, + "call": { + "arguments": { + "customer_email": "agent-benchmark-0@example.test" + }, + "name": "resolve_customer" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"customer_email\":\"agent-benchmark-0@example.test\"},\"name\":\"resolve_customer\"}", + "completion_tokens": 55, + "content": "I need to process the customer for destination Rome. Let me start with the first step.\n\n", + "content_sha256": "e16dd4db257f04920bcbb955a31511f0a7f9a9e1190fa46726d89b9a1907b2a1", + "decode_ms": 3261.8, + "decode_tokens_per_sec": 16.9, + "expected_call": { + "arguments": { + "customer_email": "agent-benchmark-0@example.test" + }, + "name": "resolve_customer" + }, + "exposed_tool_wait_ms": 0.020609, + "model_compute_ms": 3261.8, + "prediction": { + "arguments": { + "customer_email": "agent-benchmark-0@example.test" + }, + "name": "resolve_customer" + }, + "prediction_hit": true, + "prediction_reason": null, + "prediction_status": "hit", + "predictor_wall_ms": 285.480545, + "prefill_ms": 0.0, + "prefilled_tokens": 0, + "private_result_exposed_on_miss": false, + "request_wall_ms": 3561.9963290009764, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_2e555e8ccc5b1932edd87ac2", + "commit_signal_sent": false, + "commit_wait_ms": 0.020609, + "confidence": 0.75, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 3274.570428, + "expected_speedup": 1.5428733045594794, + "prediction": { + "arguments": { + "customer_email": "agent-benchmark-0@example.test" + }, + "name": "resolve_customer" + }, + "prediction_source": "qwen3-0.6b", + "predictor_wall_ms": 285.480545, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "call_ref": "jade", + "call_sha256": "fa84028f950e9e7f0c3c6265db076d178d96fdb6a5b0d34ac8e14f2982d3e3e4", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.1391930018144, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "resolve_customer" + }, + "status": "hit" + }, + "tool_call_id": "call_2e555e8ccc5b1932edd87ac2", + "tool_result": { + "call_ref": "jade", + "call_sha256": "fa84028f950e9e7f0c3c6265db076d178d96fdb6a5b0d34ac8e14f2982d3e3e4", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.1391930018144, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "resolve_customer" + }, + "tool_wall_ms": 3274.570428 + }, + { + "accept_rate": 0.6346153616905212, + "assistant_message": { + "content": "Now I need to list open orders using the customer_ref from the previous step.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"customer_ref\":\"jade\"}", + "name": "list_open_orders" + }, + "id": "call_1f8ca05cb9c76e5c84513ca1", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 658, + "call": { + "arguments": { + "customer_ref": "jade" + }, + "name": "list_open_orders" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"customer_ref\":\"jade\"},\"name\":\"list_open_orders\"}", + "completion_tokens": 46, + "content": "Now I need to list open orders using the customer_ref from the previous step.\n\n", + "content_sha256": "7de2bb2ccde5cf6b6fc9c84c9ba2744046323df3d0313b5d3cebc00f881a5dac", + "decode_ms": 2733.6, + "decode_tokens_per_sec": 16.8, + "expected_call": { + "arguments": { + "customer_ref": "jade" + }, + "name": "list_open_orders" + }, + "exposed_tool_wait_ms": 0.021571, + "model_compute_ms": 6851.0, + "prediction": { + "arguments": { + "customer_ref": "jade" + }, + "name": "list_open_orders" + }, + "prediction_hit": true, + "prediction_reason": null, + "prediction_status": "hit", + "predictor_wall_ms": 278.107146, + "prefill_ms": 4117.4, + "prefilled_tokens": 98, + "private_result_exposed_on_miss": false, + "request_wall_ms": 7143.900198996562, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_1f8ca05cb9c76e5c84513ca1", + "commit_signal_sent": false, + "commit_wait_ms": 0.021571, + "confidence": 0.75, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 6863.607577, + "expected_speedup": 1.5428733045594794, + "prediction": { + "arguments": { + "customer_ref": "jade" + }, + "name": "list_open_orders" + }, + "prediction_source": "qwen3-0.6b", + "predictor_wall_ms": 278.107146, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "call_ref": "silver", + "call_sha256": "a2696e6bea709259ca1771ea3d90facbabda20e73fed8d3c3b33efe2d39719d9", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.0727330007066, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "list_open_orders" + }, + "status": "hit" + }, + "tool_call_id": "call_1f8ca05cb9c76e5c84513ca1", + "tool_result": { + "call_ref": "silver", + "call_sha256": "a2696e6bea709259ca1771ea3d90facbabda20e73fed8d3c3b33efe2d39719d9", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.0727330007066, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "list_open_orders" + }, + "tool_wall_ms": 6863.607577 + }, + { + "accept_rate": 0.7727272510528564, + "assistant_message": { + "content": "Now I need to get order details using the orders_ref from the previous step.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"orders_ref\":\"silver\"}", + "name": "get_order_details" + }, + "id": "call_5616db367d592ec04fcd50de", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 658, + "call": { + "arguments": { + "orders_ref": "silver" + }, + "name": "get_order_details" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"orders_ref\":\"silver\"},\"name\":\"get_order_details\"}", + "completion_tokens": 45, + "content": "Now I need to get order details using the orders_ref from the previous step.\n\n", + "content_sha256": "53d3157b38fb7ba68290dce87e61a6c6a519446016f2db0acb7f5041229612de", + "decode_ms": 1915.5, + "decode_tokens_per_sec": 23.5, + "expected_call": { + "arguments": { + "orders_ref": "silver" + }, + "name": "get_order_details" + }, + "exposed_tool_wait_ms": 0.020087, + "model_compute_ms": 11780.5, + "prediction": { + "arguments": { + "orders_ref": "silver" + }, + "name": "get_order_details" + }, + "prediction_hit": true, + "prediction_reason": null, + "prediction_status": "hit", + "predictor_wall_ms": 297.29725, + "prefill_ms": 9865.0, + "prefilled_tokens": 236, + "private_result_exposed_on_miss": false, + "request_wall_ms": 12093.058860999008, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_5616db367d592ec04fcd50de", + "commit_signal_sent": false, + "commit_wait_ms": 0.020087, + "confidence": 0.75, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 11793.382272, + "expected_speedup": 1.5428733045594794, + "prediction": { + "arguments": { + "orders_ref": "silver" + }, + "name": "get_order_details" + }, + "prediction_source": "qwen3-0.6b", + "predictor_wall_ms": 297.29725, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "call_ref": "coral", + "call_sha256": "9acecd135ddfeab105553c2cb4a4fb60b47aeb0a24152723e4e319f35fe55c46", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.0927090004552, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "get_order_details" + }, + "status": "hit" + }, + "tool_call_id": "call_5616db367d592ec04fcd50de", + "tool_result": { + "call_ref": "coral", + "call_sha256": "9acecd135ddfeab105553c2cb4a4fb60b47aeb0a24152723e4e319f35fe55c46", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.0927090004552, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "get_order_details" + }, + "tool_wall_ms": 11793.382272 + } + ], + "task_id": "dependent_workflow_000", + "task_ms": 33768.915132000984 + }, + "task": { + "call_count": 3, + "customer_email": "agent-benchmark-0@example.test", + "destination": "Rome", + "id": "dependent_workflow_000" + } + }, + { + "arm_order": [ + "speculative", + "control" + ], + "control": { + "all_calls_correct": true, + "all_ds4_active": true, + "all_private_miss_results_hidden": true, + "call_count": 4, + "completion_tokens": 239, + "decode_ms": 10301.2, + "expected_final": "workflow_complete:jade", + "exposed_tool_wait_ms": 8106.0383890035155, + "final": { + "accept_rate": 0.75, + "assistant_message": { + "content": "workflow_complete:jade", + "role": "assistant" + }, + "cache_hit": true, + "cached_prefix_tokens": 687, + "call": null, + "canonical_call": null, + "completion_tokens": 8, + "content": "workflow_complete:jade", + "content_sha256": "0abc4136c23738770edf0aed5459816de88d70c9b5011c2e5359bdb52557bf42", + "decode_ms": 450.4, + "decode_tokens_per_sec": 17.8, + "model_compute_ms": 24705.100000000002, + "prefill_ms": 24254.7, + "prefilled_tokens": 564, + "request_wall_ms": 24727.02061599921, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": null, + "tool_call_id": "" + }, + "final_correct": true, + "model_compute_ms": 53200.1, + "prediction_hits": 0, + "predictor_wall_ms": 0.0, + "steps": [ + { + "accept_rate": 0.7678571343421936, + "assistant_message": { + "content": "I need to start with step 1: resolve_customer using the customer email from the request.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"customer_email\":\"agent-benchmark-1@example.test\"}", + "name": "resolve_customer" + }, + "id": "call_dc82c3b16a83eb8eb0abf576", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 627, + "call": { + "arguments": { + "customer_email": "agent-benchmark-1@example.test" + }, + "name": "resolve_customer" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"customer_email\":\"agent-benchmark-1@example.test\"},\"name\":\"resolve_customer\"}", + "completion_tokens": 58, + "content": "I need to start with step 1: resolve_customer using the customer email from the request.\n\n", + "content_sha256": "9eb32ae482f67f3e7c0fd5f011a2c9c7fb3e76804c484bdf6b40bf8add1aa381", + "decode_ms": 2651.4, + "decode_tokens_per_sec": 21.9, + "expected_call": { + "arguments": { + "customer_email": "agent-benchmark-1@example.test" + }, + "name": "resolve_customer" + }, + "exposed_tool_wait_ms": 2026.4215749994037, + "model_compute_ms": 2651.4, + "prediction": null, + "prediction_hit": false, + "prediction_reason": null, + "prediction_status": null, + "predictor_wall_ms": 0.0, + "prefill_ms": 0.0, + "prefilled_tokens": 0, + "private_result_exposed_on_miss": false, + "request_wall_ms": 2664.9901300006604, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": null, + "tool_call_id": "call_dc82c3b16a83eb8eb0abf576", + "tool_result": { + "call_ref": "teal", + "call_sha256": "c42842bc1ac50a52be75d0524ab1a3ac7acd920b2f5cd44f26c522edb56075b7", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.261090001004, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "resolve_customer" + }, + "tool_wall_ms": 2026.4215749994037 + }, + { + "accept_rate": 0.7884615659713745, + "assistant_message": { + "content": "Now I need to proceed with step 2: list_open_orders using the customer_ref from the previous result.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"customer_ref\":\"teal\"}", + "name": "list_open_orders" + }, + "id": "call_9de33f3a61b9f1b82be801f2", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 686, + "call": { + "arguments": { + "customer_ref": "teal" + }, + "name": "list_open_orders" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"customer_ref\":\"teal\"},\"name\":\"list_open_orders\"}", + "completion_tokens": 54, + "content": "Now I need to proceed with step 2: list_open_orders using the customer_ref from the previous result.\n\n", + "content_sha256": "5f6404b4b7e127c40a28941b02c473bb20d8759aef9a9e50c5baacec7a12f53a", + "decode_ms": 2140.8, + "decode_tokens_per_sec": 25.2, + "expected_call": { + "arguments": { + "customer_ref": "teal" + }, + "name": "list_open_orders" + }, + "exposed_tool_wait_ms": 2026.7217100008565, + "model_compute_ms": 6317.6, + "prediction": null, + "prediction_hit": false, + "prediction_reason": null, + "prediction_status": null, + "predictor_wall_ms": 0.0, + "prefill_ms": 4176.8, + "prefilled_tokens": 94, + "private_result_exposed_on_miss": false, + "request_wall_ms": 6337.450651000836, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": null, + "tool_call_id": "call_9de33f3a61b9f1b82be801f2", + "tool_result": { + "call_ref": "maple", + "call_sha256": "5f7c0536c4a387c5334ef97c8834cd3472841f24b358d7de06412ab784c031f6", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.1781250030035, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "list_open_orders" + }, + "tool_wall_ms": 2026.7217100008565 + }, + { + "accept_rate": 0.7115384340286255, + "assistant_message": { + "content": "Now I proceed to step 3: get_order_details using the orders_ref from the previous result.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"orders_ref\":\"maple\"}", + "name": "get_order_details" + }, + "id": "call_c389a4a304a4855583d096e3", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 835, + "call": { + "arguments": { + "orders_ref": "maple" + }, + "name": "get_order_details" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"orders_ref\":\"maple\"},\"name\":\"get_order_details\"}", + "completion_tokens": 50, + "content": "Now I proceed to step 3: get_order_details using the orders_ref from the previous result.\n\n", + "content_sha256": "bc190728cc301be15a35259dc01ce8c5880575dc6bfcc34e09e1382ec03aa00b", + "decode_ms": 2395.8, + "decode_tokens_per_sec": 20.9, + "expected_call": { + "arguments": { + "orders_ref": "maple" + }, + "name": "get_order_details" + }, + "exposed_tool_wait_ms": 2027.0823430000746, + "model_compute_ms": 6656.7, + "prediction": null, + "prediction_hit": false, + "prediction_reason": null, + "prediction_status": null, + "predictor_wall_ms": 0.0, + "prefill_ms": 4260.9, + "prefilled_tokens": 99, + "private_result_exposed_on_miss": false, + "request_wall_ms": 6677.666080002382, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": null, + "tool_call_id": "call_c389a4a304a4855583d096e3", + "tool_result": { + "call_ref": "teal", + "call_sha256": "b390ab8cab21b6aa9ac895e1c383fbeac5835b056f6158ee5bac5384eee6c90c", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.0883639986569, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "get_order_details" + }, + "tool_wall_ms": 2027.0823430000746 + }, + { + "accept_rate": 0.828125, + "assistant_message": { + "content": "Now I proceed to step 4: calculate_shipping using the order_ref from the previous result and the destination from the user request.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"destination\":\"Milan\",\"order_ref\":\"teal\"}", + "name": "calculate_shipping" + }, + "id": "call_0ee1078a3640325878166687", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 835, + "call": { + "arguments": { + "destination": "Milan", + "order_ref": "teal" + }, + "name": "calculate_shipping" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"destination\":\"Milan\",\"order_ref\":\"teal\"},\"name\":\"calculate_shipping\"}", + "completion_tokens": 69, + "content": "Now I proceed to step 4: calculate_shipping using the order_ref from the previous result and the destination from the user request.\n\n", + "content_sha256": "eeddf10343ca1f79fbf9009b3ce6ecb712a9d084d51a516f0d84e085b8e69f56", + "decode_ms": 2662.8, + "decode_tokens_per_sec": 25.9, + "expected_call": { + "arguments": { + "destination": "Milan", + "order_ref": "teal" + }, + "name": "calculate_shipping" + }, + "exposed_tool_wait_ms": 2025.8127610031806, + "model_compute_ms": 12869.3, + "prediction": null, + "prediction_hit": false, + "prediction_reason": null, + "prediction_status": null, + "predictor_wall_ms": 0.0, + "prefill_ms": 10206.5, + "prefilled_tokens": 240, + "private_result_exposed_on_miss": false, + "request_wall_ms": 12891.017510002712, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": null, + "tool_call_id": "call_0ee1078a3640325878166687", + "tool_result": { + "call_ref": "jade", + "call_sha256": "75ff43afae4473f01580bb761f28d2e8152af72cb46c29ead46cd8817feeddf7", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.1382260015816, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "calculate_shipping" + }, + "tool_wall_ms": 2025.8127610031806 + } + ], + "task_id": "dependent_workflow_001", + "task_ms": 61406.28354300134 + }, + "pair_index": 1, + "speculative": { + "all_calls_correct": true, + "all_ds4_active": true, + "all_private_miss_results_hidden": true, + "call_count": 4, + "completion_tokens": 239, + "decode_ms": 10295.7, + "expected_final": "workflow_complete:jade", + "exposed_tool_wait_ms": 0.089957, + "final": { + "accept_rate": 0.75, + "assistant_message": { + "content": "workflow_complete:jade", + "role": "assistant" + }, + "cache_hit": true, + "cached_prefix_tokens": 578, + "call": null, + "canonical_call": null, + "completion_tokens": 8, + "content": "workflow_complete:jade", + "content_sha256": "0abc4136c23738770edf0aed5459816de88d70c9b5011c2e5359bdb52557bf42", + "decode_ms": 446.5, + "decode_tokens_per_sec": 17.9, + "model_compute_ms": 29053.9, + "prefill_ms": 28607.4, + "prefilled_tokens": 673, + "request_wall_ms": 29070.463013002154, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": null, + "tool_call_id": "" + }, + "final_correct": true, + "model_compute_ms": 57444.0, + "prediction_hits": 4, + "predictor_wall_ms": 1266.8827410000001, + "steps": [ + { + "accept_rate": 0.7678571343421936, + "assistant_message": { + "content": "I need to start with step 1: resolve_customer using the customer email from the request.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"customer_email\":\"agent-benchmark-1@example.test\"}", + "name": "resolve_customer" + }, + "id": "call_fecf851a9599a5ae4934a551", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 627, + "call": { + "arguments": { + "customer_email": "agent-benchmark-1@example.test" + }, + "name": "resolve_customer" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"customer_email\":\"agent-benchmark-1@example.test\"},\"name\":\"resolve_customer\"}", + "completion_tokens": 58, + "content": "I need to start with step 1: resolve_customer using the customer email from the request.\n\n", + "content_sha256": "9eb32ae482f67f3e7c0fd5f011a2c9c7fb3e76804c484bdf6b40bf8add1aa381", + "decode_ms": 2653.3, + "decode_tokens_per_sec": 21.9, + "expected_call": { + "arguments": { + "customer_email": "agent-benchmark-1@example.test" + }, + "name": "resolve_customer" + }, + "exposed_tool_wait_ms": 0.019887, + "model_compute_ms": 2653.3, + "prediction": { + "arguments": { + "customer_email": "agent-benchmark-1@example.test" + }, + "name": "resolve_customer" + }, + "prediction_hit": true, + "prediction_reason": null, + "prediction_status": "hit", + "predictor_wall_ms": 290.616952, + "prefill_ms": 0.0, + "prefilled_tokens": 0, + "private_result_exposed_on_miss": false, + "request_wall_ms": 2958.456275999197, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_fecf851a9599a5ae4934a551", + "commit_signal_sent": false, + "commit_wait_ms": 0.019887, + "confidence": 0.75, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 2665.994315, + "expected_speedup": 1.5428733045594794, + "prediction": { + "arguments": { + "customer_email": "agent-benchmark-1@example.test" + }, + "name": "resolve_customer" + }, + "prediction_source": "qwen3-0.6b", + "predictor_wall_ms": 290.616952, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "call_ref": "teal", + "call_sha256": "c42842bc1ac50a52be75d0524ab1a3ac7acd920b2f5cd44f26c522edb56075b7", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.2695100010897, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "resolve_customer" + }, + "status": "hit" + }, + "tool_call_id": "call_fecf851a9599a5ae4934a551", + "tool_result": { + "call_ref": "teal", + "call_sha256": "c42842bc1ac50a52be75d0524ab1a3ac7acd920b2f5cd44f26c522edb56075b7", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.2695100010897, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "resolve_customer" + }, + "tool_wall_ms": 2665.994315 + }, + { + "accept_rate": 0.7884615659713745, + "assistant_message": { + "content": "Now I need to proceed with step 2: list_open_orders using the customer_ref from the previous result.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"customer_ref\":\"teal\"}", + "name": "list_open_orders" + }, + "id": "call_bacb659055ea0a6988c7a8eb", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 686, + "call": { + "arguments": { + "customer_ref": "teal" + }, + "name": "list_open_orders" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"customer_ref\":\"teal\"},\"name\":\"list_open_orders\"}", + "completion_tokens": 54, + "content": "Now I need to proceed with step 2: list_open_orders using the customer_ref from the previous result.\n\n", + "content_sha256": "5f6404b4b7e127c40a28941b02c473bb20d8759aef9a9e50c5baacec7a12f53a", + "decode_ms": 2146.2, + "decode_tokens_per_sec": 25.2, + "expected_call": { + "arguments": { + "customer_ref": "teal" + }, + "name": "list_open_orders" + }, + "exposed_tool_wait_ms": 0.020778, + "model_compute_ms": 6288.3, + "prediction": { + "arguments": { + "customer_ref": "teal" + }, + "name": "list_open_orders" + }, + "prediction_hit": true, + "prediction_reason": null, + "prediction_status": "hit", + "predictor_wall_ms": 283.382872, + "prefill_ms": 4142.1, + "prefilled_tokens": 94, + "private_result_exposed_on_miss": false, + "request_wall_ms": 6587.413663000916, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_bacb659055ea0a6988c7a8eb", + "commit_signal_sent": false, + "commit_wait_ms": 0.020778, + "confidence": 0.75, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 6301.160629, + "expected_speedup": 1.5428733045594794, + "prediction": { + "arguments": { + "customer_ref": "teal" + }, + "name": "list_open_orders" + }, + "prediction_source": "qwen3-0.6b", + "predictor_wall_ms": 283.382872, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "call_ref": "maple", + "call_sha256": "5f7c0536c4a387c5334ef97c8834cd3472841f24b358d7de06412ab784c031f6", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.1142179971794, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "list_open_orders" + }, + "status": "hit" + }, + "tool_call_id": "call_bacb659055ea0a6988c7a8eb", + "tool_result": { + "call_ref": "maple", + "call_sha256": "5f7c0536c4a387c5334ef97c8834cd3472841f24b358d7de06412ab784c031f6", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.1142179971794, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "list_open_orders" + }, + "tool_wall_ms": 6301.160629 + }, + { + "accept_rate": 0.7115384340286255, + "assistant_message": { + "content": "Now I proceed to step 3: get_order_details using the orders_ref from the previous result.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"orders_ref\":\"maple\"}", + "name": "get_order_details" + }, + "id": "call_6d5d5875cd96a358f7046361", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 835, + "call": { + "arguments": { + "orders_ref": "maple" + }, + "name": "get_order_details" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"orders_ref\":\"maple\"},\"name\":\"get_order_details\"}", + "completion_tokens": 50, + "content": "Now I proceed to step 3: get_order_details using the orders_ref from the previous result.\n\n", + "content_sha256": "bc190728cc301be15a35259dc01ce8c5880575dc6bfcc34e09e1382ec03aa00b", + "decode_ms": 2393.8, + "decode_tokens_per_sec": 20.9, + "expected_call": { + "arguments": { + "orders_ref": "maple" + }, + "name": "get_order_details" + }, + "exposed_tool_wait_ms": 0.019486, + "model_compute_ms": 6621.3, + "prediction": { + "arguments": { + "orders_ref": "maple" + }, + "name": "get_order_details" + }, + "prediction_hit": true, + "prediction_reason": null, + "prediction_status": "hit", + "predictor_wall_ms": 314.519747, + "prefill_ms": 4227.5, + "prefilled_tokens": 99, + "private_result_exposed_on_miss": false, + "request_wall_ms": 6952.102143000957, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_6d5d5875cd96a358f7046361", + "commit_signal_sent": false, + "commit_wait_ms": 0.019486, + "confidence": 0.75, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 6634.442859, + "expected_speedup": 1.5428733045594794, + "prediction": { + "arguments": { + "orders_ref": "maple" + }, + "name": "get_order_details" + }, + "prediction_source": "qwen3-0.6b", + "predictor_wall_ms": 314.519747, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "call_ref": "teal", + "call_sha256": "b390ab8cab21b6aa9ac895e1c383fbeac5835b056f6158ee5bac5384eee6c90c", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.0938069970289, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "get_order_details" + }, + "status": "hit" + }, + "tool_call_id": "call_6d5d5875cd96a358f7046361", + "tool_result": { + "call_ref": "teal", + "call_sha256": "b390ab8cab21b6aa9ac895e1c383fbeac5835b056f6158ee5bac5384eee6c90c", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.0938069970289, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "get_order_details" + }, + "tool_wall_ms": 6634.442859 + }, + { + "accept_rate": 0.828125, + "assistant_message": { + "content": "Now I proceed to step 4: calculate_shipping using the order_ref from the previous result and the destination from the user request.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"destination\":\"Milan\",\"order_ref\":\"teal\"}", + "name": "calculate_shipping" + }, + "id": "call_e465f52877b49a434aa95da7", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 835, + "call": { + "arguments": { + "destination": "Milan", + "order_ref": "teal" + }, + "name": "calculate_shipping" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"destination\":\"Milan\",\"order_ref\":\"teal\"},\"name\":\"calculate_shipping\"}", + "completion_tokens": 69, + "content": "Now I proceed to step 4: calculate_shipping using the order_ref from the previous result and the destination from the user request.\n\n", + "content_sha256": "eeddf10343ca1f79fbf9009b3ce6ecb712a9d084d51a516f0d84e085b8e69f56", + "decode_ms": 2655.9, + "decode_tokens_per_sec": 26.0, + "expected_call": { + "arguments": { + "destination": "Milan", + "order_ref": "teal" + }, + "name": "calculate_shipping" + }, + "exposed_tool_wait_ms": 0.029806, + "model_compute_ms": 12827.199999999999, + "prediction": { + "arguments": { + "destination": "Milan", + "order_ref": "teal" + }, + "name": "calculate_shipping" + }, + "prediction_hit": true, + "prediction_reason": null, + "prediction_status": "hit", + "predictor_wall_ms": 378.36317, + "prefill_ms": 10171.3, + "prefilled_tokens": 240, + "private_result_exposed_on_miss": false, + "request_wall_ms": 13222.511880001548, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_e465f52877b49a434aa95da7", + "commit_signal_sent": false, + "commit_wait_ms": 0.029806, + "confidence": 0.75, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 12840.559652, + "expected_speedup": 1.5428733045594794, + "prediction": { + "arguments": { + "destination": "Milan", + "order_ref": "teal" + }, + "name": "calculate_shipping" + }, + "prediction_source": "qwen3-0.6b", + "predictor_wall_ms": 378.36317, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "call_ref": "jade", + "call_sha256": "75ff43afae4473f01580bb761f28d2e8152af72cb46c29ead46cd8817feeddf7", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.1044350028678, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "calculate_shipping" + }, + "status": "hit" + }, + "tool_call_id": "call_e465f52877b49a434aa95da7", + "tool_result": { + "call_ref": "jade", + "call_sha256": "75ff43afae4473f01580bb761f28d2e8152af72cb46c29ead46cd8817feeddf7", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.1044350028678, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "calculate_shipping" + }, + "tool_wall_ms": 12840.559652 + } + ], + "task_id": "dependent_workflow_001", + "task_ms": 58791.965293999965 + }, + "task": { + "call_count": 4, + "customer_email": "agent-benchmark-1@example.test", + "destination": "Milan", + "id": "dependent_workflow_001" + } + }, + { + "arm_order": [ + "speculative", + "control" + ], + "control": { + "all_calls_correct": true, + "all_ds4_active": true, + "all_private_miss_results_hidden": true, + "call_count": 5, + "completion_tokens": 250, + "decode_ms": 11595.7, + "expected_final": "workflow_complete:ivory", + "exposed_tool_wait_ms": 10131.762909997633, + "final": { + "accept_rate": 0.75, + "assistant_message": { + "content": "workflow_complete:ivory", + "role": "assistant" + }, + "cache_hit": true, + "cached_prefix_tokens": 714, + "call": null, + "canonical_call": null, + "completion_tokens": 8, + "content": "workflow_complete:ivory", + "content_sha256": "0496080db67bb468d83025f09f59e8cd14c425908fde8632a3b9ba98c23dc9cf", + "decode_ms": 451.5, + "decode_tokens_per_sec": 17.7, + "model_compute_ms": 29204.3, + "prefill_ms": 28752.8, + "prefilled_tokens": 673, + "request_wall_ms": 29227.80936699928, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": null, + "tool_call_id": "" + }, + "final_correct": true, + "model_compute_ms": 63055.1, + "prediction_hits": 0, + "predictor_wall_ms": 0.0, + "steps": [ + { + "accept_rate": 0.625, + "assistant_message": { + "content": "I need to process this customer request through the 5-step workflow. Let me start with step 1: resolving the customer.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"customer_email\":\"agent-benchmark-2@example.test\"}", + "name": "resolve_customer" + }, + "id": "call_69cd1757544b2a9bd324f10a", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 648, + "call": { + "arguments": { + "customer_email": "agent-benchmark-2@example.test" + }, + "name": "resolve_customer" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"customer_email\":\"agent-benchmark-2@example.test\"},\"name\":\"resolve_customer\"}", + "completion_tokens": 64, + "content": "I need to process this customer request through the 5-step workflow. Let me start with step 1: resolving the customer.\n\n", + "content_sha256": "caac2c887d5589ba57ca98c8c77ea68cdd12f4569133cc824cdc1cc82c831c09", + "decode_ms": 3060.1, + "decode_tokens_per_sec": 20.9, + "expected_call": { + "arguments": { + "customer_email": "agent-benchmark-2@example.test" + }, + "name": "resolve_customer" + }, + "exposed_tool_wait_ms": 2026.8547260020569, + "model_compute_ms": 3060.1, + "prediction": null, + "prediction_hit": false, + "prediction_reason": null, + "prediction_status": null, + "predictor_wall_ms": 0.0, + "prefill_ms": 0.0, + "prefilled_tokens": 0, + "private_result_exposed_on_miss": false, + "request_wall_ms": 3073.1119080010103, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": null, + "tool_call_id": "call_69cd1757544b2a9bd324f10a", + "tool_result": { + "call_ref": "plum", + "call_sha256": "93824d1968f2c3ab058a3ef69d2625c0fea88fad3b27b8c30f95418fb83ae6cf", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.3111229998467, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "resolve_customer" + }, + "tool_wall_ms": 2026.8547260020569 + }, + { + "accept_rate": 0.7272727489471436, + "assistant_message": { + "content": "Now I need to list open orders for this resolved customer.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"customer_ref\":\"plum\"}", + "name": "list_open_orders" + }, + "id": "call_50c8ce688d2080917a63f3d4", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 713, + "call": { + "arguments": { + "customer_ref": "plum" + }, + "name": "list_open_orders" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"customer_ref\":\"plum\"},\"name\":\"list_open_orders\"}", + "completion_tokens": 43, + "content": "Now I need to list open orders for this resolved customer.\n\n", + "content_sha256": "3ce7d0f18b723f0f13b497c10e5799d52d899e25e977e82e22a5ba8e25b52e6e", + "decode_ms": 2100.4, + "decode_tokens_per_sec": 20.5, + "expected_call": { + "arguments": { + "customer_ref": "plum" + }, + "name": "list_open_orders" + }, + "exposed_tool_wait_ms": 2025.9938390008756, + "model_compute_ms": 6204.1, + "prediction": null, + "prediction_hit": false, + "prediction_reason": null, + "prediction_status": null, + "predictor_wall_ms": 0.0, + "prefill_ms": 4103.7, + "prefilled_tokens": 95, + "private_result_exposed_on_miss": false, + "request_wall_ms": 6224.153801998909, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": null, + "tool_call_id": "call_50c8ce688d2080917a63f3d4", + "tool_result": { + "call_ref": "jade", + "call_sha256": "772a4b9fde9e69fb1da323ba4c22e7a7d01061c0600c133e74de7bd73bba931f", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.209031997656, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "list_open_orders" + }, + "tool_wall_ms": 2025.9938390008756 + }, + { + "accept_rate": 0.6363636255264282, + "assistant_message": { + "content": "Now I need to inspect the selected order.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"orders_ref\":\"jade\"}", + "name": "get_order_details" + }, + "id": "call_deb21d13f2d6efbc6f1b841f", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 852, + "call": { + "arguments": { + "orders_ref": "jade" + }, + "name": "get_order_details" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"orders_ref\":\"jade\"},\"name\":\"get_order_details\"}", + "completion_tokens": 39, + "content": "Now I need to inspect the selected order.\n\n", + "content_sha256": "0501df1f5ec6fcf19c50a853fe0631d19fcc4dac651569fc7278ecc84ffcdb0c", + "decode_ms": 2273.4, + "decode_tokens_per_sec": 17.2, + "expected_call": { + "arguments": { + "orders_ref": "jade" + }, + "name": "get_order_details" + }, + "exposed_tool_wait_ms": 2026.1670279978716, + "model_compute_ms": 6370.5, + "prediction": null, + "prediction_hit": false, + "prediction_reason": null, + "prediction_status": null, + "predictor_wall_ms": 0.0, + "prefill_ms": 4097.1, + "prefilled_tokens": 95, + "private_result_exposed_on_miss": false, + "request_wall_ms": 6392.090148001444, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": null, + "tool_call_id": "call_deb21d13f2d6efbc6f1b841f", + "tool_result": { + "call_ref": "amber", + "call_sha256": "428dbfc05f17e7a9295c2b4741dbae2bb91d3f107150b77bb34c6ca25bd8af23", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.138072999107, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "get_order_details" + }, + "tool_wall_ms": 2026.1670279978716 + }, + { + "accept_rate": 0.7692307829856873, + "assistant_message": { + "content": "Now I need to calculate shipping for the order to Turin.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"destination\":\"Turin\",\"order_ref\":\"amber\"}", + "name": "calculate_shipping" + }, + "id": "call_2854e0d7420f2516ae707775", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 987, + "call": { + "arguments": { + "destination": "Turin", + "order_ref": "amber" + }, + "name": "calculate_shipping" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"destination\":\"Turin\",\"order_ref\":\"amber\"},\"name\":\"calculate_shipping\"}", + "completion_tokens": 53, + "content": "Now I need to calculate shipping for the order to Turin.\n\n", + "content_sha256": "069cd393e5fe66dd10d07d7f7927ada35244e0a01f87cb5e75c95a67b736d0ad", + "decode_ms": 2405.6, + "decode_tokens_per_sec": 22.0, + "expected_call": { + "arguments": { + "destination": "Turin", + "order_ref": "amber" + }, + "name": "calculate_shipping" + }, + "exposed_tool_wait_ms": 2026.3736019987846, + "model_compute_ms": 6388.7, + "prediction": null, + "prediction_hit": false, + "prediction_reason": null, + "prediction_status": null, + "predictor_wall_ms": 0.0, + "prefill_ms": 3983.1, + "prefilled_tokens": 93, + "private_result_exposed_on_miss": false, + "request_wall_ms": 6411.000621999847, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": null, + "tool_call_id": "call_2854e0d7420f2516ae707775", + "tool_result": { + "call_ref": "amber", + "call_sha256": "21ec3c60e83bfbc0ab4e6e8356a5c87e7210ffd16b53ca8299017261d97f70a1", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.345896001818, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "calculate_shipping" + }, + "tool_wall_ms": 2026.3736019987846 + }, + { + "accept_rate": 0.9444444179534912, + "assistant_message": { + "content": "Now I need to prepare the final customer summary.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"shipping_ref\":\"amber\"}", + "name": "prepare_customer_summary" + }, + "id": "call_b8783a11222c4962dd41d911", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 987, + "call": { + "arguments": { + "shipping_ref": "amber" + }, + "name": "prepare_customer_summary" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"shipping_ref\":\"amber\"},\"name\":\"prepare_customer_summary\"}", + "completion_tokens": 43, + "content": "Now I need to prepare the final customer summary.\n\n", + "content_sha256": "6d12faaaf38ef53fc446a8a34f8fe0f3a22a7bdc6d3dab6ef4f1f25c5ba91b44", + "decode_ms": 1304.7, + "decode_tokens_per_sec": 33.0, + "expected_call": { + "arguments": { + "shipping_ref": "amber" + }, + "name": "prepare_customer_summary" + }, + "exposed_tool_wait_ms": 2026.3737149980443, + "model_compute_ms": 11827.400000000001, + "prediction": null, + "prediction_hit": false, + "prediction_reason": null, + "prediction_status": null, + "predictor_wall_ms": 0.0, + "prefill_ms": 10522.7, + "prefilled_tokens": 242, + "private_result_exposed_on_miss": false, + "request_wall_ms": 11850.366531998588, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": null, + "tool_call_id": "call_b8783a11222c4962dd41d911", + "tool_result": { + "call_ref": "ivory", + "call_sha256": "adaa761561a84b7e457acb7881e856285fcdccf21a6d82751dd2bea984b1d6d0", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.1452489996154, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "prepare_customer_summary" + }, + "tool_wall_ms": 2026.3737149980443 + } + ], + "task_id": "dependent_workflow_002", + "task_ms": 73312.62574300126 + }, + "pair_index": 2, + "speculative": { + "all_calls_correct": true, + "all_ds4_active": true, + "all_private_miss_results_hidden": true, + "call_count": 5, + "completion_tokens": 250, + "decode_ms": 11593.5, + "expected_final": "workflow_complete:ivory", + "exposed_tool_wait_ms": 0.125415, + "final": { + "accept_rate": 0.75, + "assistant_message": { + "content": "workflow_complete:ivory", + "role": "assistant" + }, + "cache_hit": true, + "cached_prefix_tokens": 603, + "call": null, + "canonical_call": null, + "completion_tokens": 8, + "content": "workflow_complete:ivory", + "content_sha256": "0496080db67bb468d83025f09f59e8cd14c425908fde8632a3b9ba98c23dc9cf", + "decode_ms": 457.7, + "decode_tokens_per_sec": 17.5, + "model_compute_ms": 33725.1, + "prefill_ms": 33267.4, + "prefilled_tokens": 784, + "request_wall_ms": 33742.18803799886, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": null, + "tool_call_id": "" + }, + "final_correct": true, + "model_compute_ms": 67450.6, + "prediction_hits": 5, + "predictor_wall_ms": 1638.720663, + "steps": [ + { + "accept_rate": 0.625, + "assistant_message": { + "content": "I need to process this customer request through the 5-step workflow. Let me start with step 1: resolving the customer.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"customer_email\":\"agent-benchmark-2@example.test\"}", + "name": "resolve_customer" + }, + "id": "call_bec28a4286f77110bf0d49e5", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 648, + "call": { + "arguments": { + "customer_email": "agent-benchmark-2@example.test" + }, + "name": "resolve_customer" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"customer_email\":\"agent-benchmark-2@example.test\"},\"name\":\"resolve_customer\"}", + "completion_tokens": 64, + "content": "I need to process this customer request through the 5-step workflow. Let me start with step 1: resolving the customer.\n\n", + "content_sha256": "caac2c887d5589ba57ca98c8c77ea68cdd12f4569133cc824cdc1cc82c831c09", + "decode_ms": 3061.5, + "decode_tokens_per_sec": 20.9, + "expected_call": { + "arguments": { + "customer_email": "agent-benchmark-2@example.test" + }, + "name": "resolve_customer" + }, + "exposed_tool_wait_ms": 0.02094, + "model_compute_ms": 3061.5, + "prediction": { + "arguments": { + "customer_email": "agent-benchmark-2@example.test" + }, + "name": "resolve_customer" + }, + "prediction_hit": true, + "prediction_reason": null, + "prediction_status": "hit", + "predictor_wall_ms": 295.452857, + "prefill_ms": 0.0, + "prefilled_tokens": 0, + "private_result_exposed_on_miss": false, + "request_wall_ms": 3372.0264969997515, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_bec28a4286f77110bf0d49e5", + "commit_signal_sent": false, + "commit_wait_ms": 0.02094, + "confidence": 0.75, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 3074.451431, + "expected_speedup": 1.5428733045594794, + "prediction": { + "arguments": { + "customer_email": "agent-benchmark-2@example.test" + }, + "name": "resolve_customer" + }, + "prediction_source": "qwen3-0.6b", + "predictor_wall_ms": 295.452857, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "call_ref": "plum", + "call_sha256": "93824d1968f2c3ab058a3ef69d2625c0fea88fad3b27b8c30f95418fb83ae6cf", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.1503530002083, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "resolve_customer" + }, + "status": "hit" + }, + "tool_call_id": "call_bec28a4286f77110bf0d49e5", + "tool_result": { + "call_ref": "plum", + "call_sha256": "93824d1968f2c3ab058a3ef69d2625c0fea88fad3b27b8c30f95418fb83ae6cf", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.1503530002083, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "resolve_customer" + }, + "tool_wall_ms": 3074.451431 + }, + { + "accept_rate": 0.7272727489471436, + "assistant_message": { + "content": "Now I need to list open orders for this resolved customer.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"customer_ref\":\"plum\"}", + "name": "list_open_orders" + }, + "id": "call_01a8a2497c9c63207b596c32", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 713, + "call": { + "arguments": { + "customer_ref": "plum" + }, + "name": "list_open_orders" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"customer_ref\":\"plum\"},\"name\":\"list_open_orders\"}", + "completion_tokens": 43, + "content": "Now I need to list open orders for this resolved customer.\n\n", + "content_sha256": "3ce7d0f18b723f0f13b497c10e5799d52d899e25e977e82e22a5ba8e25b52e6e", + "decode_ms": 2098.9, + "decode_tokens_per_sec": 20.5, + "expected_call": { + "arguments": { + "customer_ref": "plum" + }, + "name": "list_open_orders" + }, + "exposed_tool_wait_ms": 0.030296, + "model_compute_ms": 6174.5, + "prediction": { + "arguments": { + "customer_ref": "plum" + }, + "name": "list_open_orders" + }, + "prediction_hit": true, + "prediction_reason": null, + "prediction_status": "hit", + "predictor_wall_ms": 287.479375, + "prefill_ms": 4075.6, + "prefilled_tokens": 95, + "private_result_exposed_on_miss": false, + "request_wall_ms": 6477.406130001327, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_01a8a2497c9c63207b596c32", + "commit_signal_sent": false, + "commit_wait_ms": 0.030296, + "confidence": 0.75, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 6187.553086, + "expected_speedup": 1.5428733045594794, + "prediction": { + "arguments": { + "customer_ref": "plum" + }, + "name": "list_open_orders" + }, + "prediction_source": "qwen3-0.6b", + "predictor_wall_ms": 287.479375, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "call_ref": "jade", + "call_sha256": "772a4b9fde9e69fb1da323ba4c22e7a7d01061c0600c133e74de7bd73bba931f", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.0936220021686, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "list_open_orders" + }, + "status": "hit" + }, + "tool_call_id": "call_01a8a2497c9c63207b596c32", + "tool_result": { + "call_ref": "jade", + "call_sha256": "772a4b9fde9e69fb1da323ba4c22e7a7d01061c0600c133e74de7bd73bba931f", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.0936220021686, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "list_open_orders" + }, + "tool_wall_ms": 6187.553086 + }, + { + "accept_rate": 0.6363636255264282, + "assistant_message": { + "content": "Now I need to inspect the selected order.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"orders_ref\":\"jade\"}", + "name": "get_order_details" + }, + "id": "call_210fd40fb9531732179be43e", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 852, + "call": { + "arguments": { + "orders_ref": "jade" + }, + "name": "get_order_details" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"orders_ref\":\"jade\"},\"name\":\"get_order_details\"}", + "completion_tokens": 39, + "content": "Now I need to inspect the selected order.\n\n", + "content_sha256": "0501df1f5ec6fcf19c50a853fe0631d19fcc4dac651569fc7278ecc84ffcdb0c", + "decode_ms": 2273.1, + "decode_tokens_per_sec": 17.2, + "expected_call": { + "arguments": { + "orders_ref": "jade" + }, + "name": "get_order_details" + }, + "exposed_tool_wait_ms": 0.020418, + "model_compute_ms": 6347.6, + "prediction": { + "arguments": { + "orders_ref": "jade" + }, + "name": "get_order_details" + }, + "prediction_hit": true, + "prediction_reason": null, + "prediction_status": "hit", + "predictor_wall_ms": 314.968808, + "prefill_ms": 4074.5, + "prefilled_tokens": 95, + "private_result_exposed_on_miss": false, + "request_wall_ms": 6678.62192100074, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_210fd40fb9531732179be43e", + "commit_signal_sent": false, + "commit_wait_ms": 0.020418, + "confidence": 0.75, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 6360.937862, + "expected_speedup": 1.5428733045594794, + "prediction": { + "arguments": { + "orders_ref": "jade" + }, + "name": "get_order_details" + }, + "prediction_source": "qwen3-0.6b", + "predictor_wall_ms": 314.968808, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "call_ref": "amber", + "call_sha256": "428dbfc05f17e7a9295c2b4741dbae2bb91d3f107150b77bb34c6ca25bd8af23", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.123739002447, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "get_order_details" + }, + "status": "hit" + }, + "tool_call_id": "call_210fd40fb9531732179be43e", + "tool_result": { + "call_ref": "amber", + "call_sha256": "428dbfc05f17e7a9295c2b4741dbae2bb91d3f107150b77bb34c6ca25bd8af23", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.123739002447, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "get_order_details" + }, + "tool_wall_ms": 6360.937862 + }, + { + "accept_rate": 0.7692307829856873, + "assistant_message": { + "content": "Now I need to calculate shipping for the order to Turin.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"destination\":\"Turin\",\"order_ref\":\"amber\"}", + "name": "calculate_shipping" + }, + "id": "call_b3b83d71f87f78b979030f5b", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 987, + "call": { + "arguments": { + "destination": "Turin", + "order_ref": "amber" + }, + "name": "calculate_shipping" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"destination\":\"Turin\",\"order_ref\":\"amber\"},\"name\":\"calculate_shipping\"}", + "completion_tokens": 53, + "content": "Now I need to calculate shipping for the order to Turin.\n\n", + "content_sha256": "069cd393e5fe66dd10d07d7f7927ada35244e0a01f87cb5e75c95a67b736d0ad", + "decode_ms": 2398.6, + "decode_tokens_per_sec": 22.1, + "expected_call": { + "arguments": { + "destination": "Turin", + "order_ref": "amber" + }, + "name": "calculate_shipping" + }, + "exposed_tool_wait_ms": 0.019717, + "model_compute_ms": 6368.7, + "prediction": { + "arguments": { + "destination": "Turin", + "order_ref": "amber" + }, + "name": "calculate_shipping" + }, + "prediction_hit": true, + "prediction_reason": null, + "prediction_status": "hit", + "predictor_wall_ms": 371.829653, + "prefill_ms": 3970.1, + "prefilled_tokens": 93, + "private_result_exposed_on_miss": false, + "request_wall_ms": 6757.145636998757, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_b3b83d71f87f78b979030f5b", + "commit_signal_sent": false, + "commit_wait_ms": 0.019717, + "confidence": 0.75, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 6382.563455, + "expected_speedup": 1.5428733045594794, + "prediction": { + "arguments": { + "destination": "Turin", + "order_ref": "amber" + }, + "name": "calculate_shipping" + }, + "prediction_source": "qwen3-0.6b", + "predictor_wall_ms": 371.829653, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "call_ref": "amber", + "call_sha256": "21ec3c60e83bfbc0ab4e6e8356a5c87e7210ffd16b53ca8299017261d97f70a1", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.1194319993374, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "calculate_shipping" + }, + "status": "hit" + }, + "tool_call_id": "call_b3b83d71f87f78b979030f5b", + "tool_result": { + "call_ref": "amber", + "call_sha256": "21ec3c60e83bfbc0ab4e6e8356a5c87e7210ffd16b53ca8299017261d97f70a1", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.1194319993374, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "calculate_shipping" + }, + "tool_wall_ms": 6382.563455 + }, + { + "accept_rate": 0.9444444179534912, + "assistant_message": { + "content": "Now I need to prepare the final customer summary.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"shipping_ref\":\"amber\"}", + "name": "prepare_customer_summary" + }, + "id": "call_832c53b5fcceba769236c378", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 987, + "call": { + "arguments": { + "shipping_ref": "amber" + }, + "name": "prepare_customer_summary" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"shipping_ref\":\"amber\"},\"name\":\"prepare_customer_summary\"}", + "completion_tokens": 43, + "content": "Now I need to prepare the final customer summary.\n\n", + "content_sha256": "6d12faaaf38ef53fc446a8a34f8fe0f3a22a7bdc6d3dab6ef4f1f25c5ba91b44", + "decode_ms": 1303.7, + "decode_tokens_per_sec": 33.0, + "expected_call": { + "arguments": { + "shipping_ref": "amber" + }, + "name": "prepare_customer_summary" + }, + "exposed_tool_wait_ms": 0.034044, + "model_compute_ms": 11773.2, + "prediction": { + "arguments": { + "shipping_ref": "amber" + }, + "name": "prepare_customer_summary" + }, + "prediction_hit": true, + "prediction_reason": null, + "prediction_status": "hit", + "predictor_wall_ms": 368.98997, + "prefill_ms": 10469.5, + "prefilled_tokens": 242, + "private_result_exposed_on_miss": false, + "request_wall_ms": 12160.27835999921, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_832c53b5fcceba769236c378", + "commit_signal_sent": false, + "commit_wait_ms": 0.034044, + "confidence": 0.75, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 11787.198031, + "expected_speedup": 1.5428733045594794, + "prediction": { + "arguments": { + "shipping_ref": "amber" + }, + "name": "prepare_customer_summary" + }, + "prediction_source": "qwen3-0.6b", + "predictor_wall_ms": 368.98997, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "call_ref": "ivory", + "call_sha256": "adaa761561a84b7e457acb7881e856285fcdccf21a6d82751dd2bea984b1d6d0", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.1132219986175, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "prepare_customer_summary" + }, + "status": "hit" + }, + "tool_call_id": "call_832c53b5fcceba769236c378", + "tool_result": { + "call_ref": "ivory", + "call_sha256": "adaa761561a84b7e457acb7881e856285fcdccf21a6d82751dd2bea984b1d6d0", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.1132219986175, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "prepare_customer_summary" + }, + "tool_wall_ms": 11787.198031 + } + ], + "task_id": "dependent_workflow_002", + "task_ms": 69188.68444200052 + }, + "task": { + "call_count": 5, + "customer_email": "agent-benchmark-2@example.test", + "destination": "Turin", + "id": "dependent_workflow_002" + } + }, + { + "arm_order": [ + "speculative", + "control" + ], + "control": { + "all_calls_correct": true, + "all_ds4_active": true, + "all_private_miss_results_hidden": true, + "call_count": 3, + "completion_tokens": 140, + "decode_ms": 8523.9, + "expected_final": "workflow_complete:amber", + "exposed_tool_wait_ms": 6080.720226997073, + "final": { + "accept_rate": 0.625, + "assistant_message": { + "content": "workflow_complete:amber", + "role": "assistant" + }, + "cache_hit": true, + "cached_prefix_tokens": 791, + "call": null, + "canonical_call": null, + "completion_tokens": 7, + "content": "workflow_complete:amber", + "content_sha256": "432bcc6fd099a0a7a97da30984ac9e2d3760ca9fc31f5b74162f2961c04c83b8", + "decode_ms": 440.8, + "decode_tokens_per_sec": 15.9, + "model_compute_ms": 10922.3, + "prefill_ms": 10481.5, + "prefilled_tokens": 241, + "request_wall_ms": 10943.100953001704, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": null, + "tool_call_id": "" + }, + "final_correct": true, + "model_compute_ms": 32427.3, + "prediction_hits": 0, + "predictor_wall_ms": 0.0, + "steps": [ + { + "accept_rate": 0.59375, + "assistant_message": { + "content": "I need to process the customer for destination Bologna. Let me start by resolving the customer.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"customer_email\":\"agent-benchmark-3@example.test\"}", + "name": "resolve_customer" + }, + "id": "call_e984fc331504596163ab27cf", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 602, + "call": { + "arguments": { + "customer_email": "agent-benchmark-3@example.test" + }, + "name": "resolve_customer" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"customer_email\":\"agent-benchmark-3@example.test\"},\"name\":\"resolve_customer\"}", + "completion_tokens": 55, + "content": "I need to process the customer for destination Bologna. Let me start by resolving the customer.\n\n", + "content_sha256": "62c201aea98c566f6e95b6774e46fea965dc0ccd0e7b299ff5e2a3a96a5226d4", + "decode_ms": 3286.9, + "decode_tokens_per_sec": 16.7, + "expected_call": { + "arguments": { + "customer_email": "agent-benchmark-3@example.test" + }, + "name": "resolve_customer" + }, + "exposed_tool_wait_ms": 2026.4642199981608, + "model_compute_ms": 3286.9, + "prediction": null, + "prediction_hit": false, + "prediction_reason": null, + "prediction_status": null, + "predictor_wall_ms": 0.0, + "prefill_ms": 0.0, + "prefilled_tokens": 0, + "private_result_exposed_on_miss": false, + "request_wall_ms": 3299.9924879986793, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": null, + "tool_call_id": "call_e984fc331504596163ab27cf", + "tool_result": { + "call_ref": "azure", + "call_sha256": "102e55312f96c8ec8207cda555ac4117176de2eed12b6b829409dd9ce2f3cecd", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.1172100019176, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "resolve_customer" + }, + "tool_wall_ms": 2026.4642199981608 + }, + { + "accept_rate": 0.5833333134651184, + "assistant_message": { + "content": "Now I need to list open orders for this customer.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"customer_ref\":\"azure\"}", + "name": "list_open_orders" + }, + "id": "call_697fd06608e2005b6913881d", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 658, + "call": { + "arguments": { + "customer_ref": "azure" + }, + "name": "list_open_orders" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"customer_ref\":\"azure\"},\"name\":\"list_open_orders\"}", + "completion_tokens": 40, + "content": "Now I need to list open orders for this customer.\n\n", + "content_sha256": "083d3fef5ad5bbc72a6d4d9dbd01570eb5d080efc3116dfb0e78467825ef2ae0", + "decode_ms": 2335.8, + "decode_tokens_per_sec": 17.1, + "expected_call": { + "arguments": { + "customer_ref": "azure" + }, + "name": "list_open_orders" + }, + "exposed_tool_wait_ms": 2026.8712299985054, + "model_compute_ms": 6218.8, + "prediction": null, + "prediction_hit": false, + "prediction_reason": null, + "prediction_status": null, + "predictor_wall_ms": 0.0, + "prefill_ms": 3883.0, + "prefilled_tokens": 91, + "private_result_exposed_on_miss": false, + "request_wall_ms": 6240.539802998683, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": null, + "tool_call_id": "call_697fd06608e2005b6913881d", + "tool_result": { + "call_ref": "jade", + "call_sha256": "f9bb9bbfe57b8faa80fd5f25ba23eb97ae6031ad8e254049f174e1072d96142f", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.085728999693, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "list_open_orders" + }, + "tool_wall_ms": 2026.8712299985054 + }, + { + "accept_rate": 0.6136363744735718, + "assistant_message": { + "content": "Now I need to get the order details.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"orders_ref\":\"jade\"}", + "name": "get_order_details" + }, + "id": "call_fc74542559f704d7895bd360", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 658, + "call": { + "arguments": { + "orders_ref": "jade" + }, + "name": "get_order_details" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"orders_ref\":\"jade\"},\"name\":\"get_order_details\"}", + "completion_tokens": 38, + "content": "Now I need to get the order details.\n\n", + "content_sha256": "4855a9d3760fe1217b6d0b8d6dd52f84ec1cb58b5a1b797b630e5dc9519a5417", + "decode_ms": 2460.4, + "decode_tokens_per_sec": 15.4, + "expected_call": { + "arguments": { + "orders_ref": "jade" + }, + "name": "get_order_details" + }, + "exposed_tool_wait_ms": 2027.3847770004068, + "model_compute_ms": 11999.3, + "prediction": null, + "prediction_hit": false, + "prediction_reason": null, + "prediction_status": null, + "predictor_wall_ms": 0.0, + "prefill_ms": 9538.9, + "prefilled_tokens": 226, + "private_result_exposed_on_miss": false, + "request_wall_ms": 12019.706646999111, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": null, + "tool_call_id": "call_fc74542559f704d7895bd360", + "tool_result": { + "call_ref": "amber", + "call_sha256": "428dbfc05f17e7a9295c2b4741dbae2bb91d3f107150b77bb34c6ca25bd8af23", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.4602429980878, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "get_order_details" + }, + "tool_wall_ms": 2027.3847770004068 + } + ], + "task_id": "dependent_workflow_003", + "task_ms": 38585.75580899924 + }, + "pair_index": 3, + "speculative": { + "all_calls_correct": true, + "all_ds4_active": true, + "all_private_miss_results_hidden": true, + "call_count": 3, + "completion_tokens": 140, + "decode_ms": 9557.2, + "expected_final": "workflow_complete:amber", + "exposed_tool_wait_ms": 0.071994, + "final": { + "accept_rate": 0.625, + "assistant_message": { + "content": "workflow_complete:amber", + "role": "assistant" + }, + "cache_hit": true, + "cached_prefix_tokens": 659, + "call": null, + "canonical_call": null, + "completion_tokens": 7, + "content": "workflow_complete:amber", + "content_sha256": "432bcc6fd099a0a7a97da30984ac9e2d3760ca9fc31f5b74162f2961c04c83b8", + "decode_ms": 437.2, + "decode_tokens_per_sec": 16.0, + "model_compute_ms": 16659.4, + "prefill_ms": 16222.2, + "prefilled_tokens": 373, + "request_wall_ms": 16674.765046001994, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": null, + "tool_call_id": "" + }, + "final_correct": true, + "model_compute_ms": 39114.100000000006, + "prediction_hits": 3, + "predictor_wall_ms": 858.5581070000001, + "steps": [ + { + "accept_rate": 0.59375, + "assistant_message": { + "content": "I need to process the customer for destination Bologna. Let me start by resolving the customer.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"customer_email\":\"agent-benchmark-3@example.test\"}", + "name": "resolve_customer" + }, + "id": "call_83b7681c10a779328b97cd10", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 602, + "call": { + "arguments": { + "customer_email": "agent-benchmark-3@example.test" + }, + "name": "resolve_customer" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"customer_email\":\"agent-benchmark-3@example.test\"},\"name\":\"resolve_customer\"}", + "completion_tokens": 55, + "content": "I need to process the customer for destination Bologna. Let me start by resolving the customer.\n\n", + "content_sha256": "62c201aea98c566f6e95b6774e46fea965dc0ccd0e7b299ff5e2a3a96a5226d4", + "decode_ms": 4325.2, + "decode_tokens_per_sec": 12.7, + "expected_call": { + "arguments": { + "customer_email": "agent-benchmark-3@example.test" + }, + "name": "resolve_customer" + }, + "exposed_tool_wait_ms": 0.022101, + "model_compute_ms": 4325.2, + "prediction": { + "arguments": { + "customer_email": "agent-benchmark-3@example.test" + }, + "name": "resolve_customer" + }, + "prediction_hit": true, + "prediction_reason": null, + "prediction_status": "hit", + "predictor_wall_ms": 286.040186, + "prefill_ms": 0.0, + "prefilled_tokens": 0, + "private_result_exposed_on_miss": false, + "request_wall_ms": 4625.921421000385, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_83b7681c10a779328b97cd10", + "commit_signal_sent": false, + "commit_wait_ms": 0.022101, + "confidence": 0.75, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 4338.023602, + "expected_speedup": 1.5428733045594794, + "prediction": { + "arguments": { + "customer_email": "agent-benchmark-3@example.test" + }, + "name": "resolve_customer" + }, + "prediction_source": "qwen3-0.6b", + "predictor_wall_ms": 286.040186, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "call_ref": "azure", + "call_sha256": "102e55312f96c8ec8207cda555ac4117176de2eed12b6b829409dd9ce2f3cecd", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.2032990014413, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "resolve_customer" + }, + "status": "hit" + }, + "tool_call_id": "call_83b7681c10a779328b97cd10", + "tool_result": { + "call_ref": "azure", + "call_sha256": "102e55312f96c8ec8207cda555ac4117176de2eed12b6b829409dd9ce2f3cecd", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.2032990014413, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "resolve_customer" + }, + "tool_wall_ms": 4338.023602 + }, + { + "accept_rate": 0.5833333134651184, + "assistant_message": { + "content": "Now I need to list open orders for this customer.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"customer_ref\":\"azure\"}", + "name": "list_open_orders" + }, + "id": "call_750886b39d7cdf0d6340234d", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 658, + "call": { + "arguments": { + "customer_ref": "azure" + }, + "name": "list_open_orders" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"customer_ref\":\"azure\"},\"name\":\"list_open_orders\"}", + "completion_tokens": 40, + "content": "Now I need to list open orders for this customer.\n\n", + "content_sha256": "083d3fef5ad5bbc72a6d4d9dbd01570eb5d080efc3116dfb0e78467825ef2ae0", + "decode_ms": 2334.0, + "decode_tokens_per_sec": 17.1, + "expected_call": { + "arguments": { + "customer_ref": "azure" + }, + "name": "list_open_orders" + }, + "exposed_tool_wait_ms": 0.019967, + "model_compute_ms": 6185.3, + "prediction": { + "arguments": { + "customer_ref": "azure" + }, + "name": "list_open_orders" + }, + "prediction_hit": true, + "prediction_reason": null, + "prediction_status": "hit", + "predictor_wall_ms": 270.206183, + "prefill_ms": 3851.3, + "prefilled_tokens": 91, + "private_result_exposed_on_miss": false, + "request_wall_ms": 6504.486581998208, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_750886b39d7cdf0d6340234d", + "commit_signal_sent": false, + "commit_wait_ms": 0.019967, + "confidence": 0.75, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 6231.715703, + "expected_speedup": 1.5428733045594794, + "prediction": { + "arguments": { + "customer_ref": "azure" + }, + "name": "list_open_orders" + }, + "prediction_source": "qwen3-0.6b", + "predictor_wall_ms": 270.206183, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "call_ref": "jade", + "call_sha256": "f9bb9bbfe57b8faa80fd5f25ba23eb97ae6031ad8e254049f174e1072d96142f", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.112101999548, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "list_open_orders" + }, + "status": "hit" + }, + "tool_call_id": "call_750886b39d7cdf0d6340234d", + "tool_result": { + "call_ref": "jade", + "call_sha256": "f9bb9bbfe57b8faa80fd5f25ba23eb97ae6031ad8e254049f174e1072d96142f", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.112101999548, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "list_open_orders" + }, + "tool_wall_ms": 6231.715703 + }, + { + "accept_rate": 0.6136363744735718, + "assistant_message": { + "content": "Now I need to get the order details.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"orders_ref\":\"jade\"}", + "name": "get_order_details" + }, + "id": "call_b1ec97e6ea85bf4ef9afb4d5", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 658, + "call": { + "arguments": { + "orders_ref": "jade" + }, + "name": "get_order_details" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"orders_ref\":\"jade\"},\"name\":\"get_order_details\"}", + "completion_tokens": 38, + "content": "Now I need to get the order details.\n\n", + "content_sha256": "4855a9d3760fe1217b6d0b8d6dd52f84ec1cb58b5a1b797b630e5dc9519a5417", + "decode_ms": 2460.8, + "decode_tokens_per_sec": 15.4, + "expected_call": { + "arguments": { + "orders_ref": "jade" + }, + "name": "get_order_details" + }, + "exposed_tool_wait_ms": 0.029926, + "model_compute_ms": 11944.2, + "prediction": { + "arguments": { + "orders_ref": "jade" + }, + "name": "get_order_details" + }, + "prediction_hit": true, + "prediction_reason": null, + "prediction_status": "hit", + "predictor_wall_ms": 302.311738, + "prefill_ms": 9483.4, + "prefilled_tokens": 226, + "private_result_exposed_on_miss": false, + "request_wall_ms": 12262.759239001753, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_b1ec97e6ea85bf4ef9afb4d5", + "commit_signal_sent": false, + "commit_wait_ms": 0.029926, + "confidence": 0.75, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 11957.321687, + "expected_speedup": 1.5428733045594794, + "prediction": { + "arguments": { + "orders_ref": "jade" + }, + "name": "get_order_details" + }, + "prediction_source": "qwen3-0.6b", + "predictor_wall_ms": 302.311738, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "call_ref": "amber", + "call_sha256": "428dbfc05f17e7a9295c2b4741dbae2bb91d3f107150b77bb34c6ca25bd8af23", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.0955059986154, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "get_order_details" + }, + "status": "hit" + }, + "tool_call_id": "call_b1ec97e6ea85bf4ef9afb4d5", + "tool_result": { + "call_ref": "amber", + "call_sha256": "428dbfc05f17e7a9295c2b4741dbae2bb91d3f107150b77bb34c6ca25bd8af23", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.0955059986154, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "get_order_details" + }, + "tool_wall_ms": 11957.321687 + } + ], + "task_id": "dependent_workflow_003", + "task_ms": 40068.941525998525 + }, + "task": { + "call_count": 3, + "customer_email": "agent-benchmark-3@example.test", + "destination": "Bologna", + "id": "dependent_workflow_003" + } + }, + { + "arm_order": [ + "control", + "speculative" + ], + "control": { + "all_calls_correct": true, + "all_ds4_active": true, + "all_private_miss_results_hidden": true, + "call_count": 4, + "completion_tokens": 239, + "decode_ms": 10499.3, + "expected_final": "workflow_complete:cedar", + "exposed_tool_wait_ms": 8107.510414996796, + "final": { + "accept_rate": 0.75, + "assistant_message": { + "content": "workflow_complete:cedar", + "role": "assistant" + }, + "cache_hit": true, + "cached_prefix_tokens": 687, + "call": null, + "canonical_call": null, + "completion_tokens": 8, + "content": "workflow_complete:cedar", + "content_sha256": "819da1a0d70737e95681c7bb3ab9df9cfab50c9a2142f5c63b7f8e07482557c8", + "decode_ms": 453.5, + "decode_tokens_per_sec": 17.6, + "model_compute_ms": 25074.2, + "prefill_ms": 24620.7, + "prefilled_tokens": 573, + "request_wall_ms": 25096.153071997833, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": null, + "tool_call_id": "" + }, + "final_correct": true, + "model_compute_ms": 54668.5, + "prediction_hits": 0, + "predictor_wall_ms": 0.0, + "steps": [ + { + "accept_rate": 0.7678571343421936, + "assistant_message": { + "content": "I need to start with step 1: resolve_customer using the customer email from the request.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"customer_email\":\"agent-benchmark-4@example.test\"}", + "name": "resolve_customer" + }, + "id": "call_05f0932a462b9bfb40ee2bf8", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 627, + "call": { + "arguments": { + "customer_email": "agent-benchmark-4@example.test" + }, + "name": "resolve_customer" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"customer_email\":\"agent-benchmark-4@example.test\"},\"name\":\"resolve_customer\"}", + "completion_tokens": 58, + "content": "I need to start with step 1: resolve_customer using the customer email from the request.\n\n", + "content_sha256": "9eb32ae482f67f3e7c0fd5f011a2c9c7fb3e76804c484bdf6b40bf8add1aa381", + "decode_ms": 2664.3, + "decode_tokens_per_sec": 21.8, + "expected_call": { + "arguments": { + "customer_email": "agent-benchmark-4@example.test" + }, + "name": "resolve_customer" + }, + "exposed_tool_wait_ms": 2026.8038169997453, + "model_compute_ms": 2664.3, + "prediction": null, + "prediction_hit": false, + "prediction_reason": null, + "prediction_status": null, + "predictor_wall_ms": 0.0, + "prefill_ms": 0.0, + "prefilled_tokens": 0, + "private_result_exposed_on_miss": false, + "request_wall_ms": 2677.637868997408, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": null, + "tool_call_id": "call_05f0932a462b9bfb40ee2bf8", + "tool_result": { + "call_ref": "coral", + "call_sha256": "3b6684a3dbe4bce123f571a33575e15d7010b9449679680f7be2e7228dc69bb4", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.2625609995448, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "resolve_customer" + }, + "tool_wall_ms": 2026.8038169997453 + }, + { + "accept_rate": 0.7884615659713745, + "assistant_message": { + "content": "Now I need to proceed to step 2: list_open_orders using the customer_ref from the previous result.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"customer_ref\":\"coral\"}", + "name": "list_open_orders" + }, + "id": "call_84a190fa8d91b85eb6b6724e", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 686, + "call": { + "arguments": { + "customer_ref": "coral" + }, + "name": "list_open_orders" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"customer_ref\":\"coral\"},\"name\":\"list_open_orders\"}", + "completion_tokens": 54, + "content": "Now I need to proceed to step 2: list_open_orders using the customer_ref from the previous result.\n\n", + "content_sha256": "a50b5403267366caf3f836c1af4d67edc7470a96137213b4edc93bfb9c0f54fd", + "decode_ms": 2410.2, + "decode_tokens_per_sec": 22.4, + "expected_call": { + "arguments": { + "customer_ref": "coral" + }, + "name": "list_open_orders" + }, + "exposed_tool_wait_ms": 2026.8922049981484, + "model_compute_ms": 6708.9, + "prediction": null, + "prediction_hit": false, + "prediction_reason": null, + "prediction_status": null, + "predictor_wall_ms": 0.0, + "prefill_ms": 4298.7, + "prefilled_tokens": 97, + "private_result_exposed_on_miss": false, + "request_wall_ms": 6728.953044999798, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": null, + "tool_call_id": "call_84a190fa8d91b85eb6b6724e", + "tool_result": { + "call_ref": "coral", + "call_sha256": "12703ba32ec4682fbae3c4aba8cf73057f6427072573ce2f69f4252058fb7ad3", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.1366260003124, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "list_open_orders" + }, + "tool_wall_ms": 2026.8922049981484 + }, + { + "accept_rate": 0.7916666865348816, + "assistant_message": { + "content": "Now I proceed to step 3: get_order_details using the orders_ref from the previous result.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"orders_ref\":\"coral\"}", + "name": "get_order_details" + }, + "id": "call_160bcb296c5d588c79ef1016", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 838, + "call": { + "arguments": { + "orders_ref": "coral" + }, + "name": "get_order_details" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"orders_ref\":\"coral\"},\"name\":\"get_order_details\"}", + "completion_tokens": 50, + "content": "Now I proceed to step 3: get_order_details using the orders_ref from the previous result.\n\n", + "content_sha256": "bc190728cc301be15a35259dc01ce8c5880575dc6bfcc34e09e1382ec03aa00b", + "decode_ms": 2322.5, + "decode_tokens_per_sec": 21.5, + "expected_call": { + "arguments": { + "orders_ref": "coral" + }, + "name": "get_order_details" + }, + "exposed_tool_wait_ms": 2025.844790998235, + "model_compute_ms": 6352.7, + "prediction": null, + "prediction_hit": false, + "prediction_reason": null, + "prediction_status": null, + "predictor_wall_ms": 0.0, + "prefill_ms": 4030.2, + "prefilled_tokens": 93, + "private_result_exposed_on_miss": false, + "request_wall_ms": 6374.144949000765, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": null, + "tool_call_id": "call_160bcb296c5d588c79ef1016", + "tool_result": { + "call_ref": "amber", + "call_sha256": "765b3880efd7c71d4ef691862f97efc09d65b30520b309ca6e780fa539a9c14f", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.1390049983456, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "get_order_details" + }, + "tool_wall_ms": 2025.844790998235 + }, + { + "accept_rate": 0.8125, + "assistant_message": { + "content": "Now I proceed to step 4: calculate_shipping using the order_ref from the previous result and the destination from the user request.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"destination\":\"Florence\",\"order_ref\":\"amber\"}", + "name": "calculate_shipping" + }, + "id": "call_b6edca20200edc145115f6a6", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 838, + "call": { + "arguments": { + "destination": "Florence", + "order_ref": "amber" + }, + "name": "calculate_shipping" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"destination\":\"Florence\",\"order_ref\":\"amber\"},\"name\":\"calculate_shipping\"}", + "completion_tokens": 69, + "content": "Now I proceed to step 4: calculate_shipping using the order_ref from the previous result and the destination from the user request.\n\n", + "content_sha256": "eeddf10343ca1f79fbf9009b3ce6ecb712a9d084d51a516f0d84e085b8e69f56", + "decode_ms": 2648.8, + "decode_tokens_per_sec": 26.0, + "expected_call": { + "arguments": { + "destination": "Florence", + "order_ref": "amber" + }, + "name": "calculate_shipping" + }, + "exposed_tool_wait_ms": 2027.9696020006668, + "model_compute_ms": 13868.400000000001, + "prediction": null, + "prediction_hit": false, + "prediction_reason": null, + "prediction_status": null, + "predictor_wall_ms": 0.0, + "prefill_ms": 11219.6, + "prefilled_tokens": 240, + "private_result_exposed_on_miss": false, + "request_wall_ms": 13890.46880800015, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": null, + "tool_call_id": "call_b6edca20200edc145115f6a6", + "tool_result": { + "call_ref": "cedar", + "call_sha256": "ec7b7dd2a51de438258f7c506a273b314691aa7b33181bc0c1750ed790bf6820", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.3140449989587, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "calculate_shipping" + }, + "tool_wall_ms": 2027.9696020006668 + } + ], + "task_id": "dependent_workflow_004", + "task_ms": 62876.86303100054 + }, + "pair_index": 4, + "speculative": { + "all_calls_correct": true, + "all_ds4_active": true, + "all_private_miss_results_hidden": true, + "call_count": 4, + "completion_tokens": 239, + "decode_ms": 10580.1, + "expected_final": "workflow_complete:cedar", + "exposed_tool_wait_ms": 0.097472, + "final": { + "accept_rate": 0.75, + "assistant_message": { + "content": "workflow_complete:cedar", + "role": "assistant" + }, + "cache_hit": true, + "cached_prefix_tokens": 983, + "call": null, + "canonical_call": null, + "completion_tokens": 8, + "content": "workflow_complete:cedar", + "content_sha256": "819da1a0d70737e95681c7bb3ab9df9cfab50c9a2142f5c63b7f8e07482557c8", + "decode_ms": 450.3, + "decode_tokens_per_sec": 17.8, + "model_compute_ms": 13370.3, + "prefill_ms": 12920.0, + "prefilled_tokens": 277, + "request_wall_ms": 13387.4202650004, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": null, + "tool_call_id": "" + }, + "final_correct": true, + "model_compute_ms": 41931.7, + "prediction_hits": 4, + "predictor_wall_ms": 1259.667634, + "steps": [ + { + "accept_rate": 0.7678571343421936, + "assistant_message": { + "content": "I need to start with step 1: resolve_customer using the customer email from the request.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"customer_email\":\"agent-benchmark-4@example.test\"}", + "name": "resolve_customer" + }, + "id": "call_2837859c6312e6288e81c090", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 627, + "call": { + "arguments": { + "customer_email": "agent-benchmark-4@example.test" + }, + "name": "resolve_customer" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"customer_email\":\"agent-benchmark-4@example.test\"},\"name\":\"resolve_customer\"}", + "completion_tokens": 58, + "content": "I need to start with step 1: resolve_customer using the customer email from the request.\n\n", + "content_sha256": "9eb32ae482f67f3e7c0fd5f011a2c9c7fb3e76804c484bdf6b40bf8add1aa381", + "decode_ms": 2741.1, + "decode_tokens_per_sec": 21.2, + "expected_call": { + "arguments": { + "customer_email": "agent-benchmark-4@example.test" + }, + "name": "resolve_customer" + }, + "exposed_tool_wait_ms": 0.020398, + "model_compute_ms": 2741.1, + "prediction": { + "arguments": { + "customer_email": "agent-benchmark-4@example.test" + }, + "name": "resolve_customer" + }, + "prediction_hit": true, + "prediction_reason": null, + "prediction_status": "hit", + "predictor_wall_ms": 291.061038, + "prefill_ms": 0.0, + "prefilled_tokens": 0, + "private_result_exposed_on_miss": false, + "request_wall_ms": 3047.313964998466, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_2837859c6312e6288e81c090", + "commit_signal_sent": false, + "commit_wait_ms": 0.020398, + "confidence": 0.75, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 2753.997678, + "expected_speedup": 1.5428733045594794, + "prediction": { + "arguments": { + "customer_email": "agent-benchmark-4@example.test" + }, + "name": "resolve_customer" + }, + "prediction_source": "qwen3-0.6b", + "predictor_wall_ms": 291.061038, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "call_ref": "coral", + "call_sha256": "3b6684a3dbe4bce123f571a33575e15d7010b9449679680f7be2e7228dc69bb4", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.1131339995482, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "resolve_customer" + }, + "status": "hit" + }, + "tool_call_id": "call_2837859c6312e6288e81c090", + "tool_result": { + "call_ref": "coral", + "call_sha256": "3b6684a3dbe4bce123f571a33575e15d7010b9449679680f7be2e7228dc69bb4", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.1131339995482, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "resolve_customer" + }, + "tool_wall_ms": 2753.997678 + }, + { + "accept_rate": 0.7884615659713745, + "assistant_message": { + "content": "Now I need to proceed to step 2: list_open_orders using the customer_ref from the previous result.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"customer_ref\":\"coral\"}", + "name": "list_open_orders" + }, + "id": "call_5cb50c7af353303294850cab", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 686, + "call": { + "arguments": { + "customer_ref": "coral" + }, + "name": "list_open_orders" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"customer_ref\":\"coral\"},\"name\":\"list_open_orders\"}", + "completion_tokens": 54, + "content": "Now I need to proceed to step 2: list_open_orders using the customer_ref from the previous result.\n\n", + "content_sha256": "a50b5403267366caf3f836c1af4d67edc7470a96137213b4edc93bfb9c0f54fd", + "decode_ms": 2435.0, + "decode_tokens_per_sec": 22.2, + "expected_call": { + "arguments": { + "customer_ref": "coral" + }, + "name": "list_open_orders" + }, + "exposed_tool_wait_ms": 0.025598, + "model_compute_ms": 6698.4, + "prediction": { + "arguments": { + "customer_ref": "coral" + }, + "name": "list_open_orders" + }, + "prediction_hit": true, + "prediction_reason": null, + "prediction_status": "hit", + "predictor_wall_ms": 284.260567, + "prefill_ms": 4263.4, + "prefilled_tokens": 97, + "private_result_exposed_on_miss": false, + "request_wall_ms": 6999.086507999891, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_5cb50c7af353303294850cab", + "commit_signal_sent": false, + "commit_wait_ms": 0.025598, + "confidence": 0.75, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 6711.259778, + "expected_speedup": 1.5428733045594794, + "prediction": { + "arguments": { + "customer_ref": "coral" + }, + "name": "list_open_orders" + }, + "prediction_source": "qwen3-0.6b", + "predictor_wall_ms": 284.260567, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "call_ref": "coral", + "call_sha256": "12703ba32ec4682fbae3c4aba8cf73057f6427072573ce2f69f4252058fb7ad3", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.1238490003743, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "list_open_orders" + }, + "status": "hit" + }, + "tool_call_id": "call_5cb50c7af353303294850cab", + "tool_result": { + "call_ref": "coral", + "call_sha256": "12703ba32ec4682fbae3c4aba8cf73057f6427072573ce2f69f4252058fb7ad3", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.1238490003743, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "list_open_orders" + }, + "tool_wall_ms": 6711.259778 + }, + { + "accept_rate": 0.7916666865348816, + "assistant_message": { + "content": "Now I proceed to step 3: get_order_details using the orders_ref from the previous result.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"orders_ref\":\"coral\"}", + "name": "get_order_details" + }, + "id": "call_48dcbe64642e138fab1f322c", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 838, + "call": { + "arguments": { + "orders_ref": "coral" + }, + "name": "get_order_details" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"orders_ref\":\"coral\"},\"name\":\"get_order_details\"}", + "completion_tokens": 50, + "content": "Now I proceed to step 3: get_order_details using the orders_ref from the previous result.\n\n", + "content_sha256": "bc190728cc301be15a35259dc01ce8c5880575dc6bfcc34e09e1382ec03aa00b", + "decode_ms": 2313.8, + "decode_tokens_per_sec": 21.6, + "expected_call": { + "arguments": { + "orders_ref": "coral" + }, + "name": "get_order_details" + }, + "exposed_tool_wait_ms": 0.020298, + "model_compute_ms": 6294.9, + "prediction": { + "arguments": { + "orders_ref": "coral" + }, + "name": "get_order_details" + }, + "prediction_hit": true, + "prediction_reason": null, + "prediction_status": "hit", + "predictor_wall_ms": 313.131125, + "prefill_ms": 3981.1, + "prefilled_tokens": 93, + "private_result_exposed_on_miss": false, + "request_wall_ms": 6623.575071000232, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_48dcbe64642e138fab1f322c", + "commit_signal_sent": false, + "commit_wait_ms": 0.020298, + "confidence": 0.75, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 6308.194577, + "expected_speedup": 1.5428733045594794, + "prediction": { + "arguments": { + "orders_ref": "coral" + }, + "name": "get_order_details" + }, + "prediction_source": "qwen3-0.6b", + "predictor_wall_ms": 313.131125, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "call_ref": "amber", + "call_sha256": "765b3880efd7c71d4ef691862f97efc09d65b30520b309ca6e780fa539a9c14f", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.1107650023187, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "get_order_details" + }, + "status": "hit" + }, + "tool_call_id": "call_48dcbe64642e138fab1f322c", + "tool_result": { + "call_ref": "amber", + "call_sha256": "765b3880efd7c71d4ef691862f97efc09d65b30520b309ca6e780fa539a9c14f", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.1107650023187, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "get_order_details" + }, + "tool_wall_ms": 6308.194577 + }, + { + "accept_rate": 0.8125, + "assistant_message": { + "content": "Now I proceed to step 4: calculate_shipping using the order_ref from the previous result and the destination from the user request.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"destination\":\"Florence\",\"order_ref\":\"amber\"}", + "name": "calculate_shipping" + }, + "id": "call_0bf7dcfaba1ade732d19282c", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 838, + "call": { + "arguments": { + "destination": "Florence", + "order_ref": "amber" + }, + "name": "calculate_shipping" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"destination\":\"Florence\",\"order_ref\":\"amber\"},\"name\":\"calculate_shipping\"}", + "completion_tokens": 69, + "content": "Now I proceed to step 4: calculate_shipping using the order_ref from the previous result and the destination from the user request.\n\n", + "content_sha256": "eeddf10343ca1f79fbf9009b3ce6ecb712a9d084d51a516f0d84e085b8e69f56", + "decode_ms": 2639.9, + "decode_tokens_per_sec": 26.1, + "expected_call": { + "arguments": { + "destination": "Florence", + "order_ref": "amber" + }, + "name": "calculate_shipping" + }, + "exposed_tool_wait_ms": 0.031178, + "model_compute_ms": 12827.0, + "prediction": { + "arguments": { + "destination": "Florence", + "order_ref": "amber" + }, + "name": "calculate_shipping" + }, + "prediction_hit": true, + "prediction_reason": null, + "prediction_status": "hit", + "predictor_wall_ms": 371.214904, + "prefill_ms": 10187.1, + "prefilled_tokens": 240, + "private_result_exposed_on_miss": false, + "request_wall_ms": 13214.942769001937, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_0bf7dcfaba1ade732d19282c", + "commit_signal_sent": false, + "commit_wait_ms": 0.031178, + "confidence": 0.75, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 12840.298167, + "expected_speedup": 1.5428733045594794, + "prediction": { + "arguments": { + "destination": "Florence", + "order_ref": "amber" + }, + "name": "calculate_shipping" + }, + "prediction_source": "qwen3-0.6b", + "predictor_wall_ms": 371.214904, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "call_ref": "cedar", + "call_sha256": "ec7b7dd2a51de438258f7c506a273b314691aa7b33181bc0c1750ed790bf6820", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.074912000855, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "calculate_shipping" + }, + "status": "hit" + }, + "tool_call_id": "call_0bf7dcfaba1ade732d19282c", + "tool_result": { + "call_ref": "cedar", + "call_sha256": "ec7b7dd2a51de438258f7c506a273b314691aa7b33181bc0c1750ed790bf6820", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.074912000855, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "calculate_shipping" + }, + "tool_wall_ms": 12840.298167 + } + ], + "task_id": "dependent_workflow_004", + "task_ms": 43273.09860999958 + }, + "task": { + "call_count": 4, + "customer_email": "agent-benchmark-4@example.test", + "destination": "Florence", + "id": "dependent_workflow_004" + } + }, + { + "arm_order": [ + "speculative", + "control" + ], + "control": { + "all_calls_correct": true, + "all_ds4_active": true, + "all_private_miss_results_hidden": true, + "call_count": 5, + "completion_tokens": 251, + "decode_ms": 12638.6, + "expected_final": "workflow_complete:amber", + "exposed_tool_wait_ms": 10131.949799000722, + "final": { + "accept_rate": 0.625, + "assistant_message": { + "content": "workflow_complete:amber", + "role": "assistant" + }, + "cache_hit": true, + "cached_prefix_tokens": 1136, + "call": null, + "canonical_call": null, + "completion_tokens": 7, + "content": "workflow_complete:amber", + "content_sha256": "432bcc6fd099a0a7a97da30984ac9e2d3760ca9fc31f5b74162f2961c04c83b8", + "decode_ms": 447.6, + "decode_tokens_per_sec": 15.6, + "model_compute_ms": 11109.5, + "prefill_ms": 10661.9, + "prefilled_tokens": 253, + "request_wall_ms": 11133.575896001275, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": null, + "tool_call_id": "" + }, + "final_correct": true, + "model_compute_ms": 46075.2, + "prediction_hits": 0, + "predictor_wall_ms": 0.0, + "steps": [ + { + "accept_rate": 0.5921052694320679, + "assistant_message": { + "content": "I need to process this customer request through the 5-step workflow. Let me start with step 1: resolving the customer.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"customer_email\":\"agent-benchmark-5@example.test\"}", + "name": "resolve_customer" + }, + "id": "call_7f9d4f25ac2551c9726d5b28", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 648, + "call": { + "arguments": { + "customer_email": "agent-benchmark-5@example.test" + }, + "name": "resolve_customer" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"customer_email\":\"agent-benchmark-5@example.test\"},\"name\":\"resolve_customer\"}", + "completion_tokens": 64, + "content": "I need to process this customer request through the 5-step workflow. Let me start with step 1: resolving the customer.\n\n", + "content_sha256": "caac2c887d5589ba57ca98c8c77ea68cdd12f4569133cc824cdc1cc82c831c09", + "decode_ms": 3421.7, + "decode_tokens_per_sec": 18.7, + "expected_call": { + "arguments": { + "customer_email": "agent-benchmark-5@example.test" + }, + "name": "resolve_customer" + }, + "exposed_tool_wait_ms": 2025.8035280021431, + "model_compute_ms": 3421.7, + "prediction": null, + "prediction_hit": false, + "prediction_reason": null, + "prediction_status": null, + "predictor_wall_ms": 0.0, + "prefill_ms": 0.0, + "prefilled_tokens": 0, + "private_result_exposed_on_miss": false, + "request_wall_ms": 3435.3289089995087, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": null, + "tool_call_id": "call_7f9d4f25ac2551c9726d5b28", + "tool_result": { + "call_ref": "maple", + "call_sha256": "71063336b1901dd2a0050d4a7a0ebe52ad0b0755f5af079efe22db9c1df00b29", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.163259999681, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "resolve_customer" + }, + "tool_wall_ms": 2025.8035280021431 + }, + { + "accept_rate": 0.7272727489471436, + "assistant_message": { + "content": "Now I need to list open orders for this resolved customer.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"customer_ref\":\"maple\"}", + "name": "list_open_orders" + }, + "id": "call_98a842d8ac148b26dcf0635a", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 713, + "call": { + "arguments": { + "customer_ref": "maple" + }, + "name": "list_open_orders" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"customer_ref\":\"maple\"},\"name\":\"list_open_orders\"}", + "completion_tokens": 43, + "content": "Now I need to list open orders for this resolved customer.\n\n", + "content_sha256": "3ce7d0f18b723f0f13b497c10e5799d52d899e25e977e82e22a5ba8e25b52e6e", + "decode_ms": 2187.2, + "decode_tokens_per_sec": 19.7, + "expected_call": { + "arguments": { + "customer_ref": "maple" + }, + "name": "list_open_orders" + }, + "exposed_tool_wait_ms": 2026.4790990004258, + "model_compute_ms": 6212.4, + "prediction": null, + "prediction_hit": false, + "prediction_reason": null, + "prediction_status": null, + "predictor_wall_ms": 0.0, + "prefill_ms": 4025.2, + "prefilled_tokens": 93, + "private_result_exposed_on_miss": false, + "request_wall_ms": 6232.237337000697, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": null, + "tool_call_id": "call_98a842d8ac148b26dcf0635a", + "tool_result": { + "call_ref": "willow", + "call_sha256": "8da0147e626bec41f9bafd2ec61d07d01a33e176990483fd62a2b93eeea82e2f", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.1353239968012, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "list_open_orders" + }, + "tool_wall_ms": 2026.4790990004258 + }, + { + "accept_rate": 0.6363636255264282, + "assistant_message": { + "content": "Now I need to inspect the selected order.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"orders_ref\":\"willow\"}", + "name": "get_order_details" + }, + "id": "call_27afb3c2c12355b632d21f62", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 850, + "call": { + "arguments": { + "orders_ref": "willow" + }, + "name": "get_order_details" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"orders_ref\":\"willow\"},\"name\":\"get_order_details\"}", + "completion_tokens": 39, + "content": "Now I need to inspect the selected order.\n\n", + "content_sha256": "0501df1f5ec6fcf19c50a853fe0631d19fcc4dac651569fc7278ecc84ffcdb0c", + "decode_ms": 2278.5, + "decode_tokens_per_sec": 17.1, + "expected_call": { + "arguments": { + "orders_ref": "willow" + }, + "name": "get_order_details" + }, + "exposed_tool_wait_ms": 2026.0629559998051, + "model_compute_ms": 6395.9, + "prediction": null, + "prediction_hit": false, + "prediction_reason": null, + "prediction_status": null, + "predictor_wall_ms": 0.0, + "prefill_ms": 4117.4, + "prefilled_tokens": 95, + "private_result_exposed_on_miss": false, + "request_wall_ms": 6417.946880999807, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": null, + "tool_call_id": "call_27afb3c2c12355b632d21f62", + "tool_result": { + "call_ref": "olive", + "call_sha256": "66ea7a270ffff62c364786d69dc72144b0b0eb928c262f682dff72635964c6a9", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.2660539976205, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "get_order_details" + }, + "tool_wall_ms": 2026.0629559998051 + }, + { + "accept_rate": 0.7142857313156128, + "assistant_message": { + "content": "Now I need to calculate shipping for the order to Naples.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"destination\":\"Naples\",\"order_ref\":\"olive\"}", + "name": "calculate_shipping" + }, + "id": "call_43fdf79651e007386a692348", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 985, + "call": { + "arguments": { + "destination": "Naples", + "order_ref": "olive" + }, + "name": "calculate_shipping" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"destination\":\"Naples\",\"order_ref\":\"olive\"},\"name\":\"calculate_shipping\"}", + "completion_tokens": 54, + "content": "Now I need to calculate shipping for the order to Naples.\n\n", + "content_sha256": "fe9b497f45334c64b9aa739eddda6779832ad53fc5148834a71fc2d79b49e467", + "decode_ms": 2846.4, + "decode_tokens_per_sec": 19.0, + "expected_call": { + "arguments": { + "destination": "Naples", + "order_ref": "olive" + }, + "name": "calculate_shipping" + }, + "exposed_tool_wait_ms": 2026.464924998436, + "model_compute_ms": 6915.3, + "prediction": null, + "prediction_hit": false, + "prediction_reason": null, + "prediction_status": null, + "predictor_wall_ms": 0.0, + "prefill_ms": 4068.9, + "prefilled_tokens": 95, + "private_result_exposed_on_miss": false, + "request_wall_ms": 6937.539090999053, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": null, + "tool_call_id": "call_43fdf79651e007386a692348", + "tool_result": { + "call_ref": "cedar", + "call_sha256": "db784ed2c9e33569d9c6bdd36e6963076032cd8d0bf48cf954420a843cb8d593", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.2615110024635, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "calculate_shipping" + }, + "tool_wall_ms": 2026.464924998436 + }, + { + "accept_rate": 0.8500000238418579, + "assistant_message": { + "content": "Now I need to prepare the final customer summary.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"shipping_ref\":\"cedar\"}", + "name": "prepare_customer_summary" + }, + "id": "call_fe73a9484a67df2e2c858e6f", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 985, + "call": { + "arguments": { + "shipping_ref": "cedar" + }, + "name": "prepare_customer_summary" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"shipping_ref\":\"cedar\"},\"name\":\"prepare_customer_summary\"}", + "completion_tokens": 44, + "content": "Now I need to prepare the final customer summary.\n\n", + "content_sha256": "6d12faaaf38ef53fc446a8a34f8fe0f3a22a7bdc6d3dab6ef4f1f25c5ba91b44", + "decode_ms": 1457.2, + "decode_tokens_per_sec": 30.2, + "expected_call": { + "arguments": { + "shipping_ref": "cedar" + }, + "name": "prepare_customer_summary" + }, + "exposed_tool_wait_ms": 2027.1392909999122, + "model_compute_ms": 12020.400000000001, + "prediction": null, + "prediction_hit": false, + "prediction_reason": null, + "prediction_status": null, + "predictor_wall_ms": 0.0, + "prefill_ms": 10563.2, + "prefilled_tokens": 245, + "private_result_exposed_on_miss": false, + "request_wall_ms": 12043.583460002992, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": null, + "tool_call_id": "call_fe73a9484a67df2e2c858e6f", + "tool_result": { + "call_ref": "amber", + "call_sha256": "6ba7d6c0326fbe23206f51a9a4268beb41a3050097565421946c4ac32900f9bc", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.4121849997318, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "prepare_customer_summary" + }, + "tool_wall_ms": 2027.1392909999122 + } + ], + "task_id": "dependent_workflow_005", + "task_ms": 56334.70175000184 + }, + "pair_index": 5, + "speculative": { + "all_calls_correct": true, + "all_ds4_active": true, + "all_private_miss_results_hidden": true, + "call_count": 5, + "completion_tokens": 251, + "decode_ms": 12663.6, + "expected_final": "workflow_complete:amber", + "exposed_tool_wait_ms": 0.109194, + "final": { + "accept_rate": 0.625, + "assistant_message": { + "content": "workflow_complete:amber", + "role": "assistant" + }, + "cache_hit": true, + "cached_prefix_tokens": 714, + "call": null, + "canonical_call": null, + "completion_tokens": 7, + "content": "workflow_complete:amber", + "content_sha256": "432bcc6fd099a0a7a97da30984ac9e2d3760ca9fc31f5b74162f2961c04c83b8", + "decode_ms": 445.5, + "decode_tokens_per_sec": 15.7, + "model_compute_ms": 29265.4, + "prefill_ms": 28819.9, + "prefilled_tokens": 675, + "request_wall_ms": 29282.140494000487, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": null, + "tool_call_id": "" + }, + "final_correct": true, + "model_compute_ms": 64146.3, + "prediction_hits": 5, + "predictor_wall_ms": 1818.887272, + "steps": [ + { + "accept_rate": 0.5921052694320679, + "assistant_message": { + "content": "I need to process this customer request through the 5-step workflow. Let me start with step 1: resolving the customer.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"customer_email\":\"agent-benchmark-5@example.test\"}", + "name": "resolve_customer" + }, + "id": "call_09ab10b04127b5b933f396fb", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 648, + "call": { + "arguments": { + "customer_email": "agent-benchmark-5@example.test" + }, + "name": "resolve_customer" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"customer_email\":\"agent-benchmark-5@example.test\"},\"name\":\"resolve_customer\"}", + "completion_tokens": 64, + "content": "I need to process this customer request through the 5-step workflow. Let me start with step 1: resolving the customer.\n\n", + "content_sha256": "caac2c887d5589ba57ca98c8c77ea68cdd12f4569133cc824cdc1cc82c831c09", + "decode_ms": 3429.0, + "decode_tokens_per_sec": 18.7, + "expected_call": { + "arguments": { + "customer_email": "agent-benchmark-5@example.test" + }, + "name": "resolve_customer" + }, + "exposed_tool_wait_ms": 0.02144, + "model_compute_ms": 3429.0, + "prediction": { + "arguments": { + "customer_email": "agent-benchmark-5@example.test" + }, + "name": "resolve_customer" + }, + "prediction_hit": true, + "prediction_reason": null, + "prediction_status": "hit", + "predictor_wall_ms": 295.276084, + "prefill_ms": 0.0, + "prefilled_tokens": 0, + "private_result_exposed_on_miss": false, + "request_wall_ms": 3771.3140349987953, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_09ab10b04127b5b933f396fb", + "commit_signal_sent": false, + "commit_wait_ms": 0.02144, + "confidence": 0.75, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 3474.123096, + "expected_speedup": 1.5428733045594794, + "prediction": { + "arguments": { + "customer_email": "agent-benchmark-5@example.test" + }, + "name": "resolve_customer" + }, + "prediction_source": "qwen3-0.6b", + "predictor_wall_ms": 295.276084, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "call_ref": "maple", + "call_sha256": "71063336b1901dd2a0050d4a7a0ebe52ad0b0755f5af079efe22db9c1df00b29", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.1511240006948, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "resolve_customer" + }, + "status": "hit" + }, + "tool_call_id": "call_09ab10b04127b5b933f396fb", + "tool_result": { + "call_ref": "maple", + "call_sha256": "71063336b1901dd2a0050d4a7a0ebe52ad0b0755f5af079efe22db9c1df00b29", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.1511240006948, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "resolve_customer" + }, + "tool_wall_ms": 3474.123096 + }, + { + "accept_rate": 0.7272727489471436, + "assistant_message": { + "content": "Now I need to list open orders for this resolved customer.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"customer_ref\":\"maple\"}", + "name": "list_open_orders" + }, + "id": "call_63e840f8505a49839de2d13a", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 713, + "call": { + "arguments": { + "customer_ref": "maple" + }, + "name": "list_open_orders" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"customer_ref\":\"maple\"},\"name\":\"list_open_orders\"}", + "completion_tokens": 43, + "content": "Now I need to list open orders for this resolved customer.\n\n", + "content_sha256": "3ce7d0f18b723f0f13b497c10e5799d52d899e25e977e82e22a5ba8e25b52e6e", + "decode_ms": 2186.8, + "decode_tokens_per_sec": 19.7, + "expected_call": { + "arguments": { + "customer_ref": "maple" + }, + "name": "list_open_orders" + }, + "exposed_tool_wait_ms": 0.016611, + "model_compute_ms": 6183.6, + "prediction": { + "arguments": { + "customer_ref": "maple" + }, + "name": "list_open_orders" + }, + "prediction_hit": true, + "prediction_reason": null, + "prediction_status": "hit", + "predictor_wall_ms": 286.880963, + "prefill_ms": 3996.8, + "prefilled_tokens": 93, + "private_result_exposed_on_miss": false, + "request_wall_ms": 6486.068479000096, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_63e840f8505a49839de2d13a", + "commit_signal_sent": false, + "commit_wait_ms": 0.016611, + "confidence": 0.75, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 6196.614339, + "expected_speedup": 1.5428733045594794, + "prediction": { + "arguments": { + "customer_ref": "maple" + }, + "name": "list_open_orders" + }, + "prediction_source": "qwen3-0.6b", + "predictor_wall_ms": 286.880963, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "call_ref": "willow", + "call_sha256": "8da0147e626bec41f9bafd2ec61d07d01a33e176990483fd62a2b93eeea82e2f", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.1012419998005, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "list_open_orders" + }, + "status": "hit" + }, + "tool_call_id": "call_63e840f8505a49839de2d13a", + "tool_result": { + "call_ref": "willow", + "call_sha256": "8da0147e626bec41f9bafd2ec61d07d01a33e176990483fd62a2b93eeea82e2f", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.1012419998005, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "list_open_orders" + }, + "tool_wall_ms": 6196.614339 + }, + { + "accept_rate": 0.6363636255264282, + "assistant_message": { + "content": "Now I need to inspect the selected order.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"orders_ref\":\"willow\"}", + "name": "get_order_details" + }, + "id": "call_b36d77890661c729282b8106", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 850, + "call": { + "arguments": { + "orders_ref": "willow" + }, + "name": "get_order_details" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"orders_ref\":\"willow\"},\"name\":\"get_order_details\"}", + "completion_tokens": 39, + "content": "Now I need to inspect the selected order.\n\n", + "content_sha256": "0501df1f5ec6fcf19c50a853fe0631d19fcc4dac651569fc7278ecc84ffcdb0c", + "decode_ms": 2287.7, + "decode_tokens_per_sec": 17.0, + "expected_call": { + "arguments": { + "orders_ref": "willow" + }, + "name": "get_order_details" + }, + "exposed_tool_wait_ms": 0.018384, + "model_compute_ms": 6361.299999999999, + "prediction": { + "arguments": { + "orders_ref": "willow" + }, + "name": "get_order_details" + }, + "prediction_hit": true, + "prediction_reason": null, + "prediction_status": "hit", + "predictor_wall_ms": 315.010142, + "prefill_ms": 4073.6, + "prefilled_tokens": 95, + "private_result_exposed_on_miss": false, + "request_wall_ms": 6692.558883001766, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_b36d77890661c729282b8106", + "commit_signal_sent": false, + "commit_wait_ms": 0.018384, + "confidence": 0.75, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 6374.658399, + "expected_speedup": 1.5428733045594794, + "prediction": { + "arguments": { + "orders_ref": "willow" + }, + "name": "get_order_details" + }, + "prediction_source": "qwen3-0.6b", + "predictor_wall_ms": 315.010142, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "call_ref": "olive", + "call_sha256": "66ea7a270ffff62c364786d69dc72144b0b0eb928c262f682dff72635964c6a9", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.129840002046, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "get_order_details" + }, + "status": "hit" + }, + "tool_call_id": "call_b36d77890661c729282b8106", + "tool_result": { + "call_ref": "olive", + "call_sha256": "66ea7a270ffff62c364786d69dc72144b0b0eb928c262f682dff72635964c6a9", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.129840002046, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "get_order_details" + }, + "tool_wall_ms": 6374.658399 + }, + { + "accept_rate": 0.7142857313156128, + "assistant_message": { + "content": "Now I need to calculate shipping for the order to Naples.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"destination\":\"Naples\",\"order_ref\":\"olive\"}", + "name": "calculate_shipping" + }, + "id": "call_21d1fac465004f3136ee9c1d", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 985, + "call": { + "arguments": { + "destination": "Naples", + "order_ref": "olive" + }, + "name": "calculate_shipping" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"destination\":\"Naples\",\"order_ref\":\"olive\"},\"name\":\"calculate_shipping\"}", + "completion_tokens": 54, + "content": "Now I need to calculate shipping for the order to Naples.\n\n", + "content_sha256": "fe9b497f45334c64b9aa739eddda6779832ad53fc5148834a71fc2d79b49e467", + "decode_ms": 2859.3, + "decode_tokens_per_sec": 18.9, + "expected_call": { + "arguments": { + "destination": "Naples", + "order_ref": "olive" + }, + "name": "calculate_shipping" + }, + "exposed_tool_wait_ms": 0.029555, + "model_compute_ms": 6913.5, + "prediction": { + "arguments": { + "destination": "Naples", + "order_ref": "olive" + }, + "name": "calculate_shipping" + }, + "prediction_hit": true, + "prediction_reason": null, + "prediction_status": "hit", + "predictor_wall_ms": 544.435294, + "prefill_ms": 4054.2, + "prefilled_tokens": 95, + "private_result_exposed_on_miss": false, + "request_wall_ms": 7478.634543997032, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_21d1fac465004f3136ee9c1d", + "commit_signal_sent": false, + "commit_wait_ms": 0.029555, + "confidence": 0.75, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 6931.548898, + "expected_speedup": 1.5428733045594794, + "prediction": { + "arguments": { + "destination": "Naples", + "order_ref": "olive" + }, + "name": "calculate_shipping" + }, + "prediction_source": "qwen3-0.6b", + "predictor_wall_ms": 544.435294, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "call_ref": "cedar", + "call_sha256": "db784ed2c9e33569d9c6bdd36e6963076032cd8d0bf48cf954420a843cb8d593", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.118616000691, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "calculate_shipping" + }, + "status": "hit" + }, + "tool_call_id": "call_21d1fac465004f3136ee9c1d", + "tool_result": { + "call_ref": "cedar", + "call_sha256": "db784ed2c9e33569d9c6bdd36e6963076032cd8d0bf48cf954420a843cb8d593", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.118616000691, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "calculate_shipping" + }, + "tool_wall_ms": 6931.548898 + }, + { + "accept_rate": 0.8500000238418579, + "assistant_message": { + "content": "Now I need to prepare the final customer summary.\n\n", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"shipping_ref\":\"cedar\"}", + "name": "prepare_customer_summary" + }, + "id": "call_94fef0533909642dcbd44e46", + "type": "function" + } + ] + }, + "cache_hit": true, + "cached_prefix_tokens": 985, + "call": { + "arguments": { + "shipping_ref": "cedar" + }, + "name": "prepare_customer_summary" + }, + "call_correct": true, + "canonical_call": "{\"arguments\":{\"shipping_ref\":\"cedar\"},\"name\":\"prepare_customer_summary\"}", + "completion_tokens": 44, + "content": "Now I need to prepare the final customer summary.\n\n", + "content_sha256": "6d12faaaf38ef53fc446a8a34f8fe0f3a22a7bdc6d3dab6ef4f1f25c5ba91b44", + "decode_ms": 1455.3, + "decode_tokens_per_sec": 30.2, + "expected_call": { + "arguments": { + "shipping_ref": "cedar" + }, + "name": "prepare_customer_summary" + }, + "exposed_tool_wait_ms": 0.023204, + "model_compute_ms": 11993.5, + "prediction": { + "arguments": { + "shipping_ref": "cedar" + }, + "name": "prepare_customer_summary" + }, + "prediction_hit": true, + "prediction_reason": null, + "prediction_status": "hit", + "predictor_wall_ms": 377.284789, + "prefill_ms": 10538.2, + "prefilled_tokens": 245, + "private_result_exposed_on_miss": false, + "request_wall_ms": 12388.57133199781, + "semantic_hint": { + "accepted_tokens": 0, + "longest_suffix_tokens": 0, + "matched_rounds": 0, + "native_gate_misses": 0, + "proposed_tokens": 0, + "readiness_polls": 0, + "rejected_rounds": 0 + }, + "speculation": { + "accelerator_relation": "non_accelerator", + "call_id": "call_94fef0533909642dcbd44e46", + "commit_signal_sent": false, + "commit_wait_ms": 0.023204, + "confidence": 0.75, + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "cpu_affinity_isolated": true, + "decode_interference_qualified": true, + "executor_wall_ms": 12007.430824, + "expected_speedup": 1.5428733045594794, + "prediction": { + "arguments": { + "shipping_ref": "cedar" + }, + "name": "prepare_customer_summary" + }, + "prediction_source": "qwen3-0.6b", + "predictor_wall_ms": 377.284789, + "protocol": "dflash.tool-speculation.v1", + "resource_percentage": 100, + "result": { + "call_ref": "amber", + "call_sha256": "6ba7d6c0326fbe23206f51a9a4268beb41a3050097565421946c4ac32900f9bc", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.3881299999193, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "prepare_customer_summary" + }, + "status": "hit" + }, + "tool_call_id": "call_94fef0533909642dcbd44e46", + "tool_result": { + "call_ref": "amber", + "call_sha256": "6ba7d6c0326fbe23206f51a9a4268beb41a3050097565421946c4ac32900f9bc", + "cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "elapsed_ms": 2000.3881299999193, + "latency_ms": 2000, + "side_effects": false, + "tool_name": "prepare_customer_summary" + }, + "tool_wall_ms": 12007.430824 + } + ], + "task_id": "dependent_workflow_005", + "task_ms": 66100.22805599874 + }, + "task": { + "call_count": 5, + "customer_email": "agent-benchmark-5@example.test", + "destination": "Naples", + "id": "dependent_workflow_005" + } + } + ], + "production_gate": { + "checks": { + "control_calls_correct": true, + "control_finals_correct": true, + "decode_slowdown_p50": true, + "decode_slowdown_p95": false, + "ds4_active": true, + "full_task_speedup": false, + "full_task_speedup_ci": false, + "full_task_tail": false, + "model_slowdown_p50": false, + "model_slowdown_p95": false, + "prediction_hit_rate": true, + "prefix_cache_active": true, + "private_miss_results_hidden": true, + "speculative_calls_correct": true, + "speculative_finals_correct": true, + "target_calls_stable": true, + "target_outputs_stable": true, + "tool_results_stable": true + }, + "passed": false, + "thresholds": { + "max_decode_slowdown_p95_percent": 5.0, + "max_decode_slowdown_percent": 1.0, + "max_model_slowdown_p95_percent": 5.0, + "max_model_slowdown_percent": 1.0, + "min_continuation_cache_hit_rate": 0.8, + "min_hit_rate": 0.5, + "min_task_speedup": 1.4, + "min_task_speedup_ci_low": 1.0, + "min_task_speedup_p05": 1.0 + } + }, + "schema_version": 1, + "server_snapshot": { + "prefix_cache": { + "capacity": 32, + "in_use": 8, + "lifetime_hits": 12 + }, + "tool_speculation": { + "allowed_tools": [ + "calculate_shipping", + "get_order_details", + "list_open_orders", + "prepare_customer_summary", + "resolve_customer" + ], + "automatic_prediction_enabled": true, + "compute_isolation": "disjoint_cpu_affinity", + "cpu_affinity_isolated": true, + "enabled": true, + "execution_mode": "child_process_cpu_affinity", + "executor_contract": "child_process_cpu_affinity", + "hip_reserved_tool_compute_units": 0, + "hip_tool_device": null, + "max_model_slowdown_ratio": 1.05, + "model_cpu_affinity": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29 + ], + "model_expert_ownership_unique": false, + "model_routing_static": false, + "prediction_confidence": 0.75, + "prediction_source": "qwen3-0.6b", + "predictor_decode_isolated": true, + "predictor_schedule": "before-model", + "preserves_token_speculation": true, + "profile_lanes": [ + { + "accelerator_relation": "non_accelerator", + "decode_interference_qualified": true, + "model_slowdown_ratio": 1.0011341375399527, + "requires_static_model_routing": false, + "requires_unique_expert_ownership": false, + "resource_percentage": 100 + } + ], + "profile_status": "qualified", + "protocol": "dflash.tool-speculation.v1", + "requires_client_support": false, + "tool_cpu_affinity": [ + 14, + 15, + 30, + 31 + ], + "unqualified_lane_policy": "defer" + } + }, + "summary": { + "all_ds4_active": true, + "all_private_miss_results_hidden": true, + "all_tool_results_stable": true, + "calls_per_task": [ + 3, + 4, + 5, + 3, + 4, + 5 + ], + "continuation_cache_hit_rate": 1.0, + "control_call_accuracy": 1.0, + "control_exposed_tool_wait_p50_ms": 8106.7744020001555, + "control_final_accuracy": 1.0, + "control_model_compute_p50_ms": 49637.649999999994, + "control_task_p50_ms": 58870.49264650159, + "decode_slowdown_p50_percent": 0.2508973038222173, + "decode_slowdown_p95_percent": 9.284182748470627, + "exposed_tool_wait_reduction_percent": 99.99885625636658, + "exposed_tool_wait_total_speedup": 87432.1806726216, + "ideal_zero_interference_task_speedup_ceiling_p50": 1.1425721859694333, + "model_compute_slowdown_p50_percent": 7.474064005523928, + "model_compute_slowdown_p95_percent": 34.570885935240916, + "paired_saved_p50_ms": 3369.1297750010563, + "paired_task_speedup_bootstrap_95ci": [ + 0.9076229958518257, + 1.3052515771052713 + ], + "paired_task_speedup_min": 0.8522618364081324, + "paired_task_speedup_p05": 0.879942416129979, + "paired_task_speedup_p50": 1.0520357724328036, + "prediction_hit_rate": 1.0, + "prediction_hits": 24, + "predictor_per_call_p50_ms": 296.37505350000004, + "predictor_per_call_p95_ms": 378.20141285, + "predictor_per_task_p50_ms": 1263.2751875, + "semantic_hint_acceptance_rate": null, + "semantic_hint_accepted_tokens": 0, + "semantic_hint_proposed_tokens": 0, + "semantic_hint_ready_calls": 0, + "speculative_call_accuracy": 1.0, + "speculative_exposed_tool_wait_p50_ms": 0.0937145, + "speculative_final_accuracy": 1.0, + "speculative_model_compute_p50_ms": 49687.85, + "speculative_task_p50_ms": 51032.53195199977, + "target_call_stability_rate": 1.0, + "target_output_stability_rate": 1.0, + "tasks": 6, + "tasks_with_all_prediction_hits": 6, + "total_tool_calls_per_arm": 24, + "total_wall_speedup": 1.0655904186922165 + }, + "workload": { + "available_tools": [ + "resolve_customer", + "list_open_orders", + "get_order_details", + "calculate_shipping", + "prepare_customer_summary" + ], + "calls_per_task": "3-5", + "dependency": "every call after the first consumes call_ref returned by the preceding tool", + "name": "dependent multi-turn tool workflow", + "tasks": 6, + "tool_adapter": "deterministic read-only 2-second API replay; no external side effects" + } +} diff --git a/optimizations/ooo_spec_lucebox5_cpu/results/trace-compiled-engine-qwen-production-6pairs-compact.json b/optimizations/ooo_spec_lucebox5_cpu/results/trace-compiled-engine-qwen-production-6pairs-compact.json index 3031e16c2..682b57a20 100644 --- a/optimizations/ooo_spec_lucebox5_cpu/results/trace-compiled-engine-qwen-production-6pairs-compact.json +++ b/optimizations/ooo_spec_lucebox5_cpu/results/trace-compiled-engine-qwen-production-6pairs-compact.json @@ -2,6 +2,7 @@ "feature": "no-training trace-compiled speculative tool graphs", "host": "lucebox5", "methodology": { + "additive_gate_refresh": "the wrong-call privacy probe and server snapshot were refreshed after the timing arms; no recorded timing was recomputed", "argument_binding": "the harness binds validated structured inputs to a request-scoped workflow_ref before either model runs", "arm_order": "randomized per task", "compiled": "one DS4+DSpark macro authorization; independent branches execute concurrently on the Strix CPU lane", @@ -21,16 +22,16 @@ ], "compiled": { "all_ds4_active": true, - "completion_tokens": 50, - "decode_ms": 2097.5, - "exposed_tool_wait_ms": 10124.96868299786, + "completion_tokens": 46, + "decode_ms": 1319.1, + "exposed_tool_wait_ms": 10123.539726002491, "final": { - "accept_rate": 0.9166666865348816, - "completion_tokens": 14, - "content_sha256": "cfbe812f379c62a39fb1047ae84da4e1c121dc72dc0bcadda519322908c0d43e" + "accept_rate": 1.0, + "completion_tokens": 11, + "content_sha256": "1cf1bd9a085192434f76d5bd7ec2044e26bd2d8cd095aed540621b7d837b4196" }, "final_correct": true, - "graph_wall_ms": 10126.189057999, + "graph_wall_ms": 10124.20064300386, "macro_call": { "arguments": { "workflow_ref": "workflow_taska" @@ -38,32 +39,116 @@ "name": "execute_customer_workflows" }, "macro_correct": true, - "model_compute_ms": 13641.900000000001, + "model_compute_ms": 20001.2, "model_turns": 2, "prediction_hit": false, "prediction_reason": null, "prediction_source": null, "prediction_status": null, "predictor_ms": 0.0, - "task_ms": 23787.2067290009, + "task_ms": 30215.39950100123, "tool_results_count": 10, "tool_results_sha256": "eb2ed6a561ebc1e15c6f1a5226fb751de351f983d52dfe47f94c23a97d8c9cb2", "underlying_calls_count": 10, "underlying_calls_sha256": "b17c72ad474ba5d0a431bd35335fcb839c0a329504f6eed9eda5d211249e6841" }, + "interference_probe": { + "observations": [ + { + "compiled": { + "accept_rate": 0.9642857313156128, + "cache_hit": false, + "cached_prefix_tokens": 0, + "call_sha256": "26b31bf2044cd34ec9ea91d14b9758544b6a4a62a533a58bc4f79af1472a6535", + "completion_tokens": 35, + "decode_ms": 1085.7, + "model_compute_ms": 10642.6, + "prefill_ms": 9556.9 + }, + "order": [ + "compiled", + "speculative" + ], + "speculative": { + "accept_rate": 0.9642857313156128, + "cache_hit": false, + "cached_prefix_tokens": 0, + "call_sha256": "26b31bf2044cd34ec9ea91d14b9758544b6a4a62a533a58bc4f79af1472a6535", + "completion_tokens": 35, + "decode_ms": 1090.8, + "model_compute_ms": 10675.099999999999, + "prefill_ms": 9584.3 + } + }, + { + "compiled": { + "accept_rate": 0.9642857313156128, + "cache_hit": false, + "cached_prefix_tokens": 0, + "call_sha256": "26b31bf2044cd34ec9ea91d14b9758544b6a4a62a533a58bc4f79af1472a6535", + "completion_tokens": 35, + "decode_ms": 1092.3, + "model_compute_ms": 10658.599999999999, + "prefill_ms": 9566.3 + }, + "order": [ + "speculative", + "compiled" + ], + "speculative": { + "accept_rate": 0.9642857313156128, + "cache_hit": false, + "cached_prefix_tokens": 0, + "call_sha256": "26b31bf2044cd34ec9ea91d14b9758544b6a4a62a533a58bc4f79af1472a6535", + "completion_tokens": 35, + "decode_ms": 1090.4, + "model_compute_ms": 10671.199999999999, + "prefill_ms": 9580.8 + } + }, + { + "compiled": { + "accept_rate": 0.9642857313156128, + "cache_hit": false, + "cached_prefix_tokens": 0, + "call_sha256": "26b31bf2044cd34ec9ea91d14b9758544b6a4a62a533a58bc4f79af1472a6535", + "completion_tokens": 35, + "decode_ms": 1086.8, + "model_compute_ms": 10666.099999999999, + "prefill_ms": 9579.3 + }, + "order": [ + "compiled", + "speculative" + ], + "speculative": { + "accept_rate": 0.9642857313156128, + "cache_hit": false, + "cached_prefix_tokens": 0, + "call_sha256": "26b31bf2044cd34ec9ea91d14b9758544b6a4a62a533a58bc4f79af1472a6535", + "completion_tokens": 35, + "decode_ms": 1093.4, + "model_compute_ms": 10674.8, + "prefill_ms": 9581.4 + } + } + ], + "qualified": true, + "repetitions": 3 + }, "pair_index": 0, "speculative": { "all_ds4_active": true, - "completion_tokens": 50, - "decode_ms": 2087.8, - "exposed_tool_wait_ms": 0.019826, + "completion_tokens": 46, + "decode_ms": 1208.0, + "exposed_tool_wait_ms": 0.031249, "final": { - "accept_rate": 0.9166666865348816, - "completion_tokens": 14, - "content_sha256": "cfbe812f379c62a39fb1047ae84da4e1c121dc72dc0bcadda519322908c0d43e" + "accept_rate": 1.0, + "completion_tokens": 11, + "content_sha256": "1cf1bd9a085192434f76d5bd7ec2044e26bd2d8cd095aed540621b7d837b4196" }, "final_correct": true, - "graph_wall_ms": 10002.203055999416, + "graph_wall_ms": 10002.38108499616, "macro_call": { "arguments": { "workflow_ref": "workflow_taska" @@ -71,14 +156,14 @@ "name": "execute_customer_workflows" }, "macro_correct": true, - "model_compute_ms": 13568.7, + "model_compute_ms": 16392.2, "model_turns": 2, "prediction_hit": true, "prediction_reason": null, "prediction_source": "native-qwen3", "prediction_status": "hit", - "predictor_ms": 198.696329, - "task_ms": 13771.538261993555, + "predictor_ms": 198.454737, + "task_ms": 16606.234144994232, "tool_results_count": 10, "tool_results_sha256": "eb2ed6a561ebc1e15c6f1a5226fb751de351f983d52dfe47f94c23a97d8c9cb2", "underlying_calls_count": 10, @@ -86,18 +171,18 @@ }, "stage_batched": { "all_ds4_active": true, - "completion_tokens": 204, - "decode_ms": 8934.7, - "exposed_tool_wait_ms": 10135.228522005491, + "completion_tokens": 207, + "decode_ms": 7398.0, + "exposed_tool_wait_ms": 10140.203142997052, "final": { - "accept_rate": 0.9166666865348816, - "completion_tokens": 14, - "content_sha256": "cfbe812f379c62a39fb1047ae84da4e1c121dc72dc0bcadda519322908c0d43e" + "accept_rate": 1.0, + "completion_tokens": 11, + "content_sha256": "1cf1bd9a085192434f76d5bd7ec2044e26bd2d8cd095aed540621b7d837b4196" }, "final_correct": true, - "model_compute_ms": 67834.7, + "model_compute_ms": 71711.6, "model_turns": 6, - "task_ms": 77984.87404600019, + "task_ms": 81868.44566600485, "tool_results_count": 10, "tool_results_sha256": "eb2ed6a561ebc1e15c6f1a5226fb751de351f983d52dfe47f94c23a97d8c9cb2", "underlying_calls_count": 10, @@ -117,16 +202,16 @@ ], "compiled": { "all_ds4_active": true, - "completion_tokens": 51, - "decode_ms": 2239.5, - "exposed_tool_wait_ms": 10143.1347070029, + "completion_tokens": 71, + "decode_ms": 3487.0, + "exposed_tool_wait_ms": 10146.3531469999, "final": { - "accept_rate": 0.6875, - "completion_tokens": 15, - "content_sha256": "e9618263bdfa8c2639cf7326931749b0625a0f5ca309f638c95ad621c5809303" + "accept_rate": 0.75, + "completion_tokens": 12, + "content_sha256": "1f19e61b2ef79fed2c87e8d945b3cc479796bd8a235bc79ef7579f3bbc22adb3" }, "final_correct": true, - "graph_wall_ms": 10144.398914002522, + "graph_wall_ms": 10147.722080000676, "macro_call": { "arguments": { "workflow_ref": "workflow_taskb" @@ -134,32 +219,116 @@ "name": "execute_customer_workflows" }, "macro_correct": true, - "model_compute_ms": 14251.7, + "model_compute_ms": 25378.2, "model_turns": 2, "prediction_hit": false, "prediction_reason": null, "prediction_source": null, "prediction_status": null, "predictor_ms": 0.0, - "task_ms": 24427.28014900058, + "task_ms": 35538.563240996154, "tool_results_count": 15, "tool_results_sha256": "2037fc8f56106ba9bc50763cc4b82c0f3262f0ffb8682ca5ca727eaf9d46b07b", "underlying_calls_count": 15, "underlying_calls_sha256": "2263e8d373d489cb82f03725feec381fb4ec218f6a706ebff2de102ed1e7c40b" }, + "interference_probe": { + "observations": [ + { + "compiled": { + "accept_rate": 0.8035714030265808, + "cache_hit": false, + "cached_prefix_tokens": 0, + "call_sha256": "4fd23541e355718c0b1ee47f20028d7324bab11dddd5a0a5a460495dd05e3461", + "completion_tokens": 59, + "decode_ms": 2711.4, + "model_compute_ms": 12727.199999999999, + "prefill_ms": 10015.8 + }, + "order": [ + "speculative", + "compiled" + ], + "speculative": { + "accept_rate": 0.7333333492279053, + "cache_hit": false, + "cached_prefix_tokens": 0, + "call_sha256": "4fd23541e355718c0b1ee47f20028d7324bab11dddd5a0a5a460495dd05e3461", + "completion_tokens": 59, + "decode_ms": 2996.9, + "model_compute_ms": 13019.8, + "prefill_ms": 10022.9 + } + }, + { + "compiled": { + "accept_rate": 0.7333333492279053, + "cache_hit": false, + "cached_prefix_tokens": 0, + "call_sha256": "4fd23541e355718c0b1ee47f20028d7324bab11dddd5a0a5a460495dd05e3461", + "completion_tokens": 59, + "decode_ms": 2996.0, + "model_compute_ms": 13017.8, + "prefill_ms": 10021.8 + }, + "order": [ + "compiled", + "speculative" + ], + "speculative": { + "accept_rate": 0.8035714030265808, + "cache_hit": false, + "cached_prefix_tokens": 0, + "call_sha256": "4fd23541e355718c0b1ee47f20028d7324bab11dddd5a0a5a460495dd05e3461", + "completion_tokens": 59, + "decode_ms": 2722.6, + "model_compute_ms": 12733.0, + "prefill_ms": 10010.4 + } + }, + { + "compiled": { + "accept_rate": 0.8035714030265808, + "cache_hit": false, + "cached_prefix_tokens": 0, + "call_sha256": "4fd23541e355718c0b1ee47f20028d7324bab11dddd5a0a5a460495dd05e3461", + "completion_tokens": 59, + "decode_ms": 2733.1, + "model_compute_ms": 12738.4, + "prefill_ms": 10005.3 + }, + "order": [ + "speculative", + "compiled" + ], + "speculative": { + "accept_rate": 0.8035714030265808, + "cache_hit": false, + "cached_prefix_tokens": 0, + "call_sha256": "4fd23541e355718c0b1ee47f20028d7324bab11dddd5a0a5a460495dd05e3461", + "completion_tokens": 59, + "decode_ms": 2698.5, + "model_compute_ms": 12698.8, + "prefill_ms": 10000.3 + } + } + ], + "qualified": true, + "repetitions": 3 + }, "pair_index": 1, "speculative": { "all_ds4_active": true, - "completion_tokens": 51, - "decode_ms": 2241.1, - "exposed_tool_wait_ms": 0.035275, + "completion_tokens": 71, + "decode_ms": 3204.2, + "exposed_tool_wait_ms": 0.029325, "final": { - "accept_rate": 0.6875, - "completion_tokens": 15, - "content_sha256": "e9618263bdfa8c2639cf7326931749b0625a0f5ca309f638c95ad621c5809303" + "accept_rate": 0.75, + "completion_tokens": 12, + "content_sha256": "1f19e61b2ef79fed2c87e8d945b3cc479796bd8a235bc79ef7579f3bbc22adb3" }, "final_correct": true, - "graph_wall_ms": 10001.984403999813, + "graph_wall_ms": 10002.10483900446, "macro_call": { "arguments": { "workflow_ref": "workflow_taskb" @@ -167,14 +336,14 @@ "name": "execute_customer_workflows" }, "macro_correct": true, - "model_compute_ms": 14184.8, + "model_compute_ms": 19739.2, "model_turns": 2, "prediction_hit": true, "prediction_reason": null, "prediction_source": "native-qwen3", "prediction_status": "hit", - "predictor_ms": 201.174369, - "task_ms": 14390.547144001175, + "predictor_ms": 200.659826, + "task_ms": 19955.812669002626, "tool_results_count": 15, "tool_results_sha256": "2037fc8f56106ba9bc50763cc4b82c0f3262f0ffb8682ca5ca727eaf9d46b07b", "underlying_calls_count": 15, @@ -182,18 +351,18 @@ }, "stage_batched": { "all_ds4_active": true, - "completion_tokens": 210, - "decode_ms": 9601.1, - "exposed_tool_wait_ms": 10159.580159001052, + "completion_tokens": 222, + "decode_ms": 8408.3, + "exposed_tool_wait_ms": 10154.79341000173, "final": { - "accept_rate": 0.6875, - "completion_tokens": 15, - "content_sha256": "e9618263bdfa8c2639cf7326931749b0625a0f5ca309f638c95ad621c5809303" + "accept_rate": 0.75, + "completion_tokens": 12, + "content_sha256": "1f19e61b2ef79fed2c87e8d945b3cc479796bd8a235bc79ef7579f3bbc22adb3" }, "final_correct": true, - "model_compute_ms": 70723.3, + "model_compute_ms": 77967.0, "model_turns": 6, - "task_ms": 80902.57541000028, + "task_ms": 88139.69260699378, "tool_results_count": 15, "tool_results_sha256": "2037fc8f56106ba9bc50763cc4b82c0f3262f0ffb8682ca5ca727eaf9d46b07b", "underlying_calls_count": 15, @@ -213,16 +382,16 @@ ], "compiled": { "all_ds4_active": true, - "completion_tokens": 55, - "decode_ms": 2136.2, - "exposed_tool_wait_ms": 10164.059007001924, + "completion_tokens": 73, + "decode_ms": 2993.8, + "exposed_tool_wait_ms": 10166.34874400188, "final": { - "accept_rate": 0.9375, - "completion_tokens": 19, - "content_sha256": "77d293cd086d290a81c7ba651034f8d95d944b3305f9956ae1dd9dae33671953" + "accept_rate": 0.75, + "completion_tokens": 16, + "content_sha256": "49780da241938f12bcf7a288b9ebc5fdd19be49df87cc9ada53ba48ee741cb5b" }, "final_correct": true, - "graph_wall_ms": 10165.094661002513, + "graph_wall_ms": 10167.359696002677, "macro_call": { "arguments": { "workflow_ref": "workflow_taskc" @@ -230,32 +399,116 @@ "name": "execute_customer_workflows" }, "macro_correct": true, - "model_compute_ms": 14691.8, + "model_compute_ms": 26871.0, "model_turns": 2, "prediction_hit": false, "prediction_reason": null, "prediction_source": null, "prediction_status": null, "predictor_ms": 0.0, - "task_ms": 24958.613470000273, + "task_ms": 37056.5853549997, "tool_results_count": 20, "tool_results_sha256": "8eb88b665ebcafe693c031c44e5c724a2f3aebe00f5e726008dc316050fa211c", "underlying_calls_count": 20, "underlying_calls_sha256": "467ec65a34e98b78d3d4a6990fef3a6292140cb019e9ec1635fd7b5c7aa9b3ef" }, + "interference_probe": { + "observations": [ + { + "compiled": { + "accept_rate": 0.8461538553237915, + "cache_hit": false, + "cached_prefix_tokens": 0, + "call_sha256": "8450d3e1b215a7ad9c3e3b04edd56c2b099ea8677ea008e6bd13863d5c9d025b", + "completion_tokens": 57, + "decode_ms": 2318.2, + "model_compute_ms": 12788.099999999999, + "prefill_ms": 10469.9 + }, + "order": [ + "compiled", + "speculative" + ], + "speculative": { + "accept_rate": 0.8461538553237915, + "cache_hit": false, + "cached_prefix_tokens": 0, + "call_sha256": "8450d3e1b215a7ad9c3e3b04edd56c2b099ea8677ea008e6bd13863d5c9d025b", + "completion_tokens": 57, + "decode_ms": 2291.2, + "model_compute_ms": 12780.7, + "prefill_ms": 10489.5 + } + }, + { + "compiled": { + "accept_rate": 0.8461538553237915, + "cache_hit": false, + "cached_prefix_tokens": 0, + "call_sha256": "8450d3e1b215a7ad9c3e3b04edd56c2b099ea8677ea008e6bd13863d5c9d025b", + "completion_tokens": 57, + "decode_ms": 2286.6, + "model_compute_ms": 12764.4, + "prefill_ms": 10477.8 + }, + "order": [ + "speculative", + "compiled" + ], + "speculative": { + "accept_rate": 0.8461538553237915, + "cache_hit": false, + "cached_prefix_tokens": 0, + "call_sha256": "8450d3e1b215a7ad9c3e3b04edd56c2b099ea8677ea008e6bd13863d5c9d025b", + "completion_tokens": 57, + "decode_ms": 2284.4, + "model_compute_ms": 12758.6, + "prefill_ms": 10474.2 + } + }, + { + "compiled": { + "accept_rate": 0.8461538553237915, + "cache_hit": false, + "cached_prefix_tokens": 0, + "call_sha256": "8450d3e1b215a7ad9c3e3b04edd56c2b099ea8677ea008e6bd13863d5c9d025b", + "completion_tokens": 57, + "decode_ms": 3337.4, + "model_compute_ms": 13834.699999999999, + "prefill_ms": 10497.3 + }, + "order": [ + "compiled", + "speculative" + ], + "speculative": { + "accept_rate": 0.8461538553237915, + "cache_hit": false, + "cached_prefix_tokens": 0, + "call_sha256": "8450d3e1b215a7ad9c3e3b04edd56c2b099ea8677ea008e6bd13863d5c9d025b", + "completion_tokens": 57, + "decode_ms": 2293.6, + "model_compute_ms": 12788.300000000001, + "prefill_ms": 10494.7 + } + } + ], + "qualified": true, + "repetitions": 3 + }, "pair_index": 2, "speculative": { "all_ds4_active": true, - "completion_tokens": 55, - "decode_ms": 2135.2, - "exposed_tool_wait_ms": 0.03236, + "completion_tokens": 73, + "decode_ms": 3002.2999999999997, + "exposed_tool_wait_ms": 0.029506, "final": { - "accept_rate": 0.9375, - "completion_tokens": 19, - "content_sha256": "77d293cd086d290a81c7ba651034f8d95d944b3305f9956ae1dd9dae33671953" + "accept_rate": 0.75, + "completion_tokens": 16, + "content_sha256": "49780da241938f12bcf7a288b9ebc5fdd19be49df87cc9ada53ba48ee741cb5b" }, "final_correct": true, - "graph_wall_ms": 10002.267649004352, + "graph_wall_ms": 10002.156002999982, "macro_call": { "arguments": { "workflow_ref": "workflow_taskc" @@ -263,14 +516,14 @@ "name": "execute_customer_workflows" }, "macro_correct": true, - "model_compute_ms": 14628.900000000001, + "model_compute_ms": 20779.4, "model_turns": 2, "prediction_hit": true, "prediction_reason": null, "prediction_source": "native-qwen3", "prediction_status": "hit", - "predictor_ms": 205.821132, - "task_ms": 14839.366328000324, + "predictor_ms": 206.06213, + "task_ms": 21002.575256003183, "tool_results_count": 20, "tool_results_sha256": "8eb88b665ebcafe693c031c44e5c724a2f3aebe00f5e726008dc316050fa211c", "underlying_calls_count": 20, @@ -278,18 +531,18 @@ }, "stage_batched": { "all_ds4_active": true, - "completion_tokens": 211, - "decode_ms": 9963.0, - "exposed_tool_wait_ms": 10172.528195005725, + "completion_tokens": 200, + "decode_ms": 6476.0, + "exposed_tool_wait_ms": 10170.707518991549, "final": { - "accept_rate": 0.9375, - "completion_tokens": 19, - "content_sha256": "77d293cd086d290a81c7ba651034f8d95d944b3305f9956ae1dd9dae33671953" + "accept_rate": 1.0, + "completion_tokens": 16, + "content_sha256": "49780da241938f12bcf7a288b9ebc5fdd19be49df87cc9ada53ba48ee741cb5b" }, "final_correct": true, - "model_compute_ms": 73757.7, + "model_compute_ms": 80176.6, "model_turns": 6, - "task_ms": 83947.50960399688, + "task_ms": 90366.67264700372, "tool_results_count": 20, "tool_results_sha256": "8eb88b665ebcafe693c031c44e5c724a2f3aebe00f5e726008dc316050fa211c", "underlying_calls_count": 20, @@ -309,16 +562,16 @@ ], "compiled": { "all_ds4_active": true, - "completion_tokens": 50, - "decode_ms": 2561.7, - "exposed_tool_wait_ms": 10127.542959999118, + "completion_tokens": 46, + "decode_ms": 2314.2, + "exposed_tool_wait_ms": 10128.93626299774, "final": { - "accept_rate": 0.9166666865348816, - "completion_tokens": 14, - "content_sha256": "b8f612f1f27ff9b32b2fb5d747b5b09e9ff0de8e4241ea1810893b522b673906" + "accept_rate": 1.0, + "completion_tokens": 11, + "content_sha256": "dd0930fae404b66f022f2c55ec2733331ee7a67d0aba2a63cd1437bf864c5c28" }, "final_correct": true, - "graph_wall_ms": 10128.030801002751, + "graph_wall_ms": 10129.702878002718, "macro_call": { "arguments": { "workflow_ref": "workflow_taskd" @@ -326,32 +579,116 @@ "name": "execute_customer_workflows" }, "macro_correct": true, - "model_compute_ms": 14129.5, + "model_compute_ms": 17697.1, "model_turns": 2, "prediction_hit": false, "prediction_reason": null, "prediction_source": null, "prediction_status": null, "predictor_ms": 0.0, - "task_ms": 24262.652850004088, + "task_ms": 27845.55858699605, "tool_results_count": 10, "tool_results_sha256": "c2041b2f6e3aeddaafa6ffb35bcd8affbc01e75cc62de4258206a53a6559a771", "underlying_calls_count": 10, "underlying_calls_sha256": "ff04b3b23c0147caffa852e474ed9f9fec517c227e12674dfacdba771bf34e33" }, + "interference_probe": { + "observations": [ + { + "compiled": { + "accept_rate": 0.9642857313156128, + "cache_hit": false, + "cached_prefix_tokens": 0, + "call_sha256": "f1e79cd991e36c3b41fca5f4ed7143a388d932406914df86b7e5b525295e279c", + "completion_tokens": 35, + "decode_ms": 1095.7, + "model_compute_ms": 10661.300000000001, + "prefill_ms": 9565.6 + }, + "order": [ + "speculative", + "compiled" + ], + "speculative": { + "accept_rate": 0.9642857313156128, + "cache_hit": false, + "cached_prefix_tokens": 0, + "call_sha256": "f1e79cd991e36c3b41fca5f4ed7143a388d932406914df86b7e5b525295e279c", + "completion_tokens": 35, + "decode_ms": 1088.0, + "model_compute_ms": 10659.5, + "prefill_ms": 9571.5 + } + }, + { + "compiled": { + "accept_rate": 0.9642857313156128, + "cache_hit": false, + "cached_prefix_tokens": 0, + "call_sha256": "f1e79cd991e36c3b41fca5f4ed7143a388d932406914df86b7e5b525295e279c", + "completion_tokens": 35, + "decode_ms": 1102.7, + "model_compute_ms": 10700.900000000001, + "prefill_ms": 9598.2 + }, + "order": [ + "compiled", + "speculative" + ], + "speculative": { + "accept_rate": 0.9642857313156128, + "cache_hit": false, + "cached_prefix_tokens": 0, + "call_sha256": "f1e79cd991e36c3b41fca5f4ed7143a388d932406914df86b7e5b525295e279c", + "completion_tokens": 35, + "decode_ms": 1089.7, + "model_compute_ms": 10701.7, + "prefill_ms": 9612.0 + } + }, + { + "compiled": { + "accept_rate": 0.9642857313156128, + "cache_hit": false, + "cached_prefix_tokens": 0, + "call_sha256": "f1e79cd991e36c3b41fca5f4ed7143a388d932406914df86b7e5b525295e279c", + "completion_tokens": 35, + "decode_ms": 1090.2, + "model_compute_ms": 10679.5, + "prefill_ms": 9589.3 + }, + "order": [ + "speculative", + "compiled" + ], + "speculative": { + "accept_rate": 0.9642857313156128, + "cache_hit": false, + "cached_prefix_tokens": 0, + "call_sha256": "f1e79cd991e36c3b41fca5f4ed7143a388d932406914df86b7e5b525295e279c", + "completion_tokens": 35, + "decode_ms": 1100.2, + "model_compute_ms": 10679.900000000001, + "prefill_ms": 9579.7 + } + } + ], + "qualified": true, + "repetitions": 3 + }, "pair_index": 3, "speculative": { "all_ds4_active": true, - "completion_tokens": 50, - "decode_ms": 2557.7, - "exposed_tool_wait_ms": 0.020318, + "completion_tokens": 46, + "decode_ms": 1348.0, + "exposed_tool_wait_ms": 0.021149, "final": { - "accept_rate": 0.9166666865348816, - "completion_tokens": 14, - "content_sha256": "b8f612f1f27ff9b32b2fb5d747b5b09e9ff0de8e4241ea1810893b522b673906" + "accept_rate": 1.0, + "completion_tokens": 11, + "content_sha256": "dd0930fae404b66f022f2c55ec2733331ee7a67d0aba2a63cd1437bf864c5c28" }, "final_correct": true, - "graph_wall_ms": 10002.225474003353, + "graph_wall_ms": 10002.26961899898, "macro_call": { "arguments": { "workflow_ref": "workflow_taskd" @@ -359,14 +696,14 @@ "name": "execute_customer_workflows" }, "macro_correct": true, - "model_compute_ms": 14066.4, + "model_compute_ms": 20244.3, "model_turns": 2, "prediction_hit": true, "prediction_reason": null, "prediction_source": "native-qwen3", "prediction_status": "hit", - "predictor_ms": 198.264642, - "task_ms": 14328.593565005576, + "predictor_ms": 199.397724, + "task_ms": 20496.583997002745, "tool_results_count": 10, "tool_results_sha256": "c2041b2f6e3aeddaafa6ffb35bcd8affbc01e75cc62de4258206a53a6559a771", "underlying_calls_count": 10, @@ -374,18 +711,18 @@ }, "stage_batched": { "all_ds4_active": true, - "completion_tokens": 204, - "decode_ms": 8685.0, - "exposed_tool_wait_ms": 10141.129875002662, + "completion_tokens": 195, + "decode_ms": 6618.0, + "exposed_tool_wait_ms": 10132.728425989626, "final": { - "accept_rate": 0.9166666865348816, - "completion_tokens": 14, - "content_sha256": "b8f612f1f27ff9b32b2fb5d747b5b09e9ff0de8e4241ea1810893b522b673906" + "accept_rate": 1.0, + "completion_tokens": 11, + "content_sha256": "dd0930fae404b66f022f2c55ec2733331ee7a67d0aba2a63cd1437bf864c5c28" }, "final_correct": true, - "model_compute_ms": 67694.4, + "model_compute_ms": 71275.7, "model_turns": 6, - "task_ms": 77853.58790800092, + "task_ms": 81427.70019899763, "tool_results_count": 10, "tool_results_sha256": "c2041b2f6e3aeddaafa6ffb35bcd8affbc01e75cc62de4258206a53a6559a771", "underlying_calls_count": 10, @@ -405,16 +742,16 @@ ], "compiled": { "all_ds4_active": true, - "completion_tokens": 54, - "decode_ms": 2277.8, - "exposed_tool_wait_ms": 10143.549353000708, + "completion_tokens": 50, + "decode_ms": 1653.7, + "exposed_tool_wait_ms": 10150.936931000615, "final": { - "accept_rate": 0.8125, - "completion_tokens": 17, - "content_sha256": "781a9894a33fb9de5e33c892e322d4181cd9a5838ea34bfacb5bcd6dda59e9e4" + "accept_rate": 0.9166666865348816, + "completion_tokens": 14, + "content_sha256": "752191b12e47a14803f099a3a92e2ffa50510c7002045c5dbcd0a3db939f53a8" }, "final_correct": true, - "graph_wall_ms": 10144.438636001723, + "graph_wall_ms": 10152.029717006371, "macro_call": { "arguments": { "workflow_ref": "workflow_taske" @@ -422,32 +759,116 @@ "name": "execute_customer_workflows" }, "macro_correct": true, - "model_compute_ms": 14573.5, + "model_compute_ms": 18391.899999999998, "model_turns": 2, "prediction_hit": false, "prediction_reason": null, "prediction_source": null, "prediction_status": null, "predictor_ms": 0.0, - "task_ms": 24722.972717005177, + "task_ms": 28562.6701939982, "tool_results_count": 15, "tool_results_sha256": "968113db6b1f63c186b8124472fb64ac38b8d81957a7d9a05898a41eb23af577", "underlying_calls_count": 15, "underlying_calls_sha256": "0be2134f957d10c601ed26e39380751ede7d2b05ec5419d36680e9f981c2ba57" }, + "interference_probe": { + "observations": [ + { + "compiled": { + "accept_rate": 1.0, + "cache_hit": false, + "cached_prefix_tokens": 0, + "call_sha256": "1212a82be816acf93594e58cc76a9350c85d83b9dfc40c8021865dbd9e92e780", + "completion_tokens": 36, + "decode_ms": 2153.3, + "model_compute_ms": 12313.2, + "prefill_ms": 10159.9 + }, + "order": [ + "compiled", + "speculative" + ], + "speculative": { + "accept_rate": 1.0, + "cache_hit": false, + "cached_prefix_tokens": 0, + "call_sha256": "1212a82be816acf93594e58cc76a9350c85d83b9dfc40c8021865dbd9e92e780", + "completion_tokens": 36, + "decode_ms": 1144.8, + "model_compute_ms": 11316.0, + "prefill_ms": 10171.2 + } + }, + { + "compiled": { + "accept_rate": 1.0, + "cache_hit": false, + "cached_prefix_tokens": 0, + "call_sha256": "1212a82be816acf93594e58cc76a9350c85d83b9dfc40c8021865dbd9e92e780", + "completion_tokens": 36, + "decode_ms": 1142.1, + "model_compute_ms": 11324.6, + "prefill_ms": 10182.5 + }, + "order": [ + "speculative", + "compiled" + ], + "speculative": { + "accept_rate": 1.0, + "cache_hit": false, + "cached_prefix_tokens": 0, + "call_sha256": "1212a82be816acf93594e58cc76a9350c85d83b9dfc40c8021865dbd9e92e780", + "completion_tokens": 36, + "decode_ms": 1135.1, + "model_compute_ms": 11336.5, + "prefill_ms": 10201.4 + } + }, + { + "compiled": { + "accept_rate": 1.0, + "cache_hit": false, + "cached_prefix_tokens": 0, + "call_sha256": "1212a82be816acf93594e58cc76a9350c85d83b9dfc40c8021865dbd9e92e780", + "completion_tokens": 36, + "decode_ms": 1137.8, + "model_compute_ms": 11313.9, + "prefill_ms": 10176.1 + }, + "order": [ + "compiled", + "speculative" + ], + "speculative": { + "accept_rate": 1.0, + "cache_hit": false, + "cached_prefix_tokens": 0, + "call_sha256": "1212a82be816acf93594e58cc76a9350c85d83b9dfc40c8021865dbd9e92e780", + "completion_tokens": 36, + "decode_ms": 1142.4, + "model_compute_ms": 11314.699999999999, + "prefill_ms": 10172.3 + } + } + ], + "qualified": true, + "repetitions": 3 + }, "pair_index": 4, "speculative": { "all_ds4_active": true, - "completion_tokens": 54, - "decode_ms": 2283.9, - "exposed_tool_wait_ms": 0.021269, + "completion_tokens": 50, + "decode_ms": 1975.0, + "exposed_tool_wait_ms": 0.035947, "final": { - "accept_rate": 0.8125, - "completion_tokens": 17, - "content_sha256": "781a9894a33fb9de5e33c892e322d4181cd9a5838ea34bfacb5bcd6dda59e9e4" + "accept_rate": 0.9166666865348816, + "completion_tokens": 14, + "content_sha256": "752191b12e47a14803f099a3a92e2ffa50510c7002045c5dbcd0a3db939f53a8" }, "final_correct": true, - "graph_wall_ms": 10001.9813550025, + "graph_wall_ms": 10001.882305004983, "macro_call": { "arguments": { "workflow_ref": "workflow_taske" @@ -455,14 +876,14 @@ "name": "execute_customer_workflows" }, "macro_correct": true, - "model_compute_ms": 14529.800000000001, + "model_compute_ms": 24415.8, "model_turns": 2, "prediction_hit": true, "prediction_reason": null, "prediction_source": "native-qwen3", "prediction_status": "hit", - "predictor_ms": 244.327376, - "task_ms": 14804.260248994979, + "predictor_ms": 207.393702, + "task_ms": 24634.228317001543, "tool_results_count": 15, "tool_results_sha256": "968113db6b1f63c186b8124472fb64ac38b8d81957a7d9a05898a41eb23af577", "underlying_calls_count": 15, @@ -470,18 +891,18 @@ }, "stage_batched": { "all_ds4_active": true, - "completion_tokens": 213, - "decode_ms": 8830.1, - "exposed_tool_wait_ms": 10159.158977992774, + "completion_tokens": 202, + "decode_ms": 6633.0, + "exposed_tool_wait_ms": 10162.887673999649, "final": { - "accept_rate": 0.8125, - "completion_tokens": 17, - "content_sha256": "781a9894a33fb9de5e33c892e322d4181cd9a5838ea34bfacb5bcd6dda59e9e4" + "accept_rate": 0.9166666865348816, + "completion_tokens": 14, + "content_sha256": "752191b12e47a14803f099a3a92e2ffa50510c7002045c5dbcd0a3db939f53a8" }, "final_correct": true, - "model_compute_ms": 70980.1, + "model_compute_ms": 77673.4, "model_turns": 6, - "task_ms": 81157.67812900594, + "task_ms": 87855.36901299929, "tool_results_count": 15, "tool_results_sha256": "968113db6b1f63c186b8124472fb64ac38b8d81957a7d9a05898a41eb23af577", "underlying_calls_count": 15, @@ -501,16 +922,16 @@ ], "compiled": { "all_ds4_active": true, - "completion_tokens": 46, - "decode_ms": 3468.2, - "exposed_tool_wait_ms": 10165.250458005175, + "completion_tokens": 50, + "decode_ms": 1986.2, + "exposed_tool_wait_ms": 10164.91345599934, "final": { - "accept_rate": 0.5833333134651184, - "completion_tokens": 10, - "content_sha256": "4323774d53ee50de45337a0ce501766a1b32b5398ac8d64044006ed6f5143ef7" + "accept_rate": 0.6875, + "completion_tokens": 15, + "content_sha256": "ecb16885e5e4c6b794e1f17f503d17c6dd38105056884d943ac32afb2caa7c0a" }, "final_correct": true, - "graph_wall_ms": 10166.563869002857, + "graph_wall_ms": 10166.974724998, "macro_call": { "arguments": { "workflow_ref": "workflow_taskf" @@ -518,32 +939,116 @@ "name": "execute_customer_workflows" }, "macro_correct": true, - "model_compute_ms": 16199.5, + "model_compute_ms": 26039.8, "model_turns": 2, "prediction_hit": false, "prediction_reason": null, "prediction_source": null, "prediction_status": null, "predictor_ms": 0.0, - "task_ms": 26370.94549000176, + "task_ms": 36215.08621699468, "tool_results_count": 20, "tool_results_sha256": "de71e45a390fca728e07a8bf4cdc467e91604fc52f68abe6ed0bd75ca77e87d2", "underlying_calls_count": 20, "underlying_calls_sha256": "bd91a2316a76490265d64557860d9a2acdc97a8f1a18014dacaa0a21cd241e2f" }, + "interference_probe": { + "observations": [ + { + "compiled": { + "accept_rate": 0.9642857313156128, + "cache_hit": false, + "cached_prefix_tokens": 0, + "call_sha256": "bb4924fb407a01a832f9787a134f2e9494c39afaa14fe4541e5d693a73b9845d", + "completion_tokens": 35, + "decode_ms": 1161.6, + "model_compute_ms": 11761.300000000001, + "prefill_ms": 10599.7 + }, + "order": [ + "speculative", + "compiled" + ], + "speculative": { + "accept_rate": 0.9642857313156128, + "cache_hit": false, + "cached_prefix_tokens": 0, + "call_sha256": "bb4924fb407a01a832f9787a134f2e9494c39afaa14fe4541e5d693a73b9845d", + "completion_tokens": 35, + "decode_ms": 1167.4, + "model_compute_ms": 11739.8, + "prefill_ms": 10572.4 + } + }, + { + "compiled": { + "accept_rate": 0.9642857313156128, + "cache_hit": false, + "cached_prefix_tokens": 0, + "call_sha256": "bb4924fb407a01a832f9787a134f2e9494c39afaa14fe4541e5d693a73b9845d", + "completion_tokens": 35, + "decode_ms": 1179.7, + "model_compute_ms": 11772.800000000001, + "prefill_ms": 10593.1 + }, + "order": [ + "compiled", + "speculative" + ], + "speculative": { + "accept_rate": 0.9642857313156128, + "cache_hit": false, + "cached_prefix_tokens": 0, + "call_sha256": "bb4924fb407a01a832f9787a134f2e9494c39afaa14fe4541e5d693a73b9845d", + "completion_tokens": 35, + "decode_ms": 1172.7, + "model_compute_ms": 11755.5, + "prefill_ms": 10582.8 + } + }, + { + "compiled": { + "accept_rate": 0.9642857313156128, + "cache_hit": false, + "cached_prefix_tokens": 0, + "call_sha256": "bb4924fb407a01a832f9787a134f2e9494c39afaa14fe4541e5d693a73b9845d", + "completion_tokens": 35, + "decode_ms": 1164.9, + "model_compute_ms": 11746.9, + "prefill_ms": 10582.0 + }, + "order": [ + "speculative", + "compiled" + ], + "speculative": { + "accept_rate": 0.9642857313156128, + "cache_hit": false, + "cached_prefix_tokens": 0, + "call_sha256": "bb4924fb407a01a832f9787a134f2e9494c39afaa14fe4541e5d693a73b9845d", + "completion_tokens": 35, + "decode_ms": 1165.9, + "model_compute_ms": 11780.699999999999, + "prefill_ms": 10614.8 + } + } + ], + "qualified": true, + "repetitions": 3 + }, "pair_index": 5, "speculative": { "all_ds4_active": true, - "completion_tokens": 46, - "decode_ms": 2451.7, - "exposed_tool_wait_ms": 0.032681, + "completion_tokens": 50, + "decode_ms": 1985.9, + "exposed_tool_wait_ms": 0.033042, "final": { - "accept_rate": 0.5833333134651184, - "completion_tokens": 10, - "content_sha256": "4323774d53ee50de45337a0ce501766a1b32b5398ac8d64044006ed6f5143ef7" + "accept_rate": 0.6875, + "completion_tokens": 15, + "content_sha256": "ecb16885e5e4c6b794e1f17f503d17c6dd38105056884d943ac32afb2caa7c0a" }, "final_correct": true, - "graph_wall_ms": 10002.300546002516, + "graph_wall_ms": 10002.439897994918, "macro_call": { "arguments": { "workflow_ref": "workflow_taskf" @@ -551,14 +1056,14 @@ "name": "execute_customer_workflows" }, "macro_correct": true, - "model_compute_ms": 15157.8, + "model_compute_ms": 20233.4, "model_turns": 2, "prediction_hit": true, "prediction_reason": null, "prediction_source": "native-qwen3", "prediction_status": "hit", - "predictor_ms": 214.897278, - "task_ms": 15378.521895996528, + "predictor_ms": 201.386338, + "task_ms": 20451.380637998227, "tool_results_count": 20, "tool_results_sha256": "de71e45a390fca728e07a8bf4cdc467e91604fc52f68abe6ed0bd75ca77e87d2", "underlying_calls_count": 20, @@ -566,18 +1071,18 @@ }, "stage_batched": { "all_ds4_active": true, - "completion_tokens": 191, - "decode_ms": 11077.9, - "exposed_tool_wait_ms": 10172.786328992515, + "completion_tokens": 261, + "decode_ms": 10547.4, + "exposed_tool_wait_ms": 10179.923661002249, "final": { - "accept_rate": 0.5833333134651184, - "completion_tokens": 10, - "content_sha256": "4323774d53ee50de45337a0ce501766a1b32b5398ac8d64044006ed6f5143ef7" + "accept_rate": 0.9166666865348816, + "completion_tokens": 15, + "content_sha256": "ecb16885e5e4c6b794e1f17f503d17c6dd38105056884d943ac32afb2caa7c0a" }, "final_correct": true, - "model_compute_ms": 75437.0, + "model_compute_ms": 86333.0, "model_turns": 6, - "task_ms": 85663.01394799666, + "task_ms": 96537.8686490003, "tool_results_count": 20, "tool_results_sha256": "de71e45a390fca728e07a8bf4cdc467e91604fc52f68abe6ed0bd75ca77e87d2", "underlying_calls_count": 20, @@ -650,12 +1155,20 @@ "tool": "prepare_customer_summary" } ], - "training_report": "/home/lucebox5/tool-spec-cpu-20260813/results/multiturn-cached-wordref-production-6tasks.json", + "training_report": "results/multiturn-cached-wordref-production-6tasks.json", "training_report_sha256": "2475697d418bffed0e9668da26ce6c88a85a952ce97d99749f440f97f9ac5bf9", "training_traces": 2, - "workflow_registry": "/home/lucebox5/tool-spec-cpu-20260813/results/trace-workflow-registry.json", + "workflow_registry": "results/trace-workflow-registry.json", "workflow_registry_sha256": "0b73cb4f45284f07a4d6890906d1eaf5a25469c2b80eb542b1d74f1e3a9a0e1b" }, + "privacy_miss": { + "authoritative_call_sha256": "26b31bf2044cd34ec9ea91d14b9758544b6a4a62a533a58bc4f79af1472a6535", + "passed": true, + "prediction_sha256": "4fd23541e355718c0b1ee47f20028d7324bab11dddd5a0a5a460495dd05e3461", + "private_result_exposed": false, + "reason": "invocation_mismatch", + "status": "miss" + }, "production_gate": { "checks": { "calls_stable": true, @@ -667,6 +1180,7 @@ "end_to_end_tail": true, "final_answers_correct": true, "final_outputs_stable": true, + "interference_probes": true, "macro_calls_correct": true, "macro_outputs_stable": true, "model_slowdown_p50": true, @@ -674,6 +1188,7 @@ "prediction_hit_rate": true, "prediction_source": true, "prefix_cache_configured": true, + "private_miss_result_hidden": true, "sample_size": true, "speculation_incremental_ci": true, "speculation_incremental_gain": true, @@ -718,13 +1233,13 @@ "resolve_customer" ], "automatic_prediction_enabled": true, + "client_prediction_required": false, + "client_result_handling_required": true, "compute_isolation": "disjoint_cpu_affinity", "cpu_affinity_isolated": true, "enabled": true, "execution_mode": "child_process_cpu_affinity", "executor_contract": "child_process_cpu_affinity", - "hip_reserved_tool_compute_units": 0, - "hip_tool_device": null, "max_model_slowdown_ratio": 1.05, "model_cpu_affinity": [ 0, @@ -756,8 +1271,6 @@ 28, 29 ], - "model_expert_ownership_unique": false, - "model_routing_static": false, "prediction_confidence": 0.75, "prediction_source": "native-qwen3", "predictor_decode_isolated": true, @@ -768,14 +1281,11 @@ "accelerator_relation": "non_accelerator", "decode_interference_qualified": true, "model_slowdown_ratio": 1.0011341375399527, - "requires_static_model_routing": false, - "requires_unique_expert_ownership": false, "resource_percentage": 100 } ], "profile_status": "qualified", "protocol": "dflash.tool-speculation.v1", - "requires_client_support": false, "tool_cpu_affinity": [ 14, 15, @@ -790,6 +1300,7 @@ "all_ds4_active": true, "all_final_answers_correct": true, "all_final_outputs_stable": true, + "all_interference_probes_qualified": true, "all_macro_calls_correct": true, "all_predictions_from_qwen": true, "all_tool_results_stable": true, @@ -801,66 +1312,67 @@ 15, 20 ], - "compiled_exposed_tool_wait_p50_ms": 10143.342030001804, + "compiled_exposed_tool_wait_p50_ms": 10148.645039000257, "compiled_model_turns_p50": 2.0, - "compiled_task_p50_ms": 24575.12643300288, + "compiled_task_p50_ms": 32876.98137099869, "compiled_to_speculative_bootstrap_95ci": [ - 1.6759547511337496, - 1.7210318416877688 + 1.2590085918648417, + 1.800192103452801 ], - "compiled_to_speculative_speedup_p05": 1.6729725830674844, - "compiled_to_speculative_speedup_p50": 1.6953781778482404, - "continuation_cache_hit_rate": 0.0, - "decode_slowdown_p50_percent": -0.10147920266865285, - "decode_slowdown_p95_percent": 0.21871282872425457, + "compiled_to_speculative_speedup_p05": 1.2092397320590198, + "compiled_to_speculative_speedup_p50": 1.7675861222850164, + "continuation_cache_hit_rate": 0.14285714285714285, + "decode_slowdown_p50_percent": -0.6578265762836599, + "decode_slowdown_p95_percent": 0.3737683368205902, "macro_output_stability_rate": 1.0, - "model_compute_slowdown_p50_percent": -0.4580005364335671, - "model_compute_slowdown_p95_percent": -0.3319269946081671, + "model_compute_slowdown_p50_percent": -0.02706040193529713, + "model_compute_slowdown_p95_percent": 0.09042853676124452, "pattern_prediction_hit_rate": 1.0, - "predictor_p50_ms": 203.4977505, + "predictor_p50_ms": 201.023082, "prefix_cache_configured": true, "prefix_cache_lifetime_hit_delta": 0, - "speculative_exposed_tool_wait_p50_ms": 0.026814499999999998, - "speculative_task_p50_ms": 14597.403696498077, + "private_miss_result_hidden": true, + "speculative_exposed_tool_wait_p50_ms": 0.0303775, + "speculative_task_p50_ms": 20473.982317500486, "speedup_by_call_count": { "10": { - "combined_speedup_p50": 5.548099679951088, - "compiled_task_p50_ms": 24024.929789502494, - "speculation_speedup_p50": 1.7102881019726668, - "speculative_task_p50_ms": 14050.065913499566, - "stage_batched_task_p50_ms": 77919.23097700055, + "combined_speedup_p50": 4.4513637744591055, + "compiled_task_p50_ms": 29030.47904399864, + "speculation_speedup_p50": 1.589033888881625, + "speculative_task_p50_ms": 18551.40907099849, + "stage_batched_task_p50_ms": 81648.07293250124, "tasks": 2 }, "15": { - "combined_speedup_p50": 5.551986886996318, - "compiled_task_p50_ms": 24575.12643300288, - "speculation_speedup_p50": 1.683721802277213, - "speculative_task_p50_ms": 14597.403696498077, - "stage_batched_task_p50_ms": 81030.12676950311, + "combined_speedup_p50": 3.9915685175553506, + "compiled_task_p50_ms": 32050.616717497178, + "speculation_speedup_p50": 1.470166806436018, + "speculative_task_p50_ms": 22295.020493002085, + "stage_batched_task_p50_ms": 87997.53080999653, "tasks": 2 }, "20": { - "combined_speedup_p50": 5.613692001294089, - "compiled_task_p50_ms": 25664.779480001016, - "speculation_speedup_p50": 1.698354866419879, - "speculative_task_p50_ms": 15108.944111998426, - "stage_batched_task_p50_ms": 84805.26177599677, + "combined_speedup_p50": 4.511503368783014, + "compiled_task_p50_ms": 36635.83578599719, + "speculation_speedup_p50": 1.7675861222850164, + "speculative_task_p50_ms": 20726.977947000705, + "stage_batched_task_p50_ms": 93452.27064800201, "tasks": 2 } }, - "stage_batched_exposed_tool_wait_p50_ms": 10159.369568496913, + "stage_batched_exposed_tool_wait_p50_ms": 10158.84054200069, "stage_batched_model_turns_p50": 6.0, - "stage_batched_task_p50_ms": 81030.12676950311, - "stage_batched_to_compiled_speedup_p50": 3.2805602399674787, + "stage_batched_task_p50_ms": 87997.53080999653, + "stage_batched_to_compiled_speedup_p50": 2.687587552693997, "stage_batched_to_speculative_bootstrap_95ci": [ - 5.457745637116071, - 5.659919390842823 + 3.769569566480701, + 4.8251710527069305 ], - "stage_batched_to_speculative_speedup_min": 5.433442406946423, - "stage_batched_to_speculative_speedup_p05": 5.445594022031247, - "stage_batched_to_speculative_speedup_p50": 5.5961135402826, + "stage_batched_to_speculative_speedup_min": 3.5663942008836984, + "stage_batched_to_speculative_speedup_p05": 3.6679818836822, + "stage_batched_to_speculative_speedup_p50": 4.359695041609839, "tasks": 6, - "total_wall_speedup": 5.570717496895011 + "total_wall_speedup": 4.272913990402306 }, "workload": { "branches_per_task": "2-4", diff --git a/optimizations/ooo_spec_lucebox5_cpu/results/trace-compiled-training-traces.json b/optimizations/ooo_spec_lucebox5_cpu/results/trace-compiled-training-traces.json deleted file mode 100644 index 089220994..000000000 --- a/optimizations/ooo_spec_lucebox5_cpu/results/trace-compiled-training-traces.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "schema_version": 1, - "source": "successful read-only workflow traces", - "traces": [ - { - "root": { - "customer_email": "agent-benchmark-2@example.test", - "destination": "Turin" - }, - "calls": [ - {"name": "resolve_customer", "arguments": {"customer_email": "agent-benchmark-2@example.test"}}, - {"name": "list_open_orders", "arguments": {"customer_ref": "plum"}}, - {"name": "get_order_details", "arguments": {"orders_ref": "jade"}}, - {"name": "calculate_shipping", "arguments": {"destination": "Turin", "order_ref": "amber"}}, - {"name": "prepare_customer_summary", "arguments": {"shipping_ref": "amber"}} - ], - "results": [ - {"call_ref": "plum", "call_sha256": "93824d1968f2c3ab058a3ef69d2625c0fea88fad3b27b8c30f95418fb83ae6cf", "tool_name": "resolve_customer", "side_effects": false}, - {"call_ref": "jade", "call_sha256": "772a4b9fde9e69fb1da323ba4c22e7a7d01061c0600c133e74de7bd73bba931f", "tool_name": "list_open_orders", "side_effects": false}, - {"call_ref": "amber", "call_sha256": "428dbfc05f17e7a9295c2b4741dbae2bb91d3f107150b77bb34c6ca25bd8af23", "tool_name": "get_order_details", "side_effects": false}, - {"call_ref": "amber", "call_sha256": "21ec3c60e83bfbc0ab4e6e8356a5c87e7210ffd16b53ca8299017261d97f70a1", "tool_name": "calculate_shipping", "side_effects": false}, - {"call_ref": "ivory", "call_sha256": "adaa761561a84b7e457acb7881e856285fcdccf21a6d82751dd2bea984b1d6d0", "tool_name": "prepare_customer_summary", "side_effects": false} - ] - }, - { - "root": { - "customer_email": "agent-benchmark-5@example.test", - "destination": "Naples" - }, - "calls": [ - {"name": "resolve_customer", "arguments": {"customer_email": "agent-benchmark-5@example.test"}}, - {"name": "list_open_orders", "arguments": {"customer_ref": "maple"}}, - {"name": "get_order_details", "arguments": {"orders_ref": "willow"}}, - {"name": "calculate_shipping", "arguments": {"destination": "Naples", "order_ref": "olive"}}, - {"name": "prepare_customer_summary", "arguments": {"shipping_ref": "cedar"}} - ], - "results": [ - {"call_ref": "maple", "call_sha256": "71063336b1901dd2a0050d4a7a0ebe52ad0b0755f5af079efe22db9c1df00b29", "tool_name": "resolve_customer", "side_effects": false}, - {"call_ref": "willow", "call_sha256": "8da0147e626bec41f9bafd2ec61d07d01a33e176990483fd62a2b93eeea82e2f", "tool_name": "list_open_orders", "side_effects": false}, - {"call_ref": "olive", "call_sha256": "66ea7a270ffff62c364786d69dc72144b0b0eb928c262f682dff72635964c6a9", "tool_name": "get_order_details", "side_effects": false}, - {"call_ref": "cedar", "call_sha256": "db784ed2c9e33569d9c6bdd36e6963076032cd8d0bf48cf954420a843cb8d593", "tool_name": "calculate_shipping", "side_effects": false}, - {"call_ref": "amber", "call_sha256": "6ba7d6c0326fbe23206f51a9a4268beb41a3050097565421946c4ac32900f9bc", "tool_name": "prepare_customer_summary", "side_effects": false} - ] - } - ] -} diff --git a/optimizations/ooo_spec_lucebox5_cpu/results/trace-workflow-registry.json b/optimizations/ooo_spec_lucebox5_cpu/results/trace-workflow-registry.json new file mode 100644 index 000000000..cd42484d2 --- /dev/null +++ b/optimizations/ooo_spec_lucebox5_cpu/results/trace-workflow-registry.json @@ -0,0 +1,121 @@ +{ + "pattern_fingerprint": "06d95882b485daf3f30f3fc12ea62ccfd18656de9deafa2f4ed77c49bb8c0645", + "schema_version": 1, + "workflows": { + "workflow_taska": { + "items": [ + { + "customer_email": "agent-taska-taska@example.test", + "destination": "Rome" + }, + { + "customer_email": "agent-taska-taskb@example.test", + "destination": "Milan" + } + ], + "pattern_fingerprint": "06d95882b485daf3f30f3fc12ea62ccfd18656de9deafa2f4ed77c49bb8c0645" + }, + "workflow_taskb": { + "items": [ + { + "customer_email": "agent-taskb-taskcw@example.test", + "destination": "Naples" + }, + { + "customer_email": "agent-taskb-taskcx@example.test", + "destination": "Rome" + }, + { + "customer_email": "agent-taskb-taskcy@example.test", + "destination": "Milan" + } + ], + "pattern_fingerprint": "06d95882b485daf3f30f3fc12ea62ccfd18656de9deafa2f4ed77c49bb8c0645" + }, + "workflow_taskc": { + "items": [ + { + "customer_email": "agent-taskc-taskgs@example.test", + "destination": "Florence" + }, + { + "customer_email": "agent-taskc-taskgt@example.test", + "destination": "Naples" + }, + { + "customer_email": "agent-taskc-taskgv@example.test", + "destination": "Milan" + }, + { + "customer_email": "agent-taskc-taskhb@example.test", + "destination": "Milan" + } + ], + "pattern_fingerprint": "06d95882b485daf3f30f3fc12ea62ccfd18656de9deafa2f4ed77c49bb8c0645" + }, + "workflow_taskd": { + "items": [ + { + "customer_email": "agent-taskd-taskko@example.test", + "destination": "Bologna" + }, + { + "customer_email": "agent-taskd-taskkp@example.test", + "destination": "Florence" + } + ], + "pattern_fingerprint": "06d95882b485daf3f30f3fc12ea62ccfd18656de9deafa2f4ed77c49bb8c0645" + }, + "workflow_taske": { + "items": [ + { + "customer_email": "agent-taske-taskok@example.test", + "destination": "Turin" + }, + { + "customer_email": "agent-taske-taskom@example.test", + "destination": "Florence" + }, + { + "customer_email": "agent-taske-taskon@example.test", + "destination": "Naples" + } + ], + "pattern_fingerprint": "06d95882b485daf3f30f3fc12ea62ccfd18656de9deafa2f4ed77c49bb8c0645" + }, + "workflow_taskf": { + "items": [ + { + "customer_email": "agent-taskf-tasksg@example.test", + "destination": "Milan" + }, + { + "customer_email": "agent-taskf-tasksh@example.test", + "destination": "Turin" + }, + { + "customer_email": "agent-taskf-tasksi@example.test", + "destination": "Bologna" + }, + { + "customer_email": "agent-taskf-tasksz@example.test", + "destination": "Turin" + } + ], + "pattern_fingerprint": "06d95882b485daf3f30f3fc12ea62ccfd18656de9deafa2f4ed77c49bb8c0645" + }, + "workflow_warmalm": { + "items": [ + { + "customer_email": "agent-warmalm-taska@example.test", + "destination": "Turin" + }, + { + "customer_email": "agent-warmalm-taskb@example.test", + "destination": "Bologna" + } + ], + "pattern_fingerprint": "06d95882b485daf3f30f3fc12ea62ccfd18656de9deafa2f4ed77c49bb8c0645" + } + } +} diff --git a/optimizations/ooo_spec_lucebox5_cpu/run_native_cpu_server_lucebox5.sh b/optimizations/ooo_spec_lucebox5_cpu/run_native_cpu_server_lucebox5.sh index 2e067aab9..4d876782f 100755 --- a/optimizations/ooo_spec_lucebox5_cpu/run_native_cpu_server_lucebox5.sh +++ b/optimizations/ooo_spec_lucebox5_cpu/run_native_cpu_server_lucebox5.sh @@ -8,19 +8,36 @@ executor="${TOOL_SPEC_EXECUTOR:-$experiment/cpu_sparse_tool_executor}" profile="${TOOL_SPEC_PROFILE:-$experiment/profiles/lucebox5-cpu-lane-qualified.json}" allowed="${TOOL_SPEC_ALLOW:-benchmark_cpu_sparse}" native_wrapper_dir="${NATIVE_WRAPPER_DIR:-$experiment/native-wrapper}" -candidate_build="${CANDIDATE_BUILD:-$experiment/engine-ooo-spec/server/build-hip-dual}" -predictor_model="${PREDICTOR_MODEL:-$experiment/models/Qwen3-0.6B-Q8_0.gguf}" +candidate_link="$native_wrapper_dir/candidate-build" +predictor_model="$experiment/models/Qwen3-0.6B-Q8_0.gguf" -for required in \ +[[ -L "$candidate_link" ]] || { + printf 'missing durable candidate-build symlink: %s\n' "$candidate_link" >&2 + exit 2 +} +if ! candidate_build="$(readlink -f -- "$candidate_link")"; then + printf 'cannot resolve candidate-build symlink: %s\n' "$candidate_link" >&2 + exit 2 +fi +[[ -n "$candidate_build" && -d "$candidate_build" ]] || { + printf 'invalid candidate-build symlink: %s\n' "$candidate_link" >&2 + exit 2 +} + +for binary in \ "$launcher" \ "$executor" \ - "$profile" \ "$native_wrapper_dir/dflash_server" \ "$candidate_build/dflash_server" \ - "$candidate_build/backend_ipc_daemon" \ - "$predictor_model"; do - [[ -e "$required" ]] || { - printf 'missing required path: %s\n' "$required" >&2 + "$candidate_build/backend_ipc_daemon"; do + [[ -f "$binary" && -x "$binary" ]] || { + printf 'required binary is not executable: %s\n' "$binary" >&2 + exit 2 + } +done +for data_file in "$profile" "$predictor_model"; do + [[ -f "$data_file" ]] || { + printf 'required data file is not regular: %s\n' "$data_file" >&2 exit 2 } done @@ -40,14 +57,6 @@ exec env \ PATH="$root/.local/bin:/opt/rocm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \ ENGINE_DIR="$root/lucebox-engine-0731" \ BUILD_DIR="$native_wrapper_dir" \ - CANDIDATE_BUILD="$candidate_build" \ - PREDICTOR_MODEL="$predictor_model" \ - PREDICTOR_GPU="${PREDICTOR_GPU:-1}" \ - PREDICTOR_MAX_CTX="${PREDICTOR_MAX_CTX:-4096}" \ - PREDICTOR_MAX_TOKENS="${PREDICTOR_MAX_TOKENS:-256}" \ - PREDICTOR_CONFIDENCE="${PREDICTOR_CONFIDENCE:-0.75}" \ - PREDICTOR_SCHEDULE="${PREDICTOR_SCHEDULE:-before-model}" \ - PREFIX_CACHE_SLOTS_OVERRIDE="${PREFIX_CACHE_SLOTS_OVERRIDE:-}" \ QUALIFIED_CONFIG_DIR="/opt/lucebox-manage/qualified/r9700_deepseek/runtime-config" \ TARGET_MODEL="$root/lucebox-models/DeepSeek-V4-Flash-0731-ROCMFPX-MIX-STRIX.gguf" \ DRAFT_MODEL="$root/lucebox-models/DeepSeek-V4-Flash-0731-DSpark-draft-Q4RMFP4-denseF16.gguf" \ diff --git a/optimizations/ooo_spec_lucebox5_cpu/test_benchmark_cpu_tool_speculation.py b/optimizations/ooo_spec_lucebox5_cpu/test_benchmark_cpu_tool_speculation.py index 8e03e399b..3506248d0 100644 --- a/optimizations/ooo_spec_lucebox5_cpu/test_benchmark_cpu_tool_speculation.py +++ b/optimizations/ooo_spec_lucebox5_cpu/test_benchmark_cpu_tool_speculation.py @@ -10,6 +10,7 @@ parse_cpu_list, percentile, props_url, + require_qualified_cpu_tool_props, request_body, ) @@ -77,6 +78,37 @@ def test_normalizes_deepseek_single_parameter_envelope(self) -> None: }, ) + def test_automatic_gate_requires_qualified_disjoint_lane(self) -> None: + args = argparse.Namespace(tool_cpus=[14, 15]) + tool_props = { + "enabled": True, + "automatic_prediction_enabled": True, + "execution_mode": "child_process_cpu_affinity", + "profile_status": "qualified", + "compute_isolation": "disjoint_cpu_affinity", + "cpu_affinity_isolated": True, + "preserves_token_speculation": True, + "tool_cpu_affinity": [14, 15], + "model_cpu_affinity": [0, 1], + } + self.assertIs( + require_qualified_cpu_tool_props( + {"tool_speculation": tool_props}, args, automatic=True + ), + tool_props, + ) + with self.assertRaisesRegex(SystemExit, "profile_status"): + require_qualified_cpu_tool_props( + { + "tool_speculation": { + **tool_props, + "profile_status": "unqualified", + } + }, + args, + automatic=True, + ) + if __name__ == "__main__": unittest.main() diff --git a/optimizations/ooo_spec_lucebox5_cpu/test_benchmark_trace_compiled_workflows.py b/optimizations/ooo_spec_lucebox5_cpu/test_benchmark_trace_compiled_workflows.py index 7f4ff69a2..7f635d20a 100644 --- a/optimizations/ooo_spec_lucebox5_cpu/test_benchmark_trace_compiled_workflows.py +++ b/optimizations/ooo_spec_lucebox5_cpu/test_benchmark_trace_compiled_workflows.py @@ -11,9 +11,11 @@ alphabetic_identifier, compact_arm, final_answer_correct, + interference_probe_qualified, load_training_traces, load_partial_pairs, make_task, + measure_private_miss, mine_pattern, model_observation, parse_request_customers, @@ -293,6 +295,8 @@ def test_production_gate_requires_two_x_and_exact_outputs(self) -> None: "compiled_to_speculative_bootstrap_95ci": [1.01, 1.2], "pattern_prediction_hit_rate": 1.0, "all_predictions_from_qwen": True, + "private_miss_result_hidden": True, + "all_interference_probes_qualified": True, "model_compute_slowdown_p50_percent": 0.0, "model_compute_slowdown_p95_percent": 0.0, "decode_slowdown_p50_percent": 0.0, @@ -323,22 +327,80 @@ def test_production_gate_requires_two_x_and_exact_outputs(self) -> None: all(value for key, value in checks.items() if key != "end_to_end_speedup") ) - def test_final_turn_is_identical_and_context_free_for_every_arm(self) -> None: + def test_interference_probe_accepts_matched_cache_misses(self) -> None: + observation = { + "cache_hit": False, + "cached_prefix_tokens": 0, + "completion_tokens": 35, + "call_sha256": "same-call", + "accept_rate": 0.9, + } + probe = { + "compiled": dict(observation), + "speculative": dict(observation), + } + self.assertTrue(interference_probe_qualified(probe)) + probe["speculative"]["cached_prefix_tokens"] = 128 + self.assertFalse(interference_probe_qualified(probe)) + + def test_private_miss_requires_no_result_in_engine_metadata(self) -> None: + target = make_task(0, 2, self.pattern) + predicted = make_task(1, 3, self.pattern) + target_call = { + "name": self.pattern.macro_name, + "arguments": {"workflow_ref": workflow_reference(target, self.pattern)}, + } + predicted_call = { + "name": self.pattern.macro_name, + "arguments": { + "workflow_ref": workflow_reference(predicted, self.pattern) + }, + } + response = { + "calls": [{"call": target_call}], + "speculation": { + "status": "miss", + "reason": "invocation_mismatch", + "prediction": predicted_call, + }, + } + args = argparse.Namespace(max_branches=4, macro_max_tokens=128) + with patch( + "benchmark_trace_compiled_workflows.post_turn", + return_value=response, + ) as mocked: + result = measure_private_miss( + args, target, predicted, self.pattern + ) + self.assertTrue(result["passed"]) + self.assertFalse(result["private_result_exposed"]) + self.assertEqual( + mocked.call_args.kwargs["tool_speculation"]["call"], predicted_call + ) + + def test_final_turn_consumes_the_real_tool_conversation(self) -> None: args = argparse.Namespace(final_max_tokens=32) + messages = [ + {"role": "user", "content": "run it"}, + {"role": "assistant", "content": "", "tool_calls": []}, + { + "role": "tool", + "tool_call_id": "call_1", + "content": '{"items":[{"final_ref":"plum"}]}', + }, + ] with patch( "benchmark_trace_compiled_workflows.post_turn", return_value={"content": "workflow_complete:plum"}, ) as mocked: - post_final(args, "workflow_complete:plum") + post_final(args, messages) call_args = mocked.call_args.args self.assertEqual(call_args[2], []) self.assertEqual(call_args[3], "none") self.assertEqual(call_args[4], 32) - self.assertEqual( - call_args[1][-1], - {"role": "user", "content": "workflow_complete:plum"}, - ) + self.assertEqual(call_args[1][:-1], messages) + self.assertNotIn("plum", call_args[1][-1]["content"]) def test_final_receipt_accepts_literal_or_equivalent_json(self) -> None: expected = "workflow_complete:plum,ivory" diff --git a/optimizations/ooo_spec_lucebox5_cpu/test_trace_compiled_tool_executor.py b/optimizations/ooo_spec_lucebox5_cpu/test_trace_compiled_tool_executor.py index 9c95884bf..82f12b9e1 100644 --- a/optimizations/ooo_spec_lucebox5_cpu/test_trace_compiled_tool_executor.py +++ b/optimizations/ooo_spec_lucebox5_cpu/test_trace_compiled_tool_executor.py @@ -3,6 +3,7 @@ import unittest from benchmark_trace_compiled_workflows import mine_pattern, simulated_tool_result +from bfcl_replay_tool_executor import execute as execute_bfcl from test_benchmark_trace_compiled_workflows import workflow_trace from trace_compiled_tool_executor import execute_macro, resolve_items @@ -70,6 +71,31 @@ def test_rejects_unknown_or_missing_inputs(self) -> None: with self.assertRaisesRegex(ValueError, "workflow_ref"): resolve_items({"items": self.items}, self.pattern, self.registry) + def test_rejects_non_object_leaf_response(self) -> None: + request = { + "call": { + "name": self.pattern.macro_name, + "arguments": {"workflow_ref": self.workflow_ref}, + } + } + with self.assertRaisesRegex(RuntimeError, "invalid result"): + execute_macro( + request, + self.pattern, + lambda _request: None, # type: ignore[arg-type,return-value] + self.registry, + ) + + def test_leaf_rejects_explicit_null_affinity(self) -> None: + with self.assertRaisesRegex(ValueError, "cpu_affinity"): + execute_bfcl( + { + "protocol": "dflash.tool-speculation.v1", + "cpu_affinity": None, + "call": {"name": "lookup", "arguments": {}}, + } + ) + if __name__ == "__main__": unittest.main() diff --git a/optimizations/ooo_spec_lucebox5_cpu/trace_compiled_tool_executor.py b/optimizations/ooo_spec_lucebox5_cpu/trace_compiled_tool_executor.py index 20cd771fc..15736940f 100755 --- a/optimizations/ooo_spec_lucebox5_cpu/trace_compiled_tool_executor.py +++ b/optimizations/ooo_spec_lucebox5_cpu/trace_compiled_tool_executor.py @@ -34,7 +34,10 @@ def load_pattern() -> CompiledPattern: - default = Path(__file__).with_name("results") / "trace-compiled-training-traces.json" + default = ( + Path(__file__).with_name("results") + / "multiturn-cached-wordref-production-6tasks.json" + ) report = Path(os.environ.get(TRAINING_REPORT_ENV, str(default))) return mine_pattern(load_training_traces(report, required_steps=5)) @@ -97,7 +100,11 @@ def execute_branch( leaf_request = {**request, "call": call} envelope = leaf_executor(leaf_request) result = envelope.get("result") if isinstance(envelope, dict) else None - if not envelope.get("ok") or not isinstance(result, dict): + if ( + not isinstance(envelope, dict) + or not envelope.get("ok") + or not isinstance(result, dict) + ): raise RuntimeError("leaf tool returned an invalid result") if ( result.get("call_sha256") != call_sha256(call) diff --git a/server/src/common/backend_ipc.cpp b/server/src/common/backend_ipc.cpp index e26e47bfa..d6ee776f3 100644 --- a/server/src/common/backend_ipc.cpp +++ b/server/src/common/backend_ipc.cpp @@ -8,6 +8,10 @@ #include #include #include +#include +#include +#include +#include #include #include #include @@ -15,6 +19,7 @@ #if !defined(_WIN32) # include # include +# include # include # include # include @@ -27,6 +32,85 @@ namespace dflash::common { +namespace { + +#if !defined(_WIN32) +bool read_exact_with_timeout(int fd, void * data, size_t bytes, + int timeout_ms, bool & timed_out) { + timed_out = false; + auto * cursor = static_cast(data); + size_t received = 0; + const auto deadline = std::chrono::steady_clock::now() + + std::chrono::milliseconds(timeout_ms); + while (received < bytes) { + const auto now = std::chrono::steady_clock::now(); + if (now >= deadline) { + timed_out = true; + return false; + } + const int64_t remaining = + std::chrono::duration_cast( + deadline - now).count(); + pollfd descriptor{fd, POLLIN | POLLHUP, 0}; + const int wait_ms = static_cast((std::min)( + int64_t{INT_MAX}, (std::max)(int64_t{1}, remaining))); + const int polled = ::poll(&descriptor, 1, wait_ms); + if (polled == 0) { + timed_out = true; + return false; + } + if (polled < 0) { + if (errno == EINTR) continue; + return false; + } + if (descriptor.revents & (POLLERR | POLLNVAL)) return false; + const ssize_t count = ::read(fd, cursor + received, bytes - received); + if (count == 0) return false; + if (count < 0) { + if (errno == EINTR) continue; + return false; + } + received += static_cast(count); + } + return true; +} + +bool close_descriptor_range(unsigned int first, unsigned int last) { + if (first > last) return true; +#if defined(__linux__) && defined(SYS_close_range) + int rc = -1; + do { + rc = static_cast(::syscall(SYS_close_range, first, last, 0)); + } while (rc != 0 && errno == EINTR); + return rc == 0; +#else + (void)first; + (void)last; + errno = ENOSYS; + return false; +#endif +} + +bool isolate_child_descriptors(int payload_fd, int stream_fd, int shared_fd) { + std::array keep{payload_fd, stream_fd, shared_fd}; + std::sort(keep.begin(), keep.end()); + unsigned int first = STDERR_FILENO + 1; + int previous = -1; + for (const int fd : keep) { + if (fd < static_cast(first) || fd == previous) continue; + if (!close_descriptor_range(first, static_cast(fd - 1))) { + return false; + } + first = static_cast(fd) + 1U; + previous = fd; + } + return close_descriptor_range( + first, (std::numeric_limits::max)()); +} +#endif + +} // namespace + const char * backend_ipc_mode_name(BackendIpcMode mode) { switch (mode) { case BackendIpcMode::Invalid: return "invalid"; @@ -112,7 +196,7 @@ bool BackendIpcProcess::start(const BackendIpcLaunchConfig & cfg) { #else close(); if (cfg.bin.empty() || cfg.payload_path.empty()) return false; - if (!init_work_dir(cfg.work_dir)) return false; + if (!init_work_dir(cfg.work_dir, cfg.require_private_work_dir)) return false; int cmd_pipe[2] = {-1, -1}; int payload_pipe[2] = {-1, -1}; @@ -198,6 +282,14 @@ bool BackendIpcProcess::start(const BackendIpcLaunchConfig & cfg) { argv.reserve(argv_storage.size() + 1); for (std::string & arg : argv_storage) argv.push_back(arg.data()); argv.push_back(nullptr); + if (cfg.isolate_inherited_fds && + !isolate_child_descriptors( + payload_pipe[0], stream_pipe[1], shared_payload_fd_)) { + std::fprintf(stderr, + "backend-ipc descriptor isolation failed: %s\n", + std::strerror(errno)); + _exit(127); + } ::execv(exec_bin.c_str(), argv.data()); std::fprintf(stderr, "backend-ipc exec failed: %s: %s\n", exec_bin.c_str(), std::strerror(errno)); @@ -217,7 +309,13 @@ bool BackendIpcProcess::start(const BackendIpcLaunchConfig & cfg) { return false; } int32_t status = -1; - if (!read_exact_fd(stream_fd_, &status, sizeof(status)) || status != 0) { + bool readiness_timed_out = false; + const bool status_read = cfg.readiness_timeout_ms > 0 + ? read_exact_with_timeout(stream_fd_, &status, sizeof(status), + cfg.readiness_timeout_ms, + readiness_timed_out) + : read_exact_fd(stream_fd_, &status, sizeof(status)); + if (!status_read || status != 0) { int child_status = 0; const pid_t exited = ::waitpid(pid_, &child_status, WNOHANG); if (exited == pid_) { @@ -236,9 +334,17 @@ bool BackendIpcProcess::start(const BackendIpcLaunchConfig & cfg) { } pid_ = -1; } else { - std::fprintf(stderr, "backend-ipc daemon did not become ready (status=%d)\n", status); + std::fprintf(stderr, + readiness_timed_out + ? "backend-ipc daemon readiness timed out after %d ms\n" + : "backend-ipc daemon did not become ready (status=%d)\n", + readiness_timed_out ? cfg.readiness_timeout_ms : status); + } + if (pid_ > 0 && cfg.readiness_timeout_ms > 0) { + terminate(); + } else { + close(); } - close(); return false; } active_ = true; @@ -430,7 +536,8 @@ bool BackendIpcProcess::init_shared_payload(size_t bytes) { return true; } -bool BackendIpcProcess::init_work_dir(const std::string & requested) { +bool BackendIpcProcess::init_work_dir(const std::string & requested, + bool require_private) { if (!requested.empty()) { work_dir_ = requested; owns_work_dir_ = false; @@ -442,14 +549,18 @@ bool BackendIpcProcess::init_work_dir(const std::string & requested) { } } struct stat st; - if (::lstat(work_dir_.c_str(), &st) != 0 || + const int stat_result = require_private + ? ::lstat(work_dir_.c_str(), &st) + : ::stat(work_dir_.c_str(), &st); + if (stat_result != 0 || !S_ISDIR(st.st_mode)) { std::fprintf(stderr, - "backend-ipc work_dir is not a real directory: %s\n", + "backend-ipc work_dir is not a directory: %s\n", work_dir_.c_str()); return false; } - if (st.st_uid != ::geteuid() || (st.st_mode & 0777) != 0700) { + if (require_private && + (st.st_uid != ::geteuid() || (st.st_mode & 0777) != 0700)) { std::fprintf(stderr, "backend-ipc work_dir must be owned by the server and mode 0700: %s\n", work_dir_.c_str()); diff --git a/server/src/common/backend_ipc.h b/server/src/common/backend_ipc.h index e181827ec..3c733d396 100644 --- a/server/src/common/backend_ipc.h +++ b/server/src/common/backend_ipc.h @@ -124,6 +124,15 @@ struct BackendIpcLaunchConfig { std::string work_dir; BackendIpcPayloadTransport payload_transport = BackendIpcPayloadTransport::Auto; size_t shared_payload_bytes = 0; + // Optional sidecars can fail closed instead of waiting forever for their + // initial status word. Zero preserves the legacy unbounded startup wait. + int readiness_timeout_ms = 0; + // Security-sensitive sidecars inherit only stdin/stdout/stderr and the + // explicitly configured payload/response descriptors. + bool isolate_inherited_fds = false; + // Keep legacy backend work directories compatible while allowing private + // sidecars to require an owned, non-symlink 0700 directory. + bool require_private_work_dir = false; }; struct BackendIpcPayloadSegment { @@ -165,7 +174,7 @@ class BackendIpcProcess { private: void close_impl(bool force_terminate); #if !defined(_WIN32) - bool init_work_dir(const std::string & requested); + bool init_work_dir(const std::string & requested, bool require_private); bool init_shared_payload(size_t bytes); pid_t pid_ = -1; diff --git a/server/src/common/qwen3_tool_predictor_ipc.cpp b/server/src/common/qwen3_tool_predictor_ipc.cpp index cd428ab37..0a17d5336 100644 --- a/server/src/common/qwen3_tool_predictor_ipc.cpp +++ b/server/src/common/qwen3_tool_predictor_ipc.cpp @@ -148,16 +148,19 @@ bool Qwen3ToolPredictorIpcClient::start( const std::string & model_path, int gpu, int max_ctx, - const std::string & work_dir) { + const std::string & work_dir, + int readiness_timeout_ms) { #if defined(_WIN32) (void)bin; (void)model_path; (void)gpu; (void)max_ctx; (void)work_dir; + (void)readiness_timeout_ms; std::fprintf(stderr, "Qwen3 tool-predictor IPC is only implemented on POSIX hosts\n"); return false; #else std::lock_guard lock(mutex_); close_locked(); - if (bin.empty() || model_path.empty() || max_ctx <= 0) return false; + if (bin.empty() || model_path.empty() || max_ctx <= 0 || + readiness_timeout_ms <= 0) return false; BackendIpcLaunchConfig launch; launch.bin = bin; @@ -166,6 +169,9 @@ bool Qwen3ToolPredictorIpcClient::start( launch.work_dir = work_dir; launch.args.push_back("--target-gpu=" + std::to_string(std::max(0, gpu))); launch.args.push_back("--max-ctx=" + std::to_string(max_ctx)); + launch.readiness_timeout_ms = readiness_timeout_ms; + launch.isolate_inherited_fds = true; + launch.require_private_work_dir = true; if (!process_.start(launch)) { std::fprintf(stderr, "[tool-predictor-ipc] backend process start failed\n"); return false; diff --git a/server/src/common/qwen3_tool_predictor_ipc.h b/server/src/common/qwen3_tool_predictor_ipc.h index 7c5dc0095..6b401b0ed 100644 --- a/server/src/common/qwen3_tool_predictor_ipc.h +++ b/server/src/common/qwen3_tool_predictor_ipc.h @@ -30,7 +30,8 @@ class Qwen3ToolPredictorIpcClient { const std::string & model_path, int gpu, int max_ctx, - const std::string & work_dir); + const std::string & work_dir, + int readiness_timeout_ms); // Requests are serialized: one compact predictor model owns one KV cache. // On transport or generation failure the lane closes and fails shut. diff --git a/server/src/qwen3/qwen3_backend.cpp b/server/src/qwen3/qwen3_backend.cpp index 454bd5e00..f7c798938 100644 --- a/server/src/qwen3/qwen3_backend.cpp +++ b/server/src/qwen3/qwen3_backend.cpp @@ -502,7 +502,9 @@ bool Qwen3Backend::do_decode(int committed, int n_gen, ggml_free(ectx); } - if (!do_step(embed_buf.data(), 1, committed, logits)) { + // `committed` was advanced when `next` was accepted. Write that token + // at its zero-based KV position instead of skipping one cache slot. + if (!do_step(embed_buf.data(), 1, committed - 1, logits)) { return false; } } diff --git a/server/src/qwen3/qwen3_loader.cpp b/server/src/qwen3/qwen3_loader.cpp index 52bc3291e..49e5f9226 100644 --- a/server/src/qwen3/qwen3_loader.cpp +++ b/server/src/qwen3/qwen3_loader.cpp @@ -108,6 +108,25 @@ float get_f32(gguf_context * g, const char * key, float def) { return gguf_get_val_f32(g, k); } +bool supported_weight_storage(ggml_type type) { + return type == GGML_TYPE_Q8_0 || type == GGML_TYPE_BF16 || + type == GGML_TYPE_F16; +} + +ggml_type tensor_storage_type(gguf_context * gctx, const char * name, + ggml_type fallback, bool & ok) { + const int64_t index = gguf_find_tensor(gctx, name); + if (index < 0) return fallback; + const ggml_type type = gguf_get_tensor_type(gctx, index); + if (!supported_weight_storage(type)) { + set_last_error(std::string("unsupported Qwen3-0.6B storage type for ") + + name + ": " + ggml_type_name(type)); + ok = false; + return fallback; + } + return type; +} + } // namespace bool load_qwen3_drafter_model(const std::string & path, @@ -146,9 +165,9 @@ bool load_qwen3_drafter_model(const std::string & path, wtype = gguf_get_tensor_type(gctx, tidx); } } - if (wtype == GGML_TYPE_Q8_0) { - out.weight_type = GGML_TYPE_Q8_0; - } else if (wtype != GGML_TYPE_BF16 && wtype != GGML_TYPE_F16) { + if (supported_weight_storage(wtype)) { + out.weight_type = wtype; + } else { set_last_error(std::string("unsupported Qwen3-0.6B weight type: ") + ggml_type_name(wtype)); gguf_free(gctx); @@ -178,30 +197,76 @@ bool load_qwen3_drafter_model(const std::string & path, const int n_vocab = out.n_vocab; const int q_dim = n_head * head_dim; const int kv_dim = n_head_kv * head_dim; - const ggml_type weight_type = out.weight_type; + bool storage_types_ok = true; + const ggml_type token_type = tensor_storage_type( + gctx, "token_embd.weight", out.weight_type, storage_types_ok); + const ggml_type output_type = tensor_storage_type( + gctx, "output.weight", token_type, storage_types_ok); // Top-level tensors. - out.tok_embd = ggml_new_tensor_2d(out.ctx, weight_type, n_embd, n_vocab); + out.tok_embd = ggml_new_tensor_2d(out.ctx, token_type, n_embd, n_vocab); out.out_norm = ggml_new_tensor_1d(out.ctx, GGML_TYPE_F32, n_embd); - out.output = ggml_new_tensor_2d(out.ctx, weight_type, n_embd, n_vocab); + out.output = ggml_new_tensor_2d(out.ctx, output_type, n_embd, n_vocab); ggml_set_name(out.tok_embd, "token_embd.weight"); ggml_set_name(out.out_norm, "output_norm.weight"); ggml_set_name(out.output, "output.weight"); out.layers.resize(n_layer); + char tensor_name[128]; for (int il = 0; il < n_layer; ++il) { auto & L = out.layers[il]; L.attn_norm = ggml_new_tensor_1d(out.ctx, GGML_TYPE_F32, n_embd); - L.wq = ggml_new_tensor_2d(out.ctx, weight_type, n_embd, q_dim); - L.wk = ggml_new_tensor_2d(out.ctx, weight_type, n_embd, kv_dim); - L.wv = ggml_new_tensor_2d(out.ctx, weight_type, n_embd, kv_dim); - L.wo = ggml_new_tensor_2d(out.ctx, weight_type, q_dim, n_embd); + std::snprintf(tensor_name, sizeof(tensor_name), + "blk.%d.attn_q.weight", il); + L.wq = ggml_new_tensor_2d( + out.ctx, tensor_storage_type(gctx, tensor_name, out.weight_type, + storage_types_ok), + n_embd, q_dim); + std::snprintf(tensor_name, sizeof(tensor_name), + "blk.%d.attn_k.weight", il); + L.wk = ggml_new_tensor_2d( + out.ctx, tensor_storage_type(gctx, tensor_name, out.weight_type, + storage_types_ok), + n_embd, kv_dim); + std::snprintf(tensor_name, sizeof(tensor_name), + "blk.%d.attn_v.weight", il); + L.wv = ggml_new_tensor_2d( + out.ctx, tensor_storage_type(gctx, tensor_name, out.weight_type, + storage_types_ok), + n_embd, kv_dim); + std::snprintf(tensor_name, sizeof(tensor_name), + "blk.%d.attn_output.weight", il); + L.wo = ggml_new_tensor_2d( + out.ctx, tensor_storage_type(gctx, tensor_name, out.weight_type, + storage_types_ok), + q_dim, n_embd); L.q_norm = ggml_new_tensor_1d(out.ctx, GGML_TYPE_F32, head_dim); L.k_norm = ggml_new_tensor_1d(out.ctx, GGML_TYPE_F32, head_dim); L.ffn_norm = ggml_new_tensor_1d(out.ctx, GGML_TYPE_F32, n_embd); - L.ffn_gate = ggml_new_tensor_2d(out.ctx, weight_type, n_embd, n_ff); - L.ffn_up = ggml_new_tensor_2d(out.ctx, weight_type, n_embd, n_ff); - L.ffn_down = ggml_new_tensor_2d(out.ctx, weight_type, n_ff, n_embd); + std::snprintf(tensor_name, sizeof(tensor_name), + "blk.%d.ffn_gate.weight", il); + L.ffn_gate = ggml_new_tensor_2d( + out.ctx, tensor_storage_type(gctx, tensor_name, out.weight_type, + storage_types_ok), + n_embd, n_ff); + std::snprintf(tensor_name, sizeof(tensor_name), + "blk.%d.ffn_up.weight", il); + L.ffn_up = ggml_new_tensor_2d( + out.ctx, tensor_storage_type(gctx, tensor_name, out.weight_type, + storage_types_ok), + n_embd, n_ff); + std::snprintf(tensor_name, sizeof(tensor_name), + "blk.%d.ffn_down.weight", il); + L.ffn_down = ggml_new_tensor_2d( + out.ctx, tensor_storage_type(gctx, tensor_name, out.weight_type, + storage_types_ok), + n_ff, n_embd); + } + if (!storage_types_ok) { + gguf_free(gctx); + ggml_free(out.ctx); + out.ctx = nullptr; + return false; } out.buf = ggml_backend_alloc_ctx_tensors(out.ctx, backend); diff --git a/server/src/server/chat_template.cpp b/server/src/server/chat_template.cpp index b402752dc..2d5970efa 100644 --- a/server/src/server/chat_template.cpp +++ b/server/src/server/chat_template.cpp @@ -76,8 +76,7 @@ std::string render_chat_template( ChatFormat format, bool add_generation_prompt, bool enable_thinking, - const std::string & tools_json, - bool tool_call_required) + const std::string & tools_json) { std::string result; bool has_tools = !tools_json.empty() && tools_json != "[]" && tools_json != "null"; @@ -384,13 +383,9 @@ std::string render_chat_template( result += "For each function call, you MUST return a single JSON object " "within '' and '' tags, " "containing the function name and arguments, like this:\n" - "\n" - "{\"name\": \"function_name\", \"arguments\": {\"param_name\": \"value\"}}\n" - "\n\n"; - if (tool_call_required) { - result += "You MUST call exactly one available function and emit no " - "text outside its tags.\n\n"; - } + "\n" + "{\"name\": \"function_name\", \"arguments\": {\"param_name\": \"value\"}}\n" + "\n\n"; } result += system_content; diff --git a/server/src/server/chat_template.h b/server/src/server/chat_template.h index b7825153c..ecade9217 100644 --- a/server/src/server/chat_template.h +++ b/server/src/server/chat_template.h @@ -39,17 +39,14 @@ enum class ChatFormat { // false → assistant starts with \n\n\n\n (skip thinking) // // `tools_json` is an optional JSON string containing the tool definitions -// array. When non-empty, tool-capable templates inject a tool preamble into -// the system message instructing the model how to emit tags. -// `tool_call_required` strengthens that instruction for OpenAI -// `tool_choice="required"` and forced-function requests. +// array. When non-empty, the Qwen3/3.5 template injects a tool preamble +// into the system message instructing the model how to emit tags. std::string render_chat_template( const std::vector & messages, ChatFormat format, bool add_generation_prompt = true, bool enable_thinking = false, - const std::string & tools_json = "", - bool tool_call_required = false); + const std::string & tools_json = ""); // Detect the appropriate chat format for an architecture. ChatFormat chat_format_for_arch(const std::string & arch); diff --git a/server/src/server/http_server.cpp b/server/src/server/http_server.cpp index 786f4f057..1adc717b7 100644 --- a/server/src/server/http_server.cpp +++ b/server/src/server/http_server.cpp @@ -253,6 +253,14 @@ bool should_clamp_flowkv_disk_cache( return flowkv && policy.compress; } +bool tool_choice_disables_tool_calls(const json & tool_choice) { + if (tool_choice.is_string()) { + return tool_choice.get() == "none"; + } + return tool_choice.is_object() && + tool_choice.value("type", "") == "none"; +} + } // namespace http_detail // ─── curl helpers for upstream proxy ───────────────────────────────────── @@ -817,6 +825,53 @@ SemanticToolPrediction request_semantic_tool_prediction( return finish(); } +SemanticToolPrediction predict_semantic_tool_call( + const SemanticToolPredictorConfig & config, + const json & payload, + const json & request_tools, + const std::shared_ptr & native) { + const auto started = std::chrono::steady_clock::now(); + SemanticToolPrediction native_result; + if (native && native->active()) { + native_result = native->predict(payload, request_tools); + if (native_result.ok || !config.http_enabled()) { + return native_result; + } + } + if (!config.http_enabled()) { + if (native_result.error.empty()) { + native_result.error = "native_predictor_not_initialized"; + } + return native_result; + } + + const double elapsed_ms = std::chrono::duration( + std::chrono::steady_clock::now() - started).count(); + const int remaining_ms = config.timeout_ms - + static_cast(std::ceil(elapsed_ms)); + if (remaining_ms <= 0) { + native_result.source = config.model; + native_result.ok = false; + native_result.error = native_result.error.empty() + ? "predictor_timeout" + : "native=" + native_result.error + ";http=predictor_timeout"; + native_result.wall_ms = elapsed_ms; + return native_result; + } + + SemanticToolPredictorConfig fallback_config = config; + fallback_config.timeout_ms = remaining_ms; + SemanticToolPrediction fallback = request_semantic_tool_prediction( + fallback_config, payload, request_tools); + if (!fallback.ok && !native_result.error.empty()) { + fallback.error = "native=" + native_result.error + + ";http=" + fallback.error; + } + fallback.wall_ms = std::chrono::duration( + std::chrono::steady_clock::now() - started).count(); + return fallback; +} + } // namespace // ─── /props constants ─────────────────────────────────────────────────── @@ -947,15 +1002,6 @@ static const json * find_tool_function(const json & tools, return nullptr; } -static bool tool_choice_requires_call(const json & tool_choice) { - if (tool_choice.is_string()) { - return tool_choice.get() == "required"; - } - return tool_choice.is_object() && tool_choice.contains("function") && - tool_choice["function"].is_object() && - !tool_choice["function"].value("name", "").empty(); -} - static std::string first_tool_parameter_name(const json & function_def) { const auto & params = function_def.value("parameters", json::object()); if (params.contains("required") && params["required"].is_array()) { @@ -1216,13 +1262,10 @@ json build_props_body(const ServerConfig & config, ? json(config.semantic_tool_predictor.execution_confidence) : json(nullptr)}, {"predictor_schedule", - config.semantic_tool_predictor.native_enabled() - ? json(native_tool_predictor_schedule_name( - config.semantic_tool_predictor.native_schedule)) - : config.semantic_tool_predictor.http_enabled() - ? json("overlap") : json(nullptr)}, + config.semantic_tool_predictor.enabled() + ? json("before-model") : json(nullptr)}, {"predictor_decode_isolated", - config.semantic_tool_predictor.native_runs_before_model()}, + config.semantic_tool_predictor.enabled()}, {"execution_mode", config.tool_speculation.execution_mode()}, {"profile_status", config.tool_speculation.policy.empty() @@ -1233,9 +1276,11 @@ json build_props_body(const ServerConfig & config, ? json(nullptr) : json(config.tool_speculation.policy.executor_contract())}, {"protocol", "dflash.tool-speculation.v1"}, - {"requires_client_support", - !(config.tool_speculation.enabled() && - config.semantic_tool_predictor.enabled())}, + {"client_prediction_required", + config.tool_speculation.enabled() && + !config.semantic_tool_predictor.enabled()}, + {"client_result_handling_required", + config.tool_speculation.enabled()}, {"preserves_token_speculation", true}, {"unqualified_lane_policy", "defer"}, {"allowed_tools", config.tool_speculation.allowed_tools}, @@ -2118,6 +2163,11 @@ bool HttpServer::parse_common_request_fields( } if (body.contains("tool_speculation")) { + if (http_detail::tool_choice_disables_tool_calls(req.tool_choice)) { + send_error(fd, 400, + "tool_speculation cannot be used when tool_choice is none"); + return false; + } ToolSpeculationPrediction prediction; std::string prediction_error; if (!parse_tool_speculation_prediction( @@ -2392,8 +2442,7 @@ bool HttpServer::render_and_tokenize_request( } else { rendered = render_chat_template( chat_messages, chat_format_, /*add_generation_prompt=*/true, - req.thinking_enabled, tools_json, - tool_choice_requires_call(req.tool_choice)); + req.thinking_enabled, tools_json); } req.started_in_thinking = prompt_ends_in_open_think(rendered); @@ -2434,20 +2483,17 @@ void HttpServer::log_parsed_request(const ParsedRequest & req) const { req.stop_sequences.size(), req.model.c_str()); } -void HttpServer::launch_semantic_tool_prediction(ParsedRequest & req) const { - if (req.semantic_tool_prediction.valid() || - req.automatic_tool_speculation.valid()) { - return; - } - const bool automatic_execution = - req.automatic_tool_speculation_enabled && - !req.tool_speculation.has_value() && - config_.tool_speculation.enabled(); - if (!automatic_execution || - !config_.semantic_tool_predictor.enabled() || req.tools.empty() || - !req.raw_body.is_object()) { - return; - } +void HttpServer::start_automatic_tool_speculation(ParsedRequest & req) const { + if (req.automatic_tool_speculation.has_value() || + !req.automatic_tool_speculation_enabled || + req.tool_speculation.has_value() || + !config_.tool_speculation.enabled() || + !config_.semantic_tool_predictor.enabled() || + http_detail::tool_choice_disables_tool_calls(req.tool_choice) || + req.tools.empty() || !req.raw_body.is_object()) return; + + req.automatic_tool_speculation.emplace(); + auto & launch = *req.automatic_tool_speculation; const SemanticToolPredictorConfig predictor = config_.semantic_tool_predictor; json semantic_request = req.raw_body; @@ -2464,104 +2510,44 @@ void HttpServer::launch_semantic_tool_prediction(ParsedRequest & req) const { predictor.max_tokens); const json tools = req.tools; const auto native = native_semantic_predictor_; - req.semantic_tool_prediction = std::async( - std::launch::async, - [predictor, payload, tools, native]() { - const auto prediction_started = std::chrono::steady_clock::now(); - SemanticToolPrediction native_result; - if (native && native->active()) { - native_result = native->predict(payload, tools); - if (native_result.ok || !predictor.http_enabled()) { - return native_result; - } - } - if (predictor.http_enabled()) { - const double elapsed_ms = - std::chrono::duration( - std::chrono::steady_clock::now() - - prediction_started).count(); - const int remaining_ms = predictor.timeout_ms - - static_cast(std::ceil(elapsed_ms)); - if (remaining_ms <= 0) { - native_result.source = predictor.model; - native_result.ok = false; - native_result.error = native_result.error.empty() - ? "predictor_timeout" - : "native=" + native_result.error + - ";http=predictor_timeout"; - native_result.wall_ms = elapsed_ms; - return native_result; - } - SemanticToolPredictorConfig fallback_config = predictor; - fallback_config.timeout_ms = remaining_ms; - SemanticToolPrediction fallback = - request_semantic_tool_prediction( - fallback_config, payload, tools); - if (!fallback.ok && !native_result.error.empty()) { - fallback.error = "native=" + native_result.error + - ";http=" + fallback.error; - } - fallback.wall_ms = - std::chrono::duration( - std::chrono::steady_clock::now() - - prediction_started).count(); - return fallback; - } - if (native_result.error.empty()) { - native_result.error = "native_predictor_not_initialized"; + try { + const SemanticToolPrediction semantic = predict_semantic_tool_call( + predictor, payload, tools, native); + launch.predictor_wall_ms = semantic.wall_ms; + launch.prediction_source = semantic.source; + if (!semantic.ok) { + launch.predictor_error = semantic.error.empty() + ? "predictor_unavailable" : semantic.error; + } else { + ToolSpeculationPrediction prediction; + std::string error; + const json arguments = json::parse(semantic.call.arguments.dump()); + if (!build_tool_speculation_prediction( + semantic.call.name, arguments, + predictor.execution_confidence, prediction, error)) { + launch.predictor_error = std::move(error); + } else { + auto attempt = ToolSpeculationAttempt::create( + config_.tool_speculation, prediction, req.response_id); + launch.attempt = std::shared_ptr( + std::move(attempt)); + launch.attempt->start(); } - return native_result; - }).share(); - if (automatic_execution) { - const auto semantic_prediction = req.semantic_tool_prediction; - const ToolSpeculationConfig tool_config = config_.tool_speculation; - const double confidence = predictor.execution_confidence; - const std::string request_id = req.response_id; - req.automatic_tool_speculation = std::async( - std::launch::async, - [semantic_prediction, tool_config, confidence, request_id]() { - ParsedRequest::AutomaticToolSpeculationLaunch launch; - try { - const SemanticToolPrediction & semantic = - semantic_prediction.get(); - launch.predictor_wall_ms = semantic.wall_ms; - launch.prediction_source = semantic.source; - if (!semantic.ok) { - launch.predictor_error = semantic.error.empty() - ? "predictor_unavailable" : semantic.error; - return launch; - } - ToolSpeculationPrediction prediction; - std::string error; - const json arguments = json::parse( - semantic.call.arguments.dump()); - if (!build_tool_speculation_prediction( - semantic.call.name, arguments, confidence, - prediction, error)) { - launch.predictor_error = std::move(error); - return launch; - } - auto attempt = ToolSpeculationAttempt::create( - tool_config, prediction, request_id); - launch.attempt = std::shared_ptr( - attempt.release()); - launch.attempt->start(); - } catch (const std::exception & error) { - launch.predictor_error = - std::string("automatic_prediction_failed: ") + - error.what(); - } catch (...) { - launch.predictor_error = - "automatic_prediction_failed: unknown error"; - } - return launch; - }).share(); + } + } catch (const std::exception & error) { + launch.predictor_error = + std::string("automatic_prediction_failed: ") + error.what(); + } catch (...) { + launch.predictor_error = + "automatic_prediction_failed: unknown error"; } std::fprintf(stderr, - "[tool-hint] launched predictor transport=%s%s tools=%zu execute=%s\n", + "[tool-hint] predictor complete transport=%s%s tools=%zu " + "execute=%s wall_ms=%.1f\n", native ? "native-qwen3" : "http", native && predictor.http_enabled() ? "+http-fallback" : "", - json_array_size(req.tools), automatic_execution ? "true" : "false"); + json_array_size(req.tools), launch.attempt ? "true" : "false", + launch.predictor_wall_ms); } void HttpServer::enqueue_request_and_wait(SocketHandle fd, ParsedRequest req) { @@ -2660,9 +2646,6 @@ bool HttpServer::route_request(SocketHandle fd, const HttpRequest & hr) { } if (!validate_request_context(fd, req)) return true; - if (!config_.semantic_tool_predictor.native_runs_before_model()) { - launch_semantic_tool_prediction(req); - } log_parsed_request(req); enqueue_request_and_wait(fd, std::move(req)); return true; @@ -3271,8 +3254,7 @@ void HttpServer::apply_flowkv_compression( } else { rendered = render_chat_template( chat_messages, chat_format_, /*add_generation_prompt=*/true, - req.thinking_enabled, tools_json, - tool_choice_requires_call(req.tool_choice)); + req.thinking_enabled, tools_json); } const int tokens_before = (int) prepared.tokens.size(); @@ -4345,26 +4327,6 @@ void HttpServer::process_job(ServerJob * job) { } if (req.stream) start_job_stream(job); - if (config_.semantic_tool_predictor.native_runs_before_model()) { - const auto predictor_wait_started = std::chrono::steady_clock::now(); - launch_semantic_tool_prediction(req); - if (req.automatic_tool_speculation.valid()) { - req.automatic_tool_speculation.wait(); - } else if (req.semantic_tool_prediction.valid()) { - req.semantic_tool_prediction.wait(); - } - const double predictor_wait_ms = - std::chrono::duration( - std::chrono::steady_clock::now() - predictor_wait_started) - .count(); - if (req.automatic_tool_speculation.valid() || - req.semantic_tool_prediction.valid()) { - std::fprintf(stderr, - "[tool-hint] before-model barrier complete %.1f ms\n", - predictor_wait_ms); - } - } - PreparedPrompt prepared = prepare_prompt(req); if (prepared.error_status != 0) { fail_request(prepared.error_status, prepared.error); @@ -4375,6 +4337,11 @@ void HttpServer::process_job(ServerJob * job) { return; } + // The only qualified schedule predicts before target compute. This keeps + // shared-GPU deployments deterministic; the admitted external tool then + // overlaps target generation. + start_automatic_tool_speculation(req); + std::unique_ptr tool_speculation; if (req.tool_speculation.has_value()) { tool_speculation = ToolSpeculationAttempt::create( @@ -4521,47 +4488,28 @@ void HttpServer::process_job(ServerJob * job) { metadata["prediction_source"] = "client"; return metadata; } - if (!req.automatic_tool_speculation.valid()) return std::nullopt; - try { - const ParsedRequest::AutomaticToolSpeculationLaunch & launch = - req.automatic_tool_speculation.get(); - json metadata; - if (launch.attempt) { - metadata = cancel_reason - ? launch.attempt->cancel(cancel_reason) - : launch.attempt->resolve(emitter.tool_calls()); - } else { - metadata = { - {"protocol", "dflash.tool-speculation.v1"}, - {"status", "deferred"}, - {"reason", "predictor_unavailable"}, - }; - if (!launch.predictor_error.empty()) { - metadata["detail"] = launch.predictor_error; - } - } - metadata["prediction_source"] = - launch.prediction_source.empty() - ? "predictor" : launch.prediction_source; - metadata["predictor_wall_ms"] = launch.predictor_wall_ms; - return metadata; - } catch (const std::exception & error) { - return json{ - {"protocol", "dflash.tool-speculation.v1"}, - {"status", "failed"}, - {"reason", "predictor_future_failure"}, - {"detail", error.what()}, - {"prediction_source", "predictor"}, - }; - } catch (...) { - return json{ + if (!req.automatic_tool_speculation.has_value()) return std::nullopt; + const ParsedRequest::AutomaticToolSpeculationLaunch & launch = + *req.automatic_tool_speculation; + json metadata; + if (launch.attempt) { + metadata = cancel_reason + ? launch.attempt->cancel(cancel_reason) + : launch.attempt->resolve(emitter.tool_calls()); + } else { + metadata = { {"protocol", "dflash.tool-speculation.v1"}, - {"status", "failed"}, - {"reason", "predictor_future_failure"}, - {"detail", "unknown error"}, - {"prediction_source", "predictor"}, + {"status", "deferred"}, + {"reason", "predictor_unavailable"}, }; + if (!launch.predictor_error.empty()) { + metadata["detail"] = launch.predictor_error; + } } + metadata["prediction_source"] = launch.prediction_source.empty() + ? "predictor" : launch.prediction_source; + metadata["predictor_wall_ms"] = launch.predictor_wall_ms; + return metadata; }; // A partial tool call from a failed generation is not authoritative. // Keep its speculative result private just as we do on disconnect. diff --git a/server/src/server/http_server.h b/server/src/server/http_server.h index 05fcf564c..6c4d4d602 100644 --- a/server/src/server/http_server.h +++ b/server/src/server/http_server.h @@ -41,7 +41,6 @@ #include #include #include -#include #include #include #include @@ -242,9 +241,9 @@ struct ServerConfig { // an executor, an empirical interference profile, and an explicit // read-only/idempotent tool allowlist. ToolSpeculationConfig tool_speculation; - // Model-agnostic tool-call prediction. Native predictors run before the - // model by default so shared accelerator compute cannot slow decoding; - // remote predictors may overlap. DS4 verifies the exact canonical call. + // Model-agnostic tool-call prediction. Predictors run before the target; + // only the allowlisted external tool overlaps target generation. This is + // the qualified schedule on shared and single-accelerator deployments. SemanticToolPredictorConfig semantic_tool_predictor; }; @@ -262,6 +261,9 @@ float resolve_pflash_keep_ratio(float configured_ratio, const HttpServerSessions & sessions); bool should_clamp_flowkv_disk_cache( bool flowkv, const DiskPrefixCachePolicy & policy); +// True for API dialects that explicitly prohibit a tool call. Used before +// either caller-supplied or automatic speculative execution can start. +bool tool_choice_disables_tool_calls(const json & tool_choice); } // namespace http_detail @@ -286,8 +288,8 @@ struct ParsedRequest { // The engine may execute it privately, but never exposes its result until // the model emits the exact canonical invocation. std::optional tool_speculation; - // Engine-side Qwen prediction may start a private external tool as soon - // as it is ready. The model's eventual canonical call remains authoritative. + // Engine-side prediction may start a private external tool before target + // generation. The model's eventual canonical call remains authoritative. struct AutomaticToolSpeculationLaunch { std::shared_ptr attempt; double predictor_wall_ms = 0.0; @@ -295,9 +297,8 @@ struct ParsedRequest { std::string predictor_error; }; bool automatic_tool_speculation_enabled = true; - std::shared_future + std::optional automatic_tool_speculation; - std::shared_future semantic_tool_prediction; // Response ID std::string response_id; // Thinking/reasoning state @@ -517,7 +518,7 @@ class HttpServer { ParsedRequest & req); bool validate_request_context(SocketHandle fd, const ParsedRequest & req); void log_parsed_request(const ParsedRequest & req) const; - void launch_semantic_tool_prediction(ParsedRequest & req) const; + void start_automatic_tool_speculation(ParsedRequest & req) const; void enqueue_request_and_wait(SocketHandle fd, ParsedRequest req); // Send HTTP response helpers. diff --git a/server/src/server/native_semantic_tool_predictor.cpp b/server/src/server/native_semantic_tool_predictor.cpp index c375e6b3d..0975388a2 100644 --- a/server/src/server/native_semantic_tool_predictor.cpp +++ b/server/src/server/native_semantic_tool_predictor.cpp @@ -7,6 +7,10 @@ namespace dflash::common { +namespace { +constexpr int kNativePredictorStartupTimeoutMs = 60000; +} + std::shared_ptr NativeSemanticToolPredictor::create( const SemanticToolPredictorConfig & config, @@ -26,7 +30,7 @@ NativeSemanticToolPredictor::create( if (!predictor->ipc_.start( config.native_ipc_bin, config.native_model_path, config.native_gpu, config.native_max_ctx, - config.native_work_dir)) { + config.native_work_dir, kNativePredictorStartupTimeoutMs)) { error = "native_predictor_ipc_start_failed"; return nullptr; } @@ -50,12 +54,20 @@ SemanticToolPrediction NativeSemanticToolPredictor::predict( std::string prompt_error; const std::string prompt = build_native_semantic_tool_predictor_prompt( - predictor_request, prompt_error); + predictor_request, prompt_error, &deadline); if (prompt.empty()) { prediction.error = std::move(prompt_error); return finish(); } - const std::vector prompt_ids = tokenizer_.encode(prompt); + if (std::chrono::steady_clock::now() >= deadline) { + prediction.error = "native_predictor_timeout"; + return finish(); + } + std::vector prompt_ids; + if (!tokenizer_.encode_until(prompt, deadline, prompt_ids)) { + prediction.error = "native_predictor_timeout"; + return finish(); + } if (prompt_ids.empty()) { prediction.error = "native_predictor_prompt_tokenization_failed"; return finish(); diff --git a/server/src/server/semantic_tool_hint.cpp b/server/src/server/semantic_tool_hint.cpp index cf7e7c3de..b18641357 100644 --- a/server/src/server/semantic_tool_hint.cpp +++ b/server/src/server/semantic_tool_hint.cpp @@ -8,31 +8,6 @@ namespace dflash::common { -const char * native_tool_predictor_schedule_name( - NativeToolPredictorSchedule schedule) { - switch (schedule) { - case NativeToolPredictorSchedule::BeforeModel: - return "before-model"; - case NativeToolPredictorSchedule::Overlap: - return "overlap"; - } - return "unknown"; -} - -bool parse_native_tool_predictor_schedule( - const std::string & value, - NativeToolPredictorSchedule & out) { - if (value == "before-model") { - out = NativeToolPredictorSchedule::BeforeModel; - return true; - } - if (value == "overlap") { - out = NativeToolPredictorSchedule::Overlap; - return true; - } - return false; -} - namespace { bool request_has_function(const json & tools, const std::string & name) { @@ -102,23 +77,6 @@ bool parse_call_object(const json & value, SemanticToolCall & out) { return true; } -bool parse_content_call(const std::string & content, SemanticToolCall & out) { - for (size_t offset = 0; offset < content.size(); ++offset) { - if (content[offset] != '{') continue; - try { - const auto value = json::parse( - content.begin() + static_cast(offset), - content.end(), nullptr, false); - if (!value.is_discarded() && parse_call_object(value, out)) { - return true; - } - } catch (...) { - // Continue scanning for a later strict object. - } - } - return false; -} - std::string trim_copy(std::string value) { const auto is_space = [](unsigned char ch) { return std::isspace(ch); }; value.erase(value.begin(), std::find_if_not( @@ -128,6 +86,11 @@ std::string trim_copy(std::string value) { return value; } +bool parse_content_call(const std::string & content, SemanticToolCall & out) { + const json value = json::parse(trim_copy(content), nullptr, false); + return !value.is_discarded() && parse_call_object(value, out); +} + bool parse_qwen_tagged_call_repair( const std::string & generated_text, SemanticToolCall & out) { @@ -187,14 +150,29 @@ bool parse_qwen_bare_single_tool_arguments( return true; } -std::string semantic_message_content(const json & message) { +bool semantic_deadline_expired( + const std::chrono::steady_clock::time_point * deadline) { + return deadline && std::chrono::steady_clock::now() >= *deadline; +} + +bool semantic_message_content( + const json & message, + const std::chrono::steady_clock::time_point * deadline, + std::string & text) { + text.clear(); const auto content = message.find("content"); - if (content == message.end() || content->is_null()) return {}; - if (content->is_string()) return content->get(); - if (!content->is_array()) return content->dump(); + if (content == message.end() || content->is_null()) return true; + if (content->is_string()) { + text = content->get(); + return !semantic_deadline_expired(deadline); + } + if (!content->is_array()) { + text = content->dump(); + return !semantic_deadline_expired(deadline); + } - std::string text; for (const auto & part : *content) { + if (semantic_deadline_expired(deadline)) return false; if (part.is_string()) { text += part.get(); continue; @@ -206,7 +184,7 @@ std::string semantic_message_content(const json & message) { text += part.value("text", ""); } } - return text; + return !semantic_deadline_expired(deadline); } std::string forced_tool_name(const json & choice) { @@ -240,13 +218,17 @@ bool parse_semantic_tool_prediction( bool parsed = false; const auto calls = message->find("tool_calls"); - if (calls != message->end() && calls->is_array() && calls->size() == 1) { + if (calls != message->end()) { + if (!calls->is_array() || calls->size() != 1 || + !(*calls)[0].is_object()) { + error = "predictor_response_requires_single_tool_call"; + return false; + } const auto function = (*calls)[0].find("function"); if (function != (*calls)[0].end()) { parsed = parse_call_object(*function, out); } - } - if (!parsed) { + } else { const auto content = message->find("content"); if (content != message->end() && content->is_string()) { parsed = parse_content_call(content->get(), out); @@ -337,8 +319,19 @@ json build_semantic_tool_predictor_request( std::string build_native_semantic_tool_predictor_prompt( const json & predictor_request, - std::string & error) { + std::string & error, + const std::chrono::steady_clock::time_point * deadline) { error.clear(); + if (semantic_deadline_expired(deadline)) { + error = "native_predictor_timeout"; + return {}; + } + const json choice = predictor_request.value("tool_choice", json("auto")); + if ((choice.is_string() && choice.get() == "none") || + (choice.is_object() && choice.value("type", "") == "none")) { + error = "native_predictor_tool_choice_none"; + return {}; + } const auto messages = predictor_request.find("messages"); if (messages == predictor_request.end() || !messages->is_array() || messages->empty()) { @@ -359,11 +352,20 @@ std::string build_native_semantic_tool_predictor_prompt( std::vector chat; chat.reserve(messages->size()); for (const auto & message : *messages) { + if (semantic_deadline_expired(deadline)) { + error = "native_predictor_timeout"; + return {}; + } if (!message.is_object()) continue; std::string role = message.value("role", "user"); if (role == "developer") role = "system"; + std::string content; + if (!semantic_message_content(message, deadline, content)) { + error = "native_predictor_timeout"; + return {}; + } chat.push_back({ - std::move(role), semantic_message_content(message), + std::move(role), std::move(content), message.value("tool_calls", json::array()), }); } @@ -373,7 +375,6 @@ std::string build_native_semantic_tool_predictor_prompt( } std::string constraint; - const json choice = predictor_request.value("tool_choice", json("auto")); if (choice.is_string() && choice.get() == "required") { constraint = "You must call exactly one available function."; } else if (const std::string name = forced_tool_name(choice); @@ -405,7 +406,13 @@ std::string build_native_semantic_tool_predictor_prompt( "You may call one or more functions to assist with the user query.\n\n" "You are provided with function signatures within XML tags:\n" ""; - for (const auto & tool : tools) rendered += tool.dump(); + for (const auto & tool : tools) { + if (semantic_deadline_expired(deadline)) { + error = "native_predictor_timeout"; + return {}; + } + rendered += tool.dump(); + } rendered += "\n\n\n" "For each function call, return a json object with function name and " @@ -416,6 +423,10 @@ std::string build_native_semantic_tool_predictor_prompt( bool in_tool_response = false; for (size_t index = begin; index < chat.size(); ++index) { + if (semantic_deadline_expired(deadline)) { + error = "native_predictor_timeout"; + return {}; + } const auto & message = chat[index]; if (message.role == "tool") { if (!in_tool_response) { @@ -436,6 +447,10 @@ std::string build_native_semantic_tool_predictor_prompt( rendered += "<|im_start|>" + message.role + "\n" + message.content; if (message.role == "assistant" && message.tool_calls.is_array()) { for (const auto & raw_call : message.tool_calls) { + if (semantic_deadline_expired(deadline)) { + error = "native_predictor_timeout"; + return {}; + } if (!raw_call.is_object()) continue; const json & call = raw_call.contains("function") && raw_call["function"].is_object() @@ -454,6 +469,10 @@ std::string build_native_semantic_tool_predictor_prompt( rendered += "<|im_end|>\n"; } rendered += "<|im_start|>assistant\n\n\n\n\n"; + if (semantic_deadline_expired(deadline)) { + error = "native_predictor_timeout"; + return {}; + } return rendered; } diff --git a/server/src/server/semantic_tool_hint.h b/server/src/server/semantic_tool_hint.h index 69c6f962c..254b6367d 100644 --- a/server/src/server/semantic_tool_hint.h +++ b/server/src/server/semantic_tool_hint.h @@ -4,6 +4,7 @@ #include +#include #include namespace dflash::common { @@ -11,18 +12,6 @@ namespace dflash::common { using json = nlohmann::json; using ordered_json = nlohmann::ordered_json; -enum class NativeToolPredictorSchedule { - BeforeModel, - Overlap, -}; - -const char * native_tool_predictor_schedule_name( - NativeToolPredictorSchedule schedule); - -bool parse_native_tool_predictor_schedule( - const std::string & value, - NativeToolPredictorSchedule & out); - struct SemanticToolPredictorConfig { std::string url; std::string model; @@ -33,8 +22,6 @@ struct SemanticToolPredictorConfig { int native_max_ctx = 4096; int timeout_ms = 2000; int max_tokens = 96; - NativeToolPredictorSchedule native_schedule = - NativeToolPredictorSchedule::BeforeModel; // Conservative prior used by the measured tool-execution admission // policy. The base predictor currently emits no calibrated probability. double execution_confidence = 0.75; @@ -44,10 +31,6 @@ struct SemanticToolPredictorConfig { return !native_model_path.empty() && !native_ipc_bin.empty(); } bool enabled() const { return native_enabled() || http_enabled(); } - bool native_runs_before_model() const { - return native_enabled() && - native_schedule == NativeToolPredictorSchedule::BeforeModel; - } }; struct SemanticToolCall { @@ -94,7 +77,8 @@ json build_semantic_tool_predictor_request( // decoded response is parsed semantically before any target token IDs exist. std::string build_native_semantic_tool_predictor_prompt( const json & predictor_request, - std::string & error); + std::string & error, + const std::chrono::steady_clock::time_point * deadline = nullptr); bool parse_native_semantic_tool_prediction( const std::string & generated_text, diff --git a/server/src/server/server_main.cpp b/server/src/server/server_main.cpp index fe1ec19ec..af126bbe2 100644 --- a/server/src/server/server_main.cpp +++ b/server/src/server/server_main.cpp @@ -27,6 +27,7 @@ #include "kvflash_pager.h" #include +#include #include #include #include @@ -71,6 +72,26 @@ static bool parse_double_list(const char * value, std::vector & out) { return !out.empty(); } +static bool parse_int_strict(const char * value, int & out) { + if (!value || !*value) return false; + const char * end = value + std::strlen(value); + const auto parsed = std::from_chars(value, end, out); + return parsed.ec == std::errc{} && parsed.ptr == end; +} + +static bool parse_double_strict(const char * value, double & out) { + if (!value || !*value) return false; + char * end = nullptr; + errno = 0; + const double parsed = std::strtod(value, &end); + if (errno == ERANGE || end == value || !end || *end != '\0' || + !std::isfinite(parsed)) { + return false; + } + out = parsed; + return true; +} + static bool environment_flag_enabled(const char * name) { const char * value = std::getenv(name); return value && *value && std::strcmp(value, "0") != 0; @@ -197,10 +218,6 @@ static void print_usage(const char * prog) { " --tool-hint-native-gpu Predictor GPU (default: 0).\n" " --tool-hint-native-max-ctx \n" " Predictor context capacity (default: 4096).\n" - " --tool-hint-native-schedule \n" - " before-model (default) runs Qwen before DS4\n" - " so shared-GPU decoding cannot be slowed;\n" - " overlap is an experimental throughput mode.\n" " --tool-hint-native-work-dir \n" " Optional private IPC scratch directory.\n" " --tool-hint-timeout-ms Hard native/HTTP predictor deadline\n" @@ -289,7 +306,6 @@ int main(int argc, char ** argv) { bool fast_rollback_forced_off = false; bool target_split_fast_rollback_cli = false; bool adaptive_experts_set = false; // --adaptive-experts (MoE architectures only) - bool native_tool_predictor_schedule_set = false; // Track which thinking-budget tunables the operator set via CLI. // Those values win over the model card (spec §3.1: "Explicit CLI @@ -623,36 +639,28 @@ int main(int argc, char ** argv) { sconfig.semantic_tool_predictor.native_work_dir = argv[++i]; } else if (std::strcmp(argv[i], "--tool-hint-native-gpu") == 0 && i + 1 < argc) { - sconfig.semantic_tool_predictor.native_gpu = std::atoi(argv[++i]); - if (sconfig.semantic_tool_predictor.native_gpu < 0) { + if (!parse_int_strict( + argv[++i], sconfig.semantic_tool_predictor.native_gpu) || + sconfig.semantic_tool_predictor.native_gpu < 0) { std::fprintf(stderr, "[server] --tool-hint-native-gpu must be non-negative\n"); return 2; } } else if (std::strcmp(argv[i], "--tool-hint-native-max-ctx") == 0 && i + 1 < argc) { - sconfig.semantic_tool_predictor.native_max_ctx = std::atoi(argv[++i]); - if (sconfig.semantic_tool_predictor.native_max_ctx <= 0) { + if (!parse_int_strict( + argv[++i], sconfig.semantic_tool_predictor.native_max_ctx) || + sconfig.semantic_tool_predictor.native_max_ctx <= 0) { std::fprintf(stderr, "[server] --tool-hint-native-max-ctx must be positive\n"); return 2; } - } else if (std::strcmp(argv[i], "--tool-hint-native-schedule") == 0 && - i + 1 < argc) { - native_tool_predictor_schedule_set = true; - if (!parse_native_tool_predictor_schedule( - argv[++i], - sconfig.semantic_tool_predictor.native_schedule)) { - std::fprintf(stderr, - "[server] --tool-hint-native-schedule must be " - "before-model or overlap\n"); - return 2; - } } else if ((std::strcmp(argv[i], "--tool-hint-timeout-ms") == 0 || std::strcmp(argv[i], "--tool-hint-sidecar-timeout-ms") == 0) && i + 1 < argc) { - sconfig.semantic_tool_predictor.timeout_ms = std::atoi(argv[++i]); - if (sconfig.semantic_tool_predictor.timeout_ms <= 0) { + if (!parse_int_strict( + argv[++i], sconfig.semantic_tool_predictor.timeout_ms) || + sconfig.semantic_tool_predictor.timeout_ms <= 0) { std::fprintf(stderr, "[server] --tool-hint-timeout-ms must be positive\n"); return 2; @@ -660,8 +668,9 @@ int main(int argc, char ** argv) { } else if ((std::strcmp(argv[i], "--tool-hint-max-tokens") == 0 || std::strcmp(argv[i], "--tool-hint-sidecar-max-tokens") == 0) && i + 1 < argc) { - sconfig.semantic_tool_predictor.max_tokens = std::atoi(argv[++i]); - if (sconfig.semantic_tool_predictor.max_tokens <= 0) { + if (!parse_int_strict( + argv[++i], sconfig.semantic_tool_predictor.max_tokens) || + sconfig.semantic_tool_predictor.max_tokens <= 0) { std::fprintf(stderr, "[server] --tool-hint-max-tokens must be positive\n"); return 2; @@ -669,9 +678,8 @@ int main(int argc, char ** argv) { } else if (std::strcmp( argv[i], "--tool-hint-execution-confidence") == 0 && i + 1 < argc) { - sconfig.semantic_tool_predictor.execution_confidence = - std::atof(argv[++i]); - if (!std::isfinite( + if (!parse_double_strict( + argv[++i], sconfig.semantic_tool_predictor.execution_confidence) || sconfig.semantic_tool_predictor.execution_confidence < 0.0 || sconfig.semantic_tool_predictor.execution_confidence > 1.0) { @@ -704,8 +712,9 @@ int main(int argc, char ** argv) { } } else if (std::strcmp(argv[i], "--tool-spec-timeout-ms") == 0 && i + 1 < argc) { - sconfig.tool_speculation.timeout_ms = std::atoi(argv[++i]); - if (sconfig.tool_speculation.timeout_ms <= 0) { + if (!parse_int_strict( + argv[++i], sconfig.tool_speculation.timeout_ms) || + sconfig.tool_speculation.timeout_ms <= 0) { std::fprintf(stderr, "[server] --tool-spec-timeout-ms must be positive\n"); return 2; @@ -713,9 +722,8 @@ int main(int argc, char ** argv) { } else if (std::strcmp( argv[i], "--tool-spec-max-model-slowdown") == 0 && i + 1 < argc) { - sconfig.tool_speculation.max_model_slowdown_ratio = - std::atof(argv[++i]); - if (!std::isfinite( + if (!parse_double_strict( + argv[++i], sconfig.tool_speculation.max_model_slowdown_ratio) || sconfig.tool_speculation.max_model_slowdown_ratio < 1.0) { std::fprintf(stderr, @@ -800,8 +808,7 @@ int main(int argc, char ** argv) { const bool semantic_native_predictor_requested = !sconfig.semantic_tool_predictor.native_model_path.empty() || !sconfig.semantic_tool_predictor.native_ipc_bin.empty() || - !sconfig.semantic_tool_predictor.native_work_dir.empty() || - native_tool_predictor_schedule_set; + !sconfig.semantic_tool_predictor.native_work_dir.empty(); if (semantic_native_predictor_requested && !sconfig.semantic_tool_predictor.native_enabled()) { std::fprintf(stderr, @@ -815,6 +822,12 @@ int main(int argc, char ** argv) { !sconfig.tool_speculation.allowed_tools.empty() || !sconfig.tool_speculation.cpu_affinity.empty(); if (tool_speculation_requested) { + if (!tool_speculation_executor_isolation_supported()) { + std::fprintf(stderr, + "[server] tool speculation requires Linux glibc >= 2.34 " + "for child descriptor isolation\n"); + return 2; + } if (sconfig.tool_speculation.executor_path.empty() || sconfig.tool_speculation.profile_path.empty() || sconfig.tool_speculation.allowed_tools.empty()) { @@ -1390,11 +1403,9 @@ int main(int argc, char ** argv) { std::fprintf(stderr, "[server] │ tool_hint_gpu = %d (max_ctx=%d)\n", predictor.native_gpu, predictor.native_max_ctx); - std::fprintf(stderr, - "[server] │ tool_hint_schedule= %s\n", - native_tool_predictor_schedule_name( - predictor.native_schedule)); } + std::fprintf(stderr, + "[server] │ tool_hint_schedule= before-model\n"); std::fprintf(stderr, "[server] │ tool_hint_timeout= %d ms\n", predictor.timeout_ms); std::fprintf(stderr, "[server] │ tool_hint_execute= %s (confidence=%.3f)\n", diff --git a/server/src/server/tokenizer.cpp b/server/src/server/tokenizer.cpp index 5ff4b1a78..ebc74e112 100644 --- a/server/src/server/tokenizer.cpp +++ b/server/src/server/tokenizer.cpp @@ -18,6 +18,20 @@ namespace dflash::common { +namespace { + +bool preprocessing_deadline_expired( + const std::chrono::steady_clock::time_point * deadline, + size_t & operations, + bool & timed_out) { + if (!deadline || (operations++ & 255U) != 0U) return false; + if (std::chrono::steady_clock::now() < *deadline) return false; + timed_out = true; + return true; +} + +} // namespace + // ─── Unicode helpers ──────────────────────────────────────────────────── static int utf8_len(uint8_t c) { @@ -147,11 +161,15 @@ static bool is_newline(uint32_t cp) { // \s+(?!\S) | // \s+ -std::vector Tokenizer::pre_tokenize(const std::string & text) const { +std::vector Tokenizer::pre_tokenize( + const std::string & text, + const std::chrono::steady_clock::time_point * deadline, + bool & timed_out) const { std::vector pieces; const char * s = text.c_str(); const size_t len = text.size(); size_t pos = 0; + size_t operations = 0; auto peek_cp = [&](size_t p, int * cplen) -> uint32_t { if (p >= len) { *cplen = 0; return 0; } @@ -159,6 +177,8 @@ std::vector Tokenizer::pre_tokenize(const std::string & text) const }; while (pos < len) { + if (preprocessing_deadline_expired( + deadline, operations, timed_out)) return {}; size_t start = pos; int cplen = 0; uint32_t cp = peek_cp(pos, &cplen); @@ -204,6 +224,8 @@ std::vector Tokenizer::pre_tokenize(const std::string & text) const // One or more letter/mark chars if (cl > 0 && (is_letter(c) || is_mark(c))) { while (cl > 0 && (is_letter(c) || is_mark(c))) { + if (preprocessing_deadline_expired( + deadline, operations, timed_out)) return {}; p += cl; c = peek_cp(p, &cl); } @@ -234,12 +256,16 @@ std::vector Tokenizer::pre_tokenize(const std::string & text) const size_t punc_start = p; while (cl > 0 && !is_whitespace(c) && !is_letter(c) && !is_mark(c) && !is_digit(c)) { + if (preprocessing_deadline_expired( + deadline, operations, timed_out)) return {}; p += cl; c = peek_cp(p, &cl); } if (p > punc_start) { // Trailing newlines while (cl > 0 && is_newline(c)) { + if (preprocessing_deadline_expired( + deadline, operations, timed_out)) return {}; p += cl; c = peek_cp(p, &cl); } @@ -256,11 +282,15 @@ std::vector Tokenizer::pre_tokenize(const std::string & text) const uint32_t c = peek_cp(p, &cl); // Consume leading whitespace while (cl > 0 && is_whitespace(c) && !is_newline(c)) { + if (preprocessing_deadline_expired( + deadline, operations, timed_out)) return {}; p += cl; c = peek_cp(p, &cl); } if (cl > 0 && is_newline(c)) { while (cl > 0 && is_newline(c)) { + if (preprocessing_deadline_expired( + deadline, operations, timed_out)) return {}; p += cl; c = peek_cp(p, &cl); } @@ -277,6 +307,8 @@ std::vector Tokenizer::pre_tokenize(const std::string & text) const c = peek_cp(p, &cl); size_t prev_p = pos; // position before last whitespace char while (cl > 0 && is_whitespace(c)) { + if (preprocessing_deadline_expired( + deadline, operations, timed_out)) return {}; prev_p = p; p += cl; c = peek_cp(p, &cl); @@ -351,18 +383,30 @@ static std::string byte_to_gpt2_unicode(uint8_t b) { } // Convert a raw UTF-8 text piece to GPT-2 byte-encoded form for BPE lookup. -static std::string encode_gpt2_bpe(const std::string & text) { +static std::string encode_gpt2_bpe( + const std::string & text, + const std::chrono::steady_clock::time_point * deadline, + size_t & operations, + bool & timed_out) { std::string out; out.reserve(text.size() * 2); // GPT-2 encoding may expand for (uint8_t b : text) { + if (preprocessing_deadline_expired( + deadline, operations, timed_out)) return {}; out += byte_to_gpt2_unicode(b); } return out; } // Encode a single pre-tokenized piece using BPE merges. -std::vector Tokenizer::bpe_encode_piece(const std::string & piece) const { +std::vector Tokenizer::bpe_encode_piece( + const std::string & piece, + const std::chrono::steady_clock::time_point * deadline, + bool & timed_out) const { if (piece.empty()) return {}; + size_t operations = 0; + if (preprocessing_deadline_expired( + deadline, operations, timed_out)) return {}; std::vector symbols; @@ -380,6 +424,8 @@ std::vector Tokenizer::bpe_encode_piece(const std::string & piece) cons std::string encoded; encoded.reserve(sp_piece.size()); for (char c : sp_piece) { + if (preprocessing_deadline_expired( + deadline, operations, timed_out)) return {}; if (c == ' ') { encoded += "\xe2\x96\x81"; } else { @@ -397,6 +443,8 @@ std::vector Tokenizer::bpe_encode_piece(const std::string & piece) cons const char * p = encoded.c_str(); const char * end = p + encoded.size(); while (p < end) { + if (preprocessing_deadline_expired( + deadline, operations, timed_out)) return {}; int cplen; utf8_decode(p, (size_t)(end - p), &cplen); if (cplen <= 0) cplen = 1; @@ -414,7 +462,9 @@ std::vector Tokenizer::bpe_encode_piece(const std::string & piece) cons } } else { // GPT-2 BPE: convert raw text to GPT-2 byte encoding for vocab lookup. - std::string encoded = encode_gpt2_bpe(piece); + std::string encoded = encode_gpt2_bpe( + piece, deadline, operations, timed_out); + if (timed_out) return {}; // Try to find the encoded piece as a single token first. auto it = token_to_id_.find(encoded); @@ -424,6 +474,8 @@ std::vector Tokenizer::bpe_encode_piece(const std::string & piece) cons // Split into individual GPT-2-encoded bytes as initial BPE symbols. for (size_t i = 0; i < piece.size(); i++) { + if (preprocessing_deadline_expired( + deadline, operations, timed_out)) return {}; std::string sym = byte_to_gpt2_unicode((uint8_t)piece[i]); auto sit = token_to_id_.find(sym); if (sit != token_to_id_.end()) { @@ -446,10 +498,14 @@ std::vector Tokenizer::bpe_encode_piece(const std::string & piece) cons // Iteratively merge the highest-priority pair until no more merges apply. while (symbols.size() > 1) { + if (preprocessing_deadline_expired( + deadline, operations, timed_out)) return {}; int best_rank = std::numeric_limits::max(); size_t best_pos = SIZE_MAX; for (size_t i = 0; i + 1 < symbols.size(); i++) { + if (preprocessing_deadline_expired( + deadline, operations, timed_out)) return {}; std::string pair = symbols[i] + " " + symbols[i + 1]; auto mit = merge_rank_.find(pair); if (mit != merge_rank_.end() && mit->second < best_rank) { @@ -469,6 +525,8 @@ std::vector Tokenizer::bpe_encode_piece(const std::string & piece) cons std::vector ids; ids.reserve(symbols.size()); for (const auto & sym : symbols) { + if (preprocessing_deadline_expired( + deadline, operations, timed_out)) return {}; auto sit = token_to_id_.find(sym); if (sit != token_to_id_.end()) { ids.push_back(sit->second); @@ -493,6 +551,8 @@ std::vector Tokenizer::bpe_encode_piece(const std::string & piece) cons const char * p = sym.c_str(); const char * end = p + sym.size(); while (p < end) { + if (preprocessing_deadline_expired( + deadline, operations, timed_out)) return {}; int cplen; uint32_t cp = utf8_decode(p, (size_t)(end - p), &cplen); uint8_t orig_byte; @@ -633,13 +693,25 @@ bool Tokenizer::load_from_gguf(const char * model_path) { return true; } -std::vector Tokenizer::encode(const std::string & text) const { +std::vector Tokenizer::encode_impl( + const std::string & text, + const std::chrono::steady_clock::time_point * deadline, + bool & timed_out) const { + size_t operations = 0; + if (preprocessing_deadline_expired( + deadline, operations, timed_out)) return {}; // If no added tokens, fast path: pre-tokenize → BPE entire text. if (added_tokens_.empty()) { - std::vector pieces = pre_tokenize(text); + std::vector pieces = pre_tokenize( + text, deadline, timed_out); + if (timed_out) return {}; std::vector ids; for (const auto & piece : pieces) { - auto piece_ids = bpe_encode_piece(piece); + if (preprocessing_deadline_expired( + deadline, operations, timed_out)) return {}; + auto piece_ids = bpe_encode_piece( + piece, deadline, timed_out); + if (timed_out) return {}; ids.insert(ids.end(), piece_ids.begin(), piece_ids.end()); } return ids; @@ -650,9 +722,13 @@ std::vector Tokenizer::encode(const std::string & text) const { std::vector ids; size_t pos = 0; while (pos < text.size()) { + if (preprocessing_deadline_expired( + deadline, operations, timed_out)) return {}; // Try to match any added token at current position. bool matched = false; for (const auto & [tok_str, tok_id] : added_tokens_) { + if (preprocessing_deadline_expired( + deadline, operations, timed_out)) return {}; if (pos + tok_str.size() <= text.size() && text.compare(pos, tok_str.size(), tok_str) == 0) { ids.push_back(tok_id); @@ -666,6 +742,8 @@ std::vector Tokenizer::encode(const std::string & text) const { // Find the next special token (or end of string). size_t next_special = text.size(); for (const auto & [tok_str, tok_id] : added_tokens_) { + if (preprocessing_deadline_expired( + deadline, operations, timed_out)) return {}; size_t found = text.find(tok_str, pos); if (found != std::string::npos && found < next_special) { next_special = found; @@ -674,9 +752,13 @@ std::vector Tokenizer::encode(const std::string & text) const { // Pre-tokenize + BPE the normal segment. std::string segment = text.substr(pos, next_special - pos); - std::vector pieces = pre_tokenize(segment); + std::vector pieces = pre_tokenize( + segment, deadline, timed_out); + if (timed_out) return {}; for (const auto & piece : pieces) { - auto piece_ids = bpe_encode_piece(piece); + auto piece_ids = bpe_encode_piece( + piece, deadline, timed_out); + if (timed_out) return {}; ids.insert(ids.end(), piece_ids.begin(), piece_ids.end()); } pos = next_special; @@ -684,6 +766,24 @@ std::vector Tokenizer::encode(const std::string & text) const { return ids; } +std::vector Tokenizer::encode(const std::string & text) const { + bool timed_out = false; + return encode_impl(text, nullptr, timed_out); +} + +bool Tokenizer::encode_until( + const std::string & text, + std::chrono::steady_clock::time_point deadline, + std::vector & out) const { + bool timed_out = false; + out = encode_impl(text, &deadline, timed_out); + if (timed_out || std::chrono::steady_clock::now() >= deadline) { + out.clear(); + return false; + } + return true; +} + // GPT-2 byte-level BPE uses a Unicode mapping where each byte 0-255 is // represented by a specific Unicode codepoint. Bytes that already have a // printable representation (33-126, 161-172, 174-255) map to themselves; diff --git a/server/src/server/tokenizer.h b/server/src/server/tokenizer.h index 5484fa472..045fb8aad 100644 --- a/server/src/server/tokenizer.h +++ b/server/src/server/tokenizer.h @@ -9,6 +9,7 @@ #pragma once +#include #include #include #include @@ -31,6 +32,12 @@ class Tokenizer { // ─── Encode ────────────────────────────────────────────────────── // Tokenize a UTF-8 string into token IDs. std::vector encode(const std::string & text) const; + // Predictor preprocessing uses the same tokenizer but must not overrun a + // request deadline. Returns false and clears `out` on expiry. + bool encode_until( + const std::string & text, + std::chrono::steady_clock::time_point deadline, + std::vector & out) const; // ─── Decode ────────────────────────────────────────────────────── // Convert a single token ID to its text representation. @@ -55,10 +62,20 @@ class Tokenizer { private: // Pre-tokenize text into pieces using Qwen3/3.5 regex pattern. - std::vector pre_tokenize(const std::string & text) const; + std::vector pre_tokenize( + const std::string & text, + const std::chrono::steady_clock::time_point * deadline, + bool & timed_out) const; // Apply BPE merges to a single pre-tokenized piece. - std::vector bpe_encode_piece(const std::string & piece) const; + std::vector bpe_encode_piece( + const std::string & piece, + const std::chrono::steady_clock::time_point * deadline, + bool & timed_out) const; + std::vector encode_impl( + const std::string & text, + const std::chrono::steady_clock::time_point * deadline, + bool & timed_out) const; // Vocabulary: id → token string std::vector id_to_token_; diff --git a/server/src/server/tool_speculation.cpp b/server/src/server/tool_speculation.cpp index 85f6b44fa..40c43d944 100644 --- a/server/src/server/tool_speculation.cpp +++ b/server/src/server/tool_speculation.cpp @@ -27,7 +27,15 @@ # include # include # endif -extern char ** environ; +#endif + +#if !defined(_WIN32) && defined(__GLIBC__) && defined(__GLIBC_PREREQ) +# if __GLIBC_PREREQ(2, 34) +# define DFLASH_TOOL_SPEC_HAS_CLOSEFROM 1 +# endif +#endif +#ifndef DFLASH_TOOL_SPEC_HAS_CLOSEFROM +# define DFLASH_TOOL_SPEC_HAS_CLOSEFROM 0 #endif namespace dflash::common { @@ -104,75 +112,41 @@ std::vector executor_environment( const std::string & accelerator_relation, const std::vector & cpu_affinity) { std::vector values; - static constexpr const char * kResourceKey = - "DFLASH_TOOL_SPECULATION_RESOURCE_PERCENTAGE="; - static constexpr size_t kResourceKeyLen = - sizeof("DFLASH_TOOL_SPECULATION_RESOURCE_PERCENTAGE=") - 1; - static constexpr const char * kRelationKey = - "DFLASH_TOOL_SPECULATION_ACCELERATOR_RELATION="; - static constexpr size_t kRelationKeyLen = - sizeof("DFLASH_TOOL_SPECULATION_ACCELERATOR_RELATION=") - 1; - static constexpr const char * kCpuAffinityKey = - "DFLASH_TOOL_SPECULATION_CPU_AFFINITY="; - static constexpr size_t kCpuAffinityKeyLen = - sizeof("DFLASH_TOOL_SPECULATION_CPU_AFFINITY=") - 1; - static constexpr const char * kEnabledKey = - "DFLASH_TOOL_SPECULATION="; - static constexpr size_t kEnabledKeyLen = - sizeof("DFLASH_TOOL_SPECULATION=") - 1; - bool resource_replaced = false; - bool relation_replaced = false; - bool cpu_affinity_replaced = false; - bool enabled_replaced = false; - for (char ** item = environ; item && *item; ++item) { - const std::string value(*item); - if (value.compare(0, kResourceKeyLen, kResourceKey) == 0) { - values.push_back( - std::string(kResourceKey) + - std::to_string(resource_percentage)); - resource_replaced = true; - } else if (value.compare(0, kRelationKeyLen, kRelationKey) == 0) { - values.push_back( - std::string(kRelationKey) + accelerator_relation); - relation_replaced = true; - } else if (value.compare( - 0, kCpuAffinityKeyLen, kCpuAffinityKey) == 0) { - // Drop a stale inherited value when this executor has no CPU - // lane. Otherwise replace it with the verified canonical list. - if (!cpu_affinity.empty()) { - values.push_back( - std::string(kCpuAffinityKey) + - format_cpu_affinity(cpu_affinity)); - cpu_affinity_replaced = true; - } - } else if (value.compare(0, kEnabledKeyLen, kEnabledKey) == 0) { - values.push_back(std::string(kEnabledKey) + "1"); - enabled_replaced = true; - } else { - values.push_back(value); + // Do not copy the long-running server's environment into a tool process: + // it commonly contains model-provider keys and upstream credentials. Keep + // only the small runtime surface needed by executable/script adapters. + static constexpr const char * kInherited[] = { + "PATH", + "LANG", + "LC_ALL", + "LC_CTYPE", + "TZ", + "LD_LIBRARY_PATH", + "DFLASH_TRACE_TRAINING_REPORT", + "DFLASH_TRACE_WORKFLOW_REGISTRY", + }; + for (const char * name : kInherited) { + if (const char * value = std::getenv(name); value && *value) { + values.push_back(std::string(name) + "=" + value); } } - if (!resource_replaced) { - values.push_back( - std::string(kResourceKey) + - std::to_string(resource_percentage)); - } - if (!relation_replaced) { - values.push_back(std::string(kRelationKey) + accelerator_relation); - } - if (!cpu_affinity.empty() && !cpu_affinity_replaced) { + values.push_back("DFLASH_TOOL_SPECULATION=1"); + values.push_back( + "DFLASH_TOOL_SPECULATION_RESOURCE_PERCENTAGE=" + + std::to_string(resource_percentage)); + values.push_back( + "DFLASH_TOOL_SPECULATION_ACCELERATOR_RELATION=" + + accelerator_relation); + if (!cpu_affinity.empty()) { values.push_back( - std::string(kCpuAffinityKey) + + "DFLASH_TOOL_SPECULATION_CPU_AFFINITY=" + format_cpu_affinity(cpu_affinity)); } - if (!enabled_replaced) { - values.push_back(std::string(kEnabledKey) + "1"); - } return values; } # if defined(__linux__) -bool pin_and_verify_child_cpu_affinity( +bool wait_for_child_cpu_affinity( pid_t child, const std::vector & cpus, std::string & error) { @@ -190,32 +164,42 @@ bool pin_and_verify_child_cpu_affinity( } CPU_SET(cpu, &requested); } - if (::sched_setaffinity(child, sizeof(requested), &requested) != 0) { - error = std::string("executor sched_setaffinity failed: ") + - std::strerror(errno); - return false; - } - cpu_set_t observed; - CPU_ZERO(&observed); - if (::sched_getaffinity(child, sizeof(observed), &observed) != 0) { - error = std::string("executor sched_getaffinity failed: ") + - std::strerror(errno); - return false; - } - for (int cpu = 0; cpu < CPU_SETSIZE; ++cpu) { - if (CPU_ISSET(cpu, &requested) != CPU_ISSET(cpu, &observed)) { - error = "executor CPU affinity verification mismatch"; + const auto deadline = std::chrono::steady_clock::now() + + std::chrono::milliseconds(250); + do { + cpu_set_t observed; + CPU_ZERO(&observed); + if (::sched_getaffinity(child, sizeof(observed), &observed) == 0) { + bool matches = true; + for (int cpu = 0; cpu < CPU_SETSIZE; ++cpu) { + if (CPU_ISSET(cpu, &requested) != CPU_ISSET(cpu, &observed)) { + matches = false; + break; + } + } + if (matches) { + error.clear(); + return true; + } + } else if (errno != EINTR) { + error = std::string("executor sched_getaffinity failed: ") + + std::strerror(errno); return false; } - } - error.clear(); - return true; + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } while (std::chrono::steady_clock::now() < deadline); + error = "executor CPU affinity verification mismatch"; + return false; } # endif #endif } // namespace +bool tool_speculation_executor_isolation_supported() { + return DFLASH_TOOL_SPEC_HAS_CLOSEFROM != 0; +} + bool CanonicalToolInvocation::from_parts( const std::string & name, const json & arguments, @@ -405,6 +389,10 @@ bool qualify_tool_speculation_cpu_affinity( return true; } #if defined(__linux__) + if (::access("/usr/bin/taskset", X_OK) != 0) { + error = "tool CPU affinity requires executable /usr/bin/taskset"; + return false; + } const long configured_cpus = ::sysconf(_SC_NPROCESSORS_CONF); if (configured_cpus <= 0) { error = "cannot determine configured CPU count"; @@ -770,8 +758,7 @@ void ToolSpeculationAttempt::start() { if (spawn_status == 0 && output_pipe[1] != STDOUT_FILENO) { spawn_status = posix_spawn_file_actions_addclose(&actions, output_pipe[1]); } -# if defined(__GLIBC__) && defined(__GLIBC_PREREQ) -# if __GLIBC_PREREQ(2, 34) +# if DFLASH_TOOL_SPEC_HAS_CLOSEFROM // The executor receives only stdin/stdout/stderr. In particular, it must // not inherit listening sockets, live client connections, model IPC pipes, // or accelerator descriptors from the long-running server. @@ -779,7 +766,6 @@ void ToolSpeculationAttempt::start() { spawn_status = posix_spawn_file_actions_addclosefrom_np( &actions, STDERR_FILENO + 1); } -# endif # endif std::vector env_storage = executor_environment( @@ -790,17 +776,28 @@ void ToolSpeculationAttempt::start() { for (std::string & value : env_storage) env.push_back(value.data()); env.push_back(nullptr); - std::string executable = config_.executor_path; + std::string executable = config_.cpu_affinity.empty() + ? config_.executor_path + : "/usr/bin/taskset"; std::string protocol_arg = "--dflash-tool-spec-v1"; - char * argv[] = { - executable.data(), - protocol_arg.data(), - nullptr, - }; + std::string affinity_arg = format_cpu_affinity(config_.cpu_affinity); + std::string taskset_cpu_arg = "-c"; + std::vector argv; + argv.push_back(executable.data()); + if (!config_.cpu_affinity.empty()) { + // taskset applies the mask before execve(), so no executor startup + // code can run on the model CPUs. The request payload remains withheld + // until the parent verifies the resulting mask below. + argv.push_back(taskset_cpu_arg.data()); + argv.push_back(affinity_arg.data()); + argv.push_back(config_.executor_path.data()); + } + argv.push_back(protocol_arg.data()); + argv.push_back(nullptr); pid_t child = -1; if (spawn_status == 0) { spawn_status = ::posix_spawn( - &child, executable.c_str(), &actions, &attributes, argv, + &child, executable.c_str(), &actions, &attributes, argv.data(), env.data()); } if (actions_initialized) { @@ -822,7 +819,7 @@ void ToolSpeculationAttempt::start() { # if defined(__linux__) if (!config_.cpu_affinity.empty()) { std::string affinity_error; - if (!pin_and_verify_child_cpu_affinity( + if (!wait_for_child_cpu_affinity( child, config_.cpu_affinity, affinity_error)) { signal_executor_process_group(child, SIGKILL); int child_status = 0; diff --git a/server/src/server/tool_speculation.h b/server/src/server/tool_speculation.h index 07c8de813..01544f36d 100644 --- a/server/src/server/tool_speculation.h +++ b/server/src/server/tool_speculation.h @@ -156,6 +156,7 @@ struct ToolSpeculationConfig { // Capture the model process affinity and fail closed unless it is physically // disjoint from the configured child executor CPUs. No-op when no CPU lane is // requested. +bool tool_speculation_executor_isolation_supported(); bool qualify_tool_speculation_cpu_affinity(ToolSpeculationConfig & config, std::string & error); diff --git a/server/test/smoke_qwen3_tool_predictor_ipc.cpp b/server/test/smoke_qwen3_tool_predictor_ipc.cpp index 866bbaf96..41512d44a 100644 --- a/server/test/smoke_qwen3_tool_predictor_ipc.cpp +++ b/server/test/smoke_qwen3_tool_predictor_ipc.cpp @@ -149,6 +149,7 @@ int main(int argc, char ** argv) { } const std::vector cases = production_cases(); + constexpr size_t kMinimumExactMatches = 9; size_t valid = 0; size_t name_matches = 0; size_t exact_matches = 0; @@ -178,6 +179,7 @@ int main(int argc, char ** argv) { {"valid", valid}, {"name_matches", name_matches}, {"exact_matches", exact_matches}, + {"minimum_exact_matches", kMinimumExactMatches}, {"name_accuracy", cases.empty() ? 0.0 : static_cast(name_matches) / static_cast(cases.size())}, @@ -187,5 +189,8 @@ int main(int argc, char ** argv) { {"wall_p50_ms", wall_p50}, }; std::printf("%s\n", summary.dump().c_str()); - return valid == cases.size() && name_matches == cases.size() ? 0 : 1; + return valid == cases.size() && name_matches == cases.size() && + exact_matches >= kMinimumExactMatches + ? 0 + : 1; } diff --git a/server/test/test_semantic_tool_hint.cpp b/server/test/test_semantic_tool_hint.cpp index 7860de3a0..0d7ee9cfd 100644 --- a/server/test/test_semantic_tool_hint.cpp +++ b/server/test/test_semantic_tool_hint.cpp @@ -80,6 +80,63 @@ TEST_CASE(SemanticToolHintFixture, rejects_unknown_predicted_function) { CHECK(error == "predictor_selected_unknown_function"); } +TEST_CASE(SemanticToolHintFixture, parses_strict_json_content_fallback) { + const json response = { + {"choices", json::array({{ + {"message", { + {"content", + " {\"name\":\"get_weather\",\"arguments\":{\"city\":\"Rome\"}}\n"}, + }}, + }})}, + }; + SemanticToolCall call; + std::string error; + CHECK(parse_semantic_tool_prediction( + response, weather_tools(), call, error)); + CHECK(call.name == "get_weather"); + CHECK(call.arguments["city"] == "Rome"); +} + +TEST_CASE(SemanticToolHintFixture, rejects_ambiguous_sidecar_calls) { + const json function = { + {"name", "get_weather"}, + {"arguments", "{\"city\":\"Rome\"}"}, + }; + const json response = { + {"choices", json::array({{ + {"message", { + {"content", + "{\"name\":\"get_weather\",\"arguments\":{\"city\":\"Rome\"}}"}, + {"tool_calls", json::array({ + {{"function", function}}, + {{"function", function}}, + })}, + }}, + }})}, + }; + SemanticToolCall call; + std::string error; + CHECK(!parse_semantic_tool_prediction( + response, weather_tools(), call, error)); + CHECK(error == "predictor_response_requires_single_tool_call"); +} + +TEST_CASE(SemanticToolHintFixture, rejects_prose_wrapped_json_content) { + const json response = { + {"choices", json::array({{ + {"message", { + {"content", + "call {\"name\":\"get_weather\",\"arguments\":{\"city\":\"Rome\"}}"}, + }}, + }})}, + }; + SemanticToolCall call; + std::string error; + CHECK(!parse_semantic_tool_prediction( + response, weather_tools(), call, error)); + CHECK(error == "predictor_response_has_no_valid_call"); +} + TEST_CASE(SemanticToolHintFixture, materializes_declared_optional_defaults) { json tools = weather_tools(); tools[0]["function"]["parameters"]["properties"]["unit"]["default"] = @@ -134,19 +191,20 @@ TEST_CASE(SemanticToolHintFixture, native_predictor_config_is_independent_of_htt CHECK(config.native_enabled()); CHECK(!config.http_enabled()); CHECK(config.enabled()); - CHECK(config.native_runs_before_model()); - CHECK(std::string(native_tool_predictor_schedule_name( - config.native_schedule)) == "before-model"); } -TEST_CASE(SemanticToolHintFixture, native_predictor_overlap_is_explicit) { - NativeToolPredictorSchedule schedule = - NativeToolPredictorSchedule::BeforeModel; - CHECK(parse_native_tool_predictor_schedule("overlap", schedule)); - CHECK(schedule == NativeToolPredictorSchedule::Overlap); - CHECK(std::string(native_tool_predictor_schedule_name(schedule)) == - "overlap"); - CHECK(!parse_native_tool_predictor_schedule("automatic", schedule)); +TEST_CASE(SemanticToolHintFixture, native_prompt_honors_tool_choice_none) { + const json request = { + {"messages", json::array({{ + {"role", "user"}, + {"content", "Do not call a tool"}, + }})}, + {"tools", weather_tools()}, + {"tool_choice", "none"}, + }; + std::string error; + CHECK(build_native_semantic_tool_predictor_prompt(request, error).empty()); + CHECK(error == "native_predictor_tool_choice_none"); } TEST_CASE(SemanticToolHintFixture, native_prompt_uses_qwen_tool_contract) { @@ -173,6 +231,19 @@ TEST_CASE(SemanticToolHintFixture, native_prompt_uses_qwen_tool_contract) { CHECK(prompt.find("\n\n") != std::string::npos); } +TEST_CASE(SemanticToolHintFixture, native_prompt_honors_preprocessing_deadline) { + const json request = { + {"messages", json::array({{{"role", "user"}, {"content", "weather"}}})}, + {"tools", weather_tools()}, + }; + const auto expired = std::chrono::steady_clock::now() - + std::chrono::milliseconds(1); + std::string error; + CHECK(build_native_semantic_tool_predictor_prompt( + request, error, &expired).empty()); + CHECK(error == "native_predictor_timeout"); +} + TEST_CASE(SemanticToolHintFixture, parses_native_qwen_xml_semantics) { const std::string generated = "\n" diff --git a/server/test/test_server_unit.cpp b/server/test/test_server_unit.cpp index 1730512b3..eba9fa9ed 100644 --- a/server/test/test_server_unit.cpp +++ b/server/test/test_server_unit.cpp @@ -2491,6 +2491,18 @@ TEST_CASE(ServerUnitFixture, test_tool_speculation_defaults_to_automatic_predict TEST_ASSERT(req.automatic_tool_speculation_enabled); } +TEST_CASE(ServerUnitFixture, test_tool_choice_none_disables_speculation) { + TEST_ASSERT(http_detail::tool_choice_disables_tool_calls("none")); + TEST_ASSERT(http_detail::tool_choice_disables_tool_calls( + json{{"type", "none"}})); + TEST_ASSERT(!http_detail::tool_choice_disables_tool_calls(nullptr)); + TEST_ASSERT(!http_detail::tool_choice_disables_tool_calls("auto")); + TEST_ASSERT(!http_detail::tool_choice_disables_tool_calls("required")); + TEST_ASSERT(!http_detail::tool_choice_disables_tool_calls( + json{{"type", "function"}, + {"function", {{"name", "lookup"}}}})); +} + TEST_CASE(ServerUnitFixture, test_parse_request_sampler_applies_defaults_and_overrides) { SamplingDefaults defaults; defaults.has_temperature = true; @@ -2748,43 +2760,6 @@ TEST_CASE(ServerUnitFixture, test_deepseek4_render_empty_chat_gen_prompt) { TEST_ASSERT(out == expected); } -TEST_CASE(ServerUnitFixture, test_deepseek4_render_required_tool_instructions) { - std::vector msgs = { - {"user", "What is the weather?", ""}, - }; - const std::string tools = - R"([{"type":"function","function":{"name":"weather.get","parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}}])"; - const std::string out = render_chat_template( - msgs, ChatFormat::DEEPSEEK4, - /*add_generation_prompt=*/true, - /*enable_thinking=*/false, - tools, - /*tool_call_required=*/true); - - TEST_ASSERT(out.find("\n") != std::string::npos); - TEST_ASSERT(out.find("\"name\":\"weather.get\"") != std::string::npos); - TEST_ASSERT(out.find("") != std::string::npos); - TEST_ASSERT(out.find("") != std::string::npos); - TEST_ASSERT(out.find("MUST call exactly one") != std::string::npos); - TEST_ASSERT(out.find("<｜User｜>What is the weather?") != - std::string::npos); - const std::string suffix = "<｜Assistant｜>"; - TEST_ASSERT(out.size() >= suffix.size()); - TEST_ASSERT(out.compare(out.size() - suffix.size(), suffix.size(), suffix) == 0); -} - -TEST_CASE(ServerUnitFixture, test_deepseek4_auto_tool_is_not_forced) { - std::vector msgs = {{"user", "Hello", ""}}; - const std::string tools = - R"([{"type":"function","function":{"name":"weather.get","parameters":{"type":"object","properties":{}}}}])"; - const std::string out = render_chat_template( - msgs, ChatFormat::DEEPSEEK4, true, false, tools, - /*tool_call_required=*/false); - - TEST_ASSERT(out.find("You may call functions") != std::string::npos); - TEST_ASSERT(out.find("MUST call exactly one") == std::string::npos); -} - TEST_CASE(ServerUnitFixture, test_jinja_render_basic) { std::vector msgs = { {"system", "you are helpful", ""}, @@ -4137,6 +4112,7 @@ TEST_CASE(ServerUnitFixture, test_backend_ipc_rejects_public_work_dir) { cfg.bin = "/bin/true"; cfg.payload_path = "/tmp/dflash_test_backend_ipc_payload"; cfg.work_dir = dir_path; + cfg.require_private_work_dir = true; BackendIpcProcess proc; TEST_ASSERT(!proc.start(cfg)); @@ -4144,6 +4120,33 @@ TEST_CASE(ServerUnitFixture, test_backend_ipc_rejects_public_work_dir) { rmdir(dir_path.c_str()); } +TEST_CASE(ServerUnitFixture, test_backend_ipc_readiness_timeout) { + const std::string script_path = + "/tmp/dflash_test_backend_ipc_readiness_timeout.sh"; + unlink(script_path.c_str()); + int fd = open(script_path.c_str(), O_CREAT | O_TRUNC | O_WRONLY, 0700); + TEST_ASSERT(fd >= 0); + if (fd < 0) return; + const char script[] = "#!/bin/sh\nwhile :; do :; done\n"; + TEST_ASSERT(write(fd, script, sizeof(script) - 1) == + static_cast(sizeof(script) - 1)); + close(fd); + TEST_ASSERT(chmod(script_path.c_str(), 0700) == 0); + + BackendIpcLaunchConfig cfg; + cfg.bin = script_path; + cfg.payload_path = "/tmp/dflash_test_backend_ipc_payload"; + cfg.readiness_timeout_ms = 50; + const auto started = std::chrono::steady_clock::now(); + BackendIpcProcess proc; + TEST_ASSERT(!proc.start(cfg)); + const auto elapsed_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - started).count(); + TEST_ASSERT(elapsed_ms < 1000); + TEST_ASSERT(!proc.active()); + unlink(script_path.c_str()); +} + TEST_CASE(ServerUnitFixture, test_backend_ipc_payload_pipe_round_trip) { int payload_pipe[2] = {-1, -1}; int status_pipe[2] = {-1, -1}; @@ -4913,7 +4916,8 @@ TEST_CASE(ServerUnitFixture, test_props_tool_speculation_shape) { TEST_ASSERT(disabled["executor_contract"].is_null()); TEST_ASSERT(disabled["protocol"].get() == "dflash.tool-speculation.v1"); - TEST_ASSERT(disabled["requires_client_support"].get()); + TEST_ASSERT(!disabled["client_prediction_required"].get()); + TEST_ASSERT(!disabled["client_result_handling_required"].get()); TEST_ASSERT(disabled["preserves_token_speculation"].get()); TEST_ASSERT(disabled["unqualified_lane_policy"].get() == "defer"); @@ -4953,6 +4957,8 @@ TEST_CASE(ServerUnitFixture, test_props_tool_speculation_shape) { const json & enabled = body["tool_speculation"]; TEST_ASSERT(enabled["enabled"].get()); TEST_ASSERT(!enabled["automatic_prediction_enabled"].get()); + TEST_ASSERT(enabled["client_prediction_required"].get()); + TEST_ASSERT(enabled["client_result_handling_required"].get()); TEST_ASSERT(!enabled["predictor_decode_isolated"].get()); TEST_ASSERT(enabled["profile_status"].get() == "qualified"); TEST_ASSERT(enabled["executor_contract"].get() == @@ -4978,7 +4984,8 @@ TEST_CASE(ServerUnitFixture, test_props_tool_speculation_shape) { body = build_props_body(cfg, pc, tm); const json & automatic = body["tool_speculation"]; TEST_ASSERT(automatic["automatic_prediction_enabled"].get()); - TEST_ASSERT(!automatic["requires_client_support"].get()); + TEST_ASSERT(!automatic["client_prediction_required"].get()); + TEST_ASSERT(automatic["client_result_handling_required"].get()); TEST_ASSERT(automatic["prediction_source"].get() == "native-qwen3"); TEST_ASSERT(std::fabs( @@ -4987,16 +4994,6 @@ TEST_CASE(ServerUnitFixture, test_props_tool_speculation_shape) { "before-model"); TEST_ASSERT(automatic["predictor_decode_isolated"].get()); - cfg.semantic_tool_predictor.native_schedule = - NativeToolPredictorSchedule::Overlap; - body = build_props_body(cfg, pc, tm); - const json & overlapping = body["tool_speculation"]; - TEST_ASSERT(overlapping["predictor_schedule"].get() == - "overlap"); - TEST_ASSERT(!overlapping["predictor_decode_isolated"].get()); - cfg.semantic_tool_predictor.native_schedule = - NativeToolPredictorSchedule::BeforeModel; - cfg.semantic_tool_predictor.native_model_path.clear(); cfg.semantic_tool_predictor.native_ipc_bin.clear(); cfg.semantic_tool_predictor.url = "http://127.0.0.1:9000/v1/chat/completions"; @@ -5006,7 +5003,9 @@ TEST_CASE(ServerUnitFixture, test_props_tool_speculation_shape) { TEST_ASSERT(remote["automatic_prediction_enabled"].get()); TEST_ASSERT(remote["prediction_source"].get() == "remote-predictor"); - TEST_ASSERT(!remote["predictor_decode_isolated"].get()); + TEST_ASSERT(remote["predictor_schedule"].get() == + "before-model"); + TEST_ASSERT(remote["predictor_decode_isolated"].get()); cfg.semantic_tool_predictor.url.clear(); cfg.semantic_tool_predictor.model.clear(); cfg.semantic_tool_predictor.native_model_path = "/models/qwen3-0.6b.gguf"; diff --git a/server/test/test_tool_speculation.cpp b/server/test/test_tool_speculation.cpp index b2acb3148..118522fb9 100644 --- a/server/test/test_tool_speculation.cpp +++ b/server/test/test_tool_speculation.cpp @@ -534,15 +534,23 @@ TEST_CASE(ToolSpeculationFixture, executor_does_not_inherit_server_fds) { #endif } -TEST_CASE(ToolSpeculationFixture, executor_environment_overrides_stale_enable_flag) { +TEST_CASE(ToolSpeculationFixture, executor_environment_is_minimal) { const char * previous = std::getenv("DFLASH_TOOL_SPECULATION"); const bool had_previous = previous != nullptr; const std::string previous_value = previous ? previous : ""; + const char * previous_secret = std::getenv("DFLASH_TEST_SERVER_SECRET"); + const bool had_previous_secret = previous_secret != nullptr; + const std::string previous_secret_value = previous_secret + ? previous_secret : ""; CHECK(::setenv("DFLASH_TOOL_SPECULATION", "0", 1) == 0); + CHECK(::setenv("DFLASH_TEST_SERVER_SECRET", "must-not-leak", 1) == 0); const std::string path = make_executor_script( "IFS= read -r control\n" - "printf '{\"ok\":true,\"result\":{\"enabled\":\"%s\"}}\\n' " - "\"$DFLASH_TOOL_SPECULATION\"\n"); + "secret=false\n" + "if [ \"${DFLASH_TEST_SERVER_SECRET+x}\" = x ]; then secret=true; fi\n" + "printf '{\"ok\":true,\"result\":{\"enabled\":\"%s\"," + "\"secret_inherited\":%s}}\\n' " + "\"$DFLASH_TOOL_SPECULATION\" \"$secret\"\n"); ToolSpeculationConfig config = test_config(path); auto attempt = ToolSpeculationAttempt::create( config, prediction(), "request_clean_environment"); @@ -557,9 +565,16 @@ TEST_CASE(ToolSpeculationFixture, executor_environment_overrides_stale_enable_fl } else { CHECK(::unsetenv("DFLASH_TOOL_SPECULATION") == 0); } + if (had_previous_secret) { + CHECK(::setenv( + "DFLASH_TEST_SERVER_SECRET", previous_secret_value.c_str(), 1) == 0); + } else { + CHECK(::unsetenv("DFLASH_TEST_SERVER_SECRET") == 0); + } CHECK(metadata["status"] == "hit"); CHECK(metadata["result"]["enabled"] == "1"); + CHECK(!metadata["result"]["secret_inherited"].get()); } TEST_CASE(ToolSpeculationFixture, qualified_lane_keeps_speculative_decode) { From 6c4dc2da4d7e154dd8b367c29f77fc20598cb2a2 Mon Sep 17 00:00:00 2001 From: mrciffa <49000955+davide221@users.noreply.github.com> Date: Tue, 18 Aug 2026 18:41:42 +0200 Subject: [PATCH 08/11] fix(server): close final tool speculation review gaps --- .../benchmark_cpu_tool_speculation.py | 42 ++- .../benchmark_trace_compiled_workflows.py | 66 ++++- .../run_native_cpu_server_lucebox5.sh | 2 +- .../test_benchmark_cpu_tool_speculation.py | 36 +++ ...test_benchmark_trace_compiled_workflows.py | 87 ++++++ server/src/common/backend_ipc.cpp | 258 +++++++++++++----- server/src/common/backend_ipc.h | 10 +- server/src/common/io_utils.h | 50 ++++ .../src/common/qwen3_tool_predictor_ipc.cpp | 123 +++------ server/src/server/http_server.cpp | 22 +- server/src/server/semantic_tool_hint.cpp | 134 ++++++++- server/src/server/semantic_tool_hint.h | 12 +- server/src/server/tokenizer.cpp | 45 ++- server/src/server/tool_speculation.h | 5 +- server/test/test_semantic_tool_hint.cpp | 39 ++- server/test/test_server_unit.cpp | 15 + 16 files changed, 738 insertions(+), 208 deletions(-) diff --git a/optimizations/ooo_spec_lucebox5_cpu/benchmark_cpu_tool_speculation.py b/optimizations/ooo_spec_lucebox5_cpu/benchmark_cpu_tool_speculation.py index 7b52bb0e7..4e62052cd 100755 --- a/optimizations/ooo_spec_lucebox5_cpu/benchmark_cpu_tool_speculation.py +++ b/optimizations/ooo_spec_lucebox5_cpu/benchmark_cpu_tool_speculation.py @@ -327,7 +327,7 @@ def pin_child() -> None: ) process.stdin.close() process.stdin = None - return {"process": process, "started": started} + return {"process": process, "pgid": process.pid, "started": started} def finish_executor(handle: dict[str, Any], timeout: float) -> dict[str, Any]: @@ -361,19 +361,43 @@ def finish_executor(handle: dict[str, Any], timeout: float) -> dict[str, Any]: def stop_executor(handle: dict[str, Any]) -> None: process: subprocess.Popen[str] = handle["process"] - if process.poll() is None: + pgid = int(handle.get("pgid", process.pid)) + + def group_exists() -> bool: try: - os.killpg(process.pid, signal.SIGTERM) + os.killpg(pgid, 0) + return True except ProcessLookupError: - pass - try: - process.wait(timeout=1.0) - except subprocess.TimeoutExpired: + return False + except PermissionError: + return True + + try: + os.killpg(pgid, signal.SIGTERM) + except ProcessLookupError: + pass + + grace_deadline = time.monotonic() + 1.0 + while group_exists() and time.monotonic() < grace_deadline: + if process.poll() is None: try: - os.killpg(process.pid, signal.SIGKILL) - except ProcessLookupError: + process.wait(timeout=0.01) + except subprocess.TimeoutExpired: pass + else: + time.sleep(0.01) + + if group_exists(): + try: + os.killpg(pgid, signal.SIGKILL) + except ProcessLookupError: + pass + + if process.poll() is None: + try: process.wait(timeout=5.0) + except subprocess.TimeoutExpired as error: + raise RuntimeError("CPU executor process group did not stop") from error def run_executor( diff --git a/optimizations/ooo_spec_lucebox5_cpu/benchmark_trace_compiled_workflows.py b/optimizations/ooo_spec_lucebox5_cpu/benchmark_trace_compiled_workflows.py index e0f5fa3a4..84837f014 100644 --- a/optimizations/ooo_spec_lucebox5_cpu/benchmark_trace_compiled_workflows.py +++ b/optimizations/ooo_spec_lucebox5_cpu/benchmark_trace_compiled_workflows.py @@ -774,7 +774,7 @@ def stage_result_message( "items": [ { **branch["root"], - "call_ref": branch["steps"][-1]["tool_result"]["call_ref"], + "final_ref": branch["steps"][-1]["tool_result"]["call_ref"], } for branch in branches ], @@ -1629,7 +1629,7 @@ def validate_args(parser: argparse.ArgumentParser, args: argparse.Namespace) -> "by the deployed trace executor" ) if ( - args.pairs <= 0 + args.pairs < 2 or args.warmup_tasks < 0 or not 2 <= args.min_branches <= args.max_branches <= 4 or args.timeout <= 0 @@ -1657,40 +1657,80 @@ def refresh_existing_report( if not isinstance(report, dict): raise ValueError("existing report is not a JSON object") recorded_pattern = report.get("pattern") + production_gate = report.get("production_gate") + recorded_pairs = report.get("pairs") if ( report.get("schema_version") != 1 or not isinstance(recorded_pattern, dict) + or not isinstance(production_gate, dict) + or not isinstance(recorded_pairs, list) or recorded_pattern.get("fingerprint") != pattern.fingerprint or recorded_pattern.get("training_report_sha256") != file_sha256(args.training_report) or recorded_pattern.get("workflow_registry_sha256") != file_sha256(args.workflow_registry) - or not report.get("production_gate", {}).get("passed") - or len(report.get("pairs", [])) != args.pairs + or not production_gate.get("passed") + or len(recorded_pairs) != args.pairs ): raise ValueError("existing report does not match this qualified run") - privacy_miss = measure_private_miss( - args, measured_tasks[0], measured_tasks[1], pattern - ) summary = report.get("summary") if not isinstance(summary, dict) or summary.get("tasks") != args.pairs: raise ValueError("existing report summary does not match --pairs") + required_refresh_fields = { + "stage_batched_to_speculative_speedup_p50", + "stage_batched_to_speculative_bootstrap_95ci", + "stage_batched_to_speculative_speedup_p05", + "compiled_to_speculative_speedup_p50", + "compiled_to_speculative_bootstrap_95ci", + "pattern_prediction_hit_rate", + "all_predictions_from_qwen", + "all_interference_probes_qualified", + "model_compute_slowdown_p50_percent", + "model_compute_slowdown_p95_percent", + "decode_slowdown_p50_percent", + "decode_slowdown_p95_percent", + "prefix_cache_configured", + "all_calls_stable", + "all_tool_results_stable", + "macro_output_stability_rate", + "all_final_answers_correct", + "all_final_outputs_stable", + "all_macro_calls_correct", + "all_ds4_active", + } + missing_refresh_fields = sorted(required_refresh_fields - summary.keys()) + if missing_refresh_fields: + raise ValueError( + "existing report predates interference probes; rerun the full " + "benchmark without --refresh-report (missing summary fields: " + + ", ".join(missing_refresh_fields) + + ")" + ) + methodology = report.get("methodology") + if not isinstance(methodology, dict): + raise ValueError("existing report has no methodology object") + server_snapshot = report.get("server_snapshot") + if not isinstance(server_snapshot, dict): + raise ValueError("existing report has no server_snapshot object") + privacy_miss = measure_private_miss( + args, measured_tasks[0], measured_tasks[1], pattern + ) summary["private_miss_result_hidden"] = privacy_miss["passed"] checks = production_checks(summary, args) report["privacy_miss"] = privacy_miss - report["methodology"]["additive_gate_refresh"] = ( + methodology["additive_gate_refresh"] = ( "the wrong-call privacy probe and server snapshot were refreshed after " "the timing arms; no recorded timing was recomputed" ) - report["production_gate"]["checks"] = checks - report["production_gate"]["passed"] = all(checks.values()) + production_gate["checks"] = checks + production_gate["passed"] = all(checks.values()) ending_props = get_json(props_url(args.url), args.timeout) - report["server_snapshot"]["prefix_cache_before"] = prefix_cache - report["server_snapshot"]["prefix_cache_after"] = ending_props.get( + server_snapshot["prefix_cache_before"] = prefix_cache + server_snapshot["prefix_cache_after"] = ending_props.get( "prefix_cache" ) - report["server_snapshot"]["tool_speculation"] = tool_speculation + server_snapshot["tool_speculation"] = tool_speculation args.output.write_text( json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8" ) diff --git a/optimizations/ooo_spec_lucebox5_cpu/run_native_cpu_server_lucebox5.sh b/optimizations/ooo_spec_lucebox5_cpu/run_native_cpu_server_lucebox5.sh index 4d876782f..a99698ad9 100755 --- a/optimizations/ooo_spec_lucebox5_cpu/run_native_cpu_server_lucebox5.sh +++ b/optimizations/ooo_spec_lucebox5_cpu/run_native_cpu_server_lucebox5.sh @@ -51,7 +51,7 @@ if fuser -s /dev/kfd 2>/dev/null; then exit 75 fi -exec env \ +exec env -i \ HOME="$root" \ USER="lucebox5" \ PATH="$root/.local/bin:/opt/rocm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \ diff --git a/optimizations/ooo_spec_lucebox5_cpu/test_benchmark_cpu_tool_speculation.py b/optimizations/ooo_spec_lucebox5_cpu/test_benchmark_cpu_tool_speculation.py index 3506248d0..9d497e8ab 100644 --- a/optimizations/ooo_spec_lucebox5_cpu/test_benchmark_cpu_tool_speculation.py +++ b/optimizations/ooo_spec_lucebox5_cpu/test_benchmark_cpu_tool_speculation.py @@ -1,7 +1,10 @@ from __future__ import annotations import argparse +import signal +import subprocess import unittest +from unittest.mock import Mock, patch from benchmark_cpu_tool_speculation import ( TOOL_NAME, @@ -12,10 +15,43 @@ props_url, require_qualified_cpu_tool_props, request_body, + stop_executor, ) class CpuToolSpeculationBenchmarkTest(unittest.TestCase): + def test_stop_executor_signals_group_after_leader_exit(self) -> None: + process = Mock(pid=4321) + process.poll.return_value = 0 + + def kill_group(_pgid: int, sent_signal: int) -> None: + if sent_signal == 0: + raise ProcessLookupError + + with patch( + "benchmark_cpu_tool_speculation.os.killpg", + side_effect=kill_group, + ) as killpg: + stop_executor({"process": process, "pgid": 4321}) + + killpg.assert_any_call(4321, signal.SIGTERM) + + def test_stop_executor_converts_final_wait_timeout(self) -> None: + process = Mock(pid=4321) + process.poll.return_value = None + process.wait.side_effect = subprocess.TimeoutExpired("executor", 5.0) + + def kill_group(_pgid: int, sent_signal: int) -> None: + if sent_signal == 0: + raise ProcessLookupError + + with patch( + "benchmark_cpu_tool_speculation.os.killpg", + side_effect=kill_group, + ): + with self.assertRaisesRegex(RuntimeError, "did not stop"): + stop_executor({"process": process, "pgid": 4321}) + def test_cpu_list_parser_canonicalizes_ranges(self) -> None: self.assertEqual(parse_cpu_list("30-31,15,14-15"), [14, 15, 30, 31]) with self.assertRaises(argparse.ArgumentTypeError): diff --git a/optimizations/ooo_spec_lucebox5_cpu/test_benchmark_trace_compiled_workflows.py b/optimizations/ooo_spec_lucebox5_cpu/test_benchmark_trace_compiled_workflows.py index 7f635d20a..7c7d5955a 100644 --- a/optimizations/ooo_spec_lucebox5_cpu/test_benchmark_trace_compiled_workflows.py +++ b/optimizations/ooo_spec_lucebox5_cpu/test_benchmark_trace_compiled_workflows.py @@ -2,15 +2,19 @@ import argparse import json +import sys import tempfile import unittest from pathlib import Path from unittest.mock import patch from benchmark_trace_compiled_workflows import ( + CANONICAL_TRAINING_REPORT, + CANONICAL_WORKFLOW_REGISTRY, alphabetic_identifier, compact_arm, final_answer_correct, + file_sha256, interference_probe_qualified, load_training_traces, load_partial_pairs, @@ -21,9 +25,12 @@ parse_request_customers, post_final, production_checks, + refresh_existing_report, simulated_tool_result, stage_batch_tool, stage_batched_messages, + stage_result_message, + validate_args, stage_reference, workflow_reference, ) @@ -218,6 +225,53 @@ def test_bound_stage_uses_the_request_scoped_reference(self) -> None: self.assertEqual(parameters["properties"]["stage_ref"]["enum"], [stage_ref]) self.assertEqual(stage_ref, "workflow_taskc_stage_three") + def test_stage_result_exposes_the_requested_final_ref(self) -> None: + message = stage_result_message( + self.pattern, + len(self.pattern.steps) - 1, + [ + { + "root": { + "customer_email": "a@example.test", + "destination": "Rome", + }, + "steps": [{"tool_result": {"call_ref": "plum"}}], + } + ], + "call_stage", + ) + content = json.loads(message["content"]) + self.assertEqual(content["items"][0]["final_ref"], "plum") + self.assertNotIn("call_ref", content["items"][0]) + + def test_validation_requires_two_pairs_for_privacy_probe(self) -> None: + class RaisingParser: + @staticmethod + def error(message: str) -> None: + raise ValueError(message) + + args = argparse.Namespace( + binary=Path(sys.executable), + training_report=CANONICAL_TRAINING_REPORT.resolve(), + workflow_registry=CANONICAL_WORKFLOW_REGISTRY.resolve(), + pairs=1, + warmup_tasks=0, + min_branches=2, + max_branches=4, + timeout=1.0, + call_max_tokens=1, + macro_max_tokens=1, + final_max_tokens=1, + bootstrap_resamples=1, + interference_repetitions=3, + min_production_pairs=2, + min_e2e_speedup=2.0, + min_e2e_speedup_p05=1.5, + min_incremental_speedup=1.05, + ) + with self.assertRaisesRegex(ValueError, "counts and thresholds"): + validate_args(RaisingParser(), args) + def test_parses_multiple_native_tool_calls(self) -> None: response = { "choices": [ @@ -457,6 +511,39 @@ def test_resume_checkpoint_requires_matching_task_and_arm_order(self) -> None: with self.assertRaisesRegex(ValueError, "does not match"): load_partial_pairs(path, tasks, orders) + def test_refresh_rejects_reports_that_predate_interference_probes(self) -> None: + tasks = [make_task(0, 2, self.pattern), make_task(1, 3, self.pattern)] + report = { + "schema_version": 1, + "pattern": { + "fingerprint": self.pattern.fingerprint, + "training_report_sha256": file_sha256( + CANONICAL_TRAINING_REPORT + ), + "workflow_registry_sha256": file_sha256( + CANONICAL_WORKFLOW_REGISTRY + ), + }, + "production_gate": {"passed": True}, + "pairs": [{}, {}], + "summary": {"tasks": 2}, + "methodology": {}, + "server_snapshot": {}, + } + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "report.json" + output.write_text(json.dumps(report), encoding="utf-8") + args = argparse.Namespace( + output=output, + pairs=2, + training_report=CANONICAL_TRAINING_REPORT, + workflow_registry=CANONICAL_WORKFLOW_REGISTRY, + ) + with self.assertRaisesRegex(ValueError, "predates interference"): + refresh_existing_report( + args, self.pattern, tasks, {}, {} + ) + if __name__ == "__main__": unittest.main() diff --git a/server/src/common/backend_ipc.cpp b/server/src/common/backend_ipc.cpp index d6ee776f3..ca765a21b 100644 --- a/server/src/common/backend_ipc.cpp +++ b/server/src/common/backend_ipc.cpp @@ -3,6 +3,7 @@ #include "backend_ipc.h" #include "io_utils.h" +#include #include #include #include @@ -19,9 +20,9 @@ #if !defined(_WIN32) # include # include -# include # include # include +# include # include # include # if defined(__linux__) @@ -35,77 +36,70 @@ namespace dflash::common { namespace { #if !defined(_WIN32) -bool read_exact_with_timeout(int fd, void * data, size_t bytes, - int timeout_ms, bool & timed_out) { - timed_out = false; - auto * cursor = static_cast(data); - size_t received = 0; - const auto deadline = std::chrono::steady_clock::now() + - std::chrono::milliseconds(timeout_ms); - while (received < bytes) { - const auto now = std::chrono::steady_clock::now(); - if (now >= deadline) { - timed_out = true; - return false; - } - const int64_t remaining = - std::chrono::duration_cast( - deadline - now).count(); - pollfd descriptor{fd, POLLIN | POLLHUP, 0}; - const int wait_ms = static_cast((std::min)( - int64_t{INT_MAX}, (std::max)(int64_t{1}, remaining))); - const int polled = ::poll(&descriptor, 1, wait_ms); - if (polled == 0) { - timed_out = true; - return false; - } - if (polled < 0) { - if (errno == EINTR) continue; - return false; - } - if (descriptor.revents & (POLLERR | POLLNVAL)) return false; - const ssize_t count = ::read(fd, cursor + received, bytes - received); - if (count == 0) return false; - if (count < 0) { - if (errno == EINTR) continue; - return false; - } - received += static_cast(count); - } - return true; +unsigned int descriptor_scan_limit() { + struct rlimit limit {}; + if (::getrlimit(RLIMIT_NOFILE, &limit) == 0 && + limit.rlim_cur != RLIM_INFINITY && limit.rlim_cur > 0) { + return static_cast((std::min)( + static_cast(limit.rlim_cur), + static_cast(INT_MAX))); + } + const long open_max = ::sysconf(_SC_OPEN_MAX); + if (open_max <= 0) return 0; + return static_cast((std::min)( + static_cast(open_max), + static_cast(INT_MAX))); } -bool close_descriptor_range(unsigned int first, unsigned int last) { +bool close_descriptor_range(unsigned int first, + unsigned int last, + unsigned int scan_limit) { if (first > last) return true; #if defined(__linux__) && defined(SYS_close_range) int rc = -1; do { rc = static_cast(::syscall(SYS_close_range, first, last, 0)); } while (rc != 0 && errno == EINTR); - return rc == 0; -#else - (void)first; - (void)last; - errno = ENOSYS; - return false; + if (rc == 0) return true; #endif + + // close_range is Linux-specific and may also be blocked by an older + // kernel or seccomp policy. Fall back to the POSIX descriptor space that + // was captured before fork, and verify every ambiguous close failure. + if (scan_limit == 0 || first >= scan_limit) return scan_limit != 0; + const unsigned int upper = (std::min)(last, scan_limit - 1U); + for (unsigned int fd = first; fd <= upper; ++fd) { + if (::close(static_cast(fd)) == 0 || errno == EBADF) continue; + const int close_error = errno; + errno = 0; + if (::fcntl(static_cast(fd), F_GETFD) < 0 && errno == EBADF) { + continue; + } + errno = close_error; + return false; + } + return true; } -bool isolate_child_descriptors(int payload_fd, int stream_fd, int shared_fd) { +bool isolate_child_descriptors(int payload_fd, + int stream_fd, + int shared_fd, + unsigned int scan_limit) { std::array keep{payload_fd, stream_fd, shared_fd}; std::sort(keep.begin(), keep.end()); unsigned int first = STDERR_FILENO + 1; int previous = -1; for (const int fd : keep) { if (fd < static_cast(first) || fd == previous) continue; - if (!close_descriptor_range(first, static_cast(fd - 1))) { + if (!close_descriptor_range( + first, static_cast(fd - 1), scan_limit)) { return false; } first = static_cast(fd) + 1U; previous = fd; } return close_descriptor_range( - first, (std::numeric_limits::max)()); + first, (std::numeric_limits::max)(), scan_limit); } #endif @@ -197,6 +191,14 @@ bool BackendIpcProcess::start(const BackendIpcLaunchConfig & cfg) { close(); if (cfg.bin.empty() || cfg.payload_path.empty()) return false; if (!init_work_dir(cfg.work_dir, cfg.require_private_work_dir)) return false; + const unsigned int descriptor_limit = cfg.isolate_inherited_fds + ? descriptor_scan_limit() : 0; + if (cfg.isolate_inherited_fds && descriptor_limit == 0) { + std::fprintf(stderr, + "backend-ipc cannot determine descriptor scan limit\n"); + close(); + return false; + } int cmd_pipe[2] = {-1, -1}; int payload_pipe[2] = {-1, -1}; @@ -209,6 +211,7 @@ bool BackendIpcProcess::start(const BackendIpcLaunchConfig & cfg) { if (payload_pipe[1] >= 0) ::close(payload_pipe[1]); if (stream_pipe[0] >= 0) ::close(stream_pipe[0]); if (stream_pipe[1] >= 0) ::close(stream_pipe[1]); + close(); return false; } const bool shared_required = @@ -221,6 +224,7 @@ bool BackendIpcProcess::start(const BackendIpcLaunchConfig & cfg) { ::close(cmd_pipe[0]); ::close(cmd_pipe[1]); ::close(payload_pipe[0]); ::close(payload_pipe[1]); ::close(stream_pipe[0]); ::close(stream_pipe[1]); + close(); return false; } if (shared_requested && cfg.shared_payload_bytes > 0) { @@ -251,6 +255,13 @@ bool BackendIpcProcess::start(const BackendIpcLaunchConfig & cfg) { return false; } if (pid_ == 0) { + if (cfg.require_private_work_dir && + (work_dir_fd_ < 0 || ::fchdir(work_dir_fd_) != 0)) { + std::fprintf(stderr, + "backend-ipc private work_dir chdir failed: %s\n", + std::strerror(errno)); + _exit(127); + } if (cmd_pipe[0] != STDIN_FILENO && ::dup2(cmd_pipe[0], STDIN_FILENO) < 0) { std::fprintf(stderr, "backend-ipc dup2 failed: %s\n", std::strerror(errno)); _exit(127); @@ -284,7 +295,8 @@ bool BackendIpcProcess::start(const BackendIpcLaunchConfig & cfg) { argv.push_back(nullptr); if (cfg.isolate_inherited_fds && !isolate_child_descriptors( - payload_pipe[0], stream_pipe[1], shared_payload_fd_)) { + payload_pipe[0], stream_pipe[1], shared_payload_fd_, + descriptor_limit)) { std::fprintf(stderr, "backend-ipc descriptor isolation failed: %s\n", std::strerror(errno)); @@ -310,10 +322,11 @@ bool BackendIpcProcess::start(const BackendIpcLaunchConfig & cfg) { } int32_t status = -1; bool readiness_timed_out = false; + const auto readiness_deadline = std::chrono::steady_clock::now() + + std::chrono::milliseconds((std::max)(0, cfg.readiness_timeout_ms)); const bool status_read = cfg.readiness_timeout_ms > 0 - ? read_exact_with_timeout(stream_fd_, &status, sizeof(status), - cfg.readiness_timeout_ms, - readiness_timed_out) + ? read_exact_fd_until(stream_fd_, &status, sizeof(status), + readiness_deadline, readiness_timed_out) : read_exact_fd(stream_fd_, &status, sizeof(status)); if (!status_read || status != 0) { int child_status = 0; @@ -415,8 +428,19 @@ void BackendIpcProcess::close_impl(bool force_terminate) { } pid_ = -1; } - if (owns_work_dir_ && !work_dir_.empty()) { - ::rmdir(work_dir_.c_str()); + bool remove_owned_work_dir = owns_work_dir_ && !work_dir_.empty(); + if (work_dir_fd_ >= 0) { + struct stat opened {}; + struct stat named {}; + remove_owned_work_dir = remove_owned_work_dir && + ::fstat(work_dir_fd_, &opened) == 0 && + ::lstat(work_dir_.c_str(), &named) == 0 && + opened.st_dev == named.st_dev && opened.st_ino == named.st_ino; + ::close(work_dir_fd_); + work_dir_fd_ = -1; + } + if (remove_owned_work_dir) { + (void)::rmdir(work_dir_.c_str()); } #else (void)force_terminate; @@ -435,6 +459,62 @@ std::string BackendIpcProcess::next_path(const char * prefix) { return work_dir_ + "/" + prefix + "_" + std::to_string(seq_++) + ".bin"; } +bool BackendIpcProcess::write_private_file( + const char * prefix, + const void * data, + size_t bytes, + std::string & name) { + name.clear(); +#if defined(_WIN32) + (void)prefix; (void)data; (void)bytes; + return false; +#else + if (work_dir_fd_ < 0 || !prefix || !*prefix || + (bytes > 0 && !data)) { + return false; + } + for (const char * cursor = prefix; *cursor; ++cursor) { + const unsigned char character = static_cast(*cursor); + if (!std::isalnum(character) && character != '_' && character != '-') { + return false; + } + } + for (int attempt = 0; attempt < 16; ++attempt) { + name = std::string(prefix) + "_" + std::to_string(::getpid()) + "_" + + std::to_string(seq_++) + ".bin"; + const int fd = ::openat( + work_dir_fd_, name.c_str(), + O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW | O_CLOEXEC, 0600); + if (fd < 0) { + if (errno == EEXIST) continue; + name.clear(); + return false; + } + const bool written = write_exact_fd(fd, data, bytes); + const bool closed = ::close(fd) == 0; + if (written && closed) return true; + (void)::unlinkat(work_dir_fd_, name.c_str(), 0); + name.clear(); + return false; + } + name.clear(); + return false; +#endif +} + +bool BackendIpcProcess::remove_private_file(const std::string & name) { +#if defined(_WIN32) + (void)name; + return false; +#else + if (work_dir_fd_ < 0 || name.empty() || + name.find('/') != std::string::npos) { + return false; + } + return ::unlinkat(work_dir_fd_, name.c_str(), 0) == 0 || errno == ENOENT; +#endif +} + bool BackendIpcProcess::write_shared_payload(const void * data, size_t bytes, uint64_t & seq) { BackendIpcPayloadSegment segment{data, bytes}; return write_shared_payload_segments(&segment, 1, seq); @@ -538,6 +618,61 @@ bool BackendIpcProcess::init_shared_payload(size_t bytes) { bool BackendIpcProcess::init_work_dir(const std::string & requested, bool require_private) { + if (require_private) { + std::string parent = "/tmp"; + if (!requested.empty()) { + parent = requested; + if (::mkdir(parent.c_str(), 0700) != 0 && errno != EEXIST) { + std::fprintf(stderr, "backend-ipc mkdir failed: %s: %s\n", + parent.c_str(), std::strerror(errno)); + return false; + } + struct stat parent_stat {}; + if (::lstat(parent.c_str(), &parent_stat) != 0 || + !S_ISDIR(parent_stat.st_mode) || + parent_stat.st_uid != ::geteuid() || + (parent_stat.st_mode & 0777) != 0700) { + std::fprintf(stderr, + "backend-ipc private work base must be an owned, " + "non-symlink mode-0700 directory: %s\n", + parent.c_str()); + return false; + } + } + + std::string templ = parent + "/backend-ipc-private-XXXXXX"; + std::vector buf(templ.begin(), templ.end()); + buf.push_back('\0'); + char * dir = ::mkdtemp(buf.data()); + if (!dir) { + std::fprintf(stderr, + "backend-ipc private mkdtemp failed: %s\n", + std::strerror(errno)); + return false; + } + work_dir_ = dir; + work_dir_fd_ = ::open( + work_dir_.c_str(), + O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + struct stat opened {}; + if (work_dir_fd_ < 0 || ::fstat(work_dir_fd_, &opened) != 0 || + !S_ISDIR(opened.st_mode) || opened.st_uid != ::geteuid() || + (opened.st_mode & 0777) != 0700) { + std::fprintf(stderr, + "backend-ipc cannot retain private work directory: %s\n", + std::strerror(errno)); + if (work_dir_fd_ >= 0) { + ::close(work_dir_fd_); + work_dir_fd_ = -1; + } + (void)::rmdir(work_dir_.c_str()); + work_dir_.clear(); + return false; + } + owns_work_dir_ = true; + return true; + } + if (!requested.empty()) { work_dir_ = requested; owns_work_dir_ = false; @@ -549,23 +684,12 @@ bool BackendIpcProcess::init_work_dir(const std::string & requested, } } struct stat st; - const int stat_result = require_private - ? ::lstat(work_dir_.c_str(), &st) - : ::stat(work_dir_.c_str(), &st); - if (stat_result != 0 || - !S_ISDIR(st.st_mode)) { + if (::stat(work_dir_.c_str(), &st) != 0 || !S_ISDIR(st.st_mode)) { std::fprintf(stderr, "backend-ipc work_dir is not a directory: %s\n", work_dir_.c_str()); return false; } - if (require_private && - (st.st_uid != ::geteuid() || (st.st_mode & 0777) != 0700)) { - std::fprintf(stderr, - "backend-ipc work_dir must be owned by the server and mode 0700: %s\n", - work_dir_.c_str()); - return false; - } return true; } const char * tmp = std::getenv("TMPDIR"); diff --git a/server/src/common/backend_ipc.h b/server/src/common/backend_ipc.h index 3c733d396..50ddfdc78 100644 --- a/server/src/common/backend_ipc.h +++ b/server/src/common/backend_ipc.h @@ -131,7 +131,7 @@ struct BackendIpcLaunchConfig { // explicitly configured payload/response descriptors. bool isolate_inherited_fds = false; // Keep legacy backend work directories compatible while allowing private - // sidecars to require an owned, non-symlink 0700 directory. + // sidecars to require a fresh 0700 child of an owned, non-symlink base. bool require_private_work_dir = false; }; @@ -165,6 +165,13 @@ class BackendIpcProcess { const std::string & work_dir() const { return work_dir_; } std::string next_path(const char * prefix); + // Create/remove a regular file relative to the retained private directory + // descriptor. The returned name is relative to the daemon's private cwd. + bool write_private_file(const char * prefix, + const void * data, + size_t bytes, + std::string & name); + bool remove_private_file(const std::string & name); bool write_shared_payload(const void * data, size_t bytes, uint64_t & seq); bool write_shared_payload_segments(const BackendIpcPayloadSegment * segments, size_t n_segments, @@ -178,6 +185,7 @@ class BackendIpcProcess { bool init_shared_payload(size_t bytes); pid_t pid_ = -1; + int work_dir_fd_ = -1; #endif FILE * cmd_ = nullptr; int payload_fd_ = -1; diff --git a/server/src/common/io_utils.h b/server/src/common/io_utils.h index a4818b4f5..f66b69ed4 100644 --- a/server/src/common/io_utils.h +++ b/server/src/common/io_utils.h @@ -3,6 +3,9 @@ #pragma once +#include +#include +#include #include #include #include @@ -18,6 +21,7 @@ # include #else # include +# include # include #endif @@ -110,6 +114,52 @@ static inline bool read_exact_fd(int fd, void * data, size_t bytes) { return true; } +// Read a complete protocol field under one wall-clock deadline. The helper is +// shared by startup handshakes and request/response IPC so partial reads and +// timeout behavior cannot drift between sidecars. +static inline bool read_exact_fd_until( + int fd, + void * data, + size_t bytes, + const std::chrono::steady_clock::time_point & deadline, + bool & timed_out) { + char * cursor = static_cast(data); + size_t received = 0; + timed_out = false; + while (received < bytes) { + const auto now = std::chrono::steady_clock::now(); + if (now >= deadline) { + timed_out = true; + return false; + } + const int64_t remaining = + std::chrono::duration_cast( + deadline - now).count(); + pollfd descriptor{fd, POLLIN | POLLHUP, 0}; + const int wait_ms = static_cast((std::min)( + int64_t{INT_MAX}, (std::max)(int64_t{1}, remaining))); + const int polled = ::poll(&descriptor, 1, wait_ms); + if (polled == 0) { + timed_out = true; + return false; + } + if (polled < 0) { + if (errno == EINTR) continue; + return false; + } + if (descriptor.revents & (POLLERR | POLLNVAL)) return false; + const ssize_t count = ::read( + fd, cursor + received, bytes - received); + if (count == 0) return false; + if (count < 0) { + if (errno == EINTR) continue; + return false; + } + received += static_cast(count); + } + return true; +} + static inline bool write_exact_fd(int fd, const void * data, size_t bytes) { const char * p = (const char *)data; size_t done = 0; diff --git a/server/src/common/qwen3_tool_predictor_ipc.cpp b/server/src/common/qwen3_tool_predictor_ipc.cpp index 0a17d5336..9040fea1b 100644 --- a/server/src/common/qwen3_tool_predictor_ipc.cpp +++ b/server/src/common/qwen3_tool_predictor_ipc.cpp @@ -3,87 +3,12 @@ #include "io_utils.h" #include -#include #include #include - -#if !defined(_WIN32) -# include -# include -# include -#endif +#include +#include namespace dflash::common { -namespace { - -#if !defined(_WIN32) -bool write_private_prompt_file( - const std::string & work_dir, - const std::vector & prompt_ids, - std::string & path) { - std::string pattern = work_dir + "/tool_predictor_prompt_XXXXXX"; - std::vector buffer(pattern.begin(), pattern.end()); - buffer.push_back('\0'); - const int fd = ::mkstemp(buffer.data()); - if (fd < 0) return false; - path = buffer.data(); - const bool written = ::fchmod(fd, S_IRUSR | S_IWUSR) == 0 && - write_exact_fd( - fd, prompt_ids.data(), prompt_ids.size() * sizeof(int32_t)); - const bool closed = ::close(fd) == 0; - if (!written || !closed) { - ::unlink(path.c_str()); - path.clear(); - return false; - } - return true; -} - -bool read_exact_until( - int fd, - void * data, - size_t bytes, - const std::chrono::steady_clock::time_point & deadline, - bool & timed_out) { - auto * cursor = static_cast(data); - size_t received = 0; - timed_out = false; - while (received < bytes) { - const auto now = std::chrono::steady_clock::now(); - if (now >= deadline) { - timed_out = true; - return false; - } - const auto remaining = - std::chrono::duration_cast( - deadline - now).count(); - pollfd descriptor{fd, POLLIN | POLLHUP, 0}; - const int polled = ::poll( - &descriptor, 1, - static_cast((std::max)(int64_t{1}, remaining))); - if (polled == 0) { - timed_out = true; - return false; - } - if (polled < 0) { - if (errno == EINTR) continue; - return false; - } - if (descriptor.revents & (POLLERR | POLLNVAL)) return false; - const ssize_t count = ::read( - fd, cursor + received, bytes - received); - if (count == 0) return false; - if (count < 0) { - if (errno == EINTR) continue; - return false; - } - received += static_cast(count); - } - return true; -} -#endif - -} // namespace bool read_qwen3_tool_predictor_response( int stream_fd, @@ -108,7 +33,7 @@ bool read_qwen3_tool_predictor_response( std::chrono::milliseconds(timeout_ms); bool timed_out = false; int32_t status = -1; - if (!read_exact_until( + if (!read_exact_fd_until( stream_fd, &status, sizeof(status), deadline, timed_out)) { error = timed_out ? "native_predictor_timeout" @@ -121,7 +46,7 @@ bool read_qwen3_tool_predictor_response( } int32_t count = -1; - if (!read_exact_until( + if (!read_exact_fd_until( stream_fd, &count, sizeof(count), deadline, timed_out) || count <= 0 || count > max_tokens) { error = timed_out @@ -130,7 +55,7 @@ bool read_qwen3_tool_predictor_response( return false; } output_ids.assign(static_cast(count), 0); - if (!read_exact_until( + if (!read_exact_fd_until( stream_fd, output_ids.data(), output_ids.size() * sizeof(int32_t), deadline, timed_out)) { output_ids.clear(); @@ -162,10 +87,27 @@ bool Qwen3ToolPredictorIpcClient::start( if (bin.empty() || model_path.empty() || max_ctx <= 0 || readiness_timeout_ms <= 0) return false; + std::error_code path_error; + const std::string resolved_bin = + std::filesystem::canonical(bin, path_error).string(); + if (path_error) { + std::fprintf(stderr, + "[tool-predictor-ipc] cannot resolve IPC binary: %s\n", + path_error.message().c_str()); + return false; + } + const std::string resolved_model = + std::filesystem::canonical(model_path, path_error).string(); + if (path_error) { + std::fprintf(stderr, + "[tool-predictor-ipc] cannot resolve model: %s\n", + path_error.message().c_str()); + return false; + } BackendIpcLaunchConfig launch; - launch.bin = bin; + launch.bin = resolved_bin; launch.mode = BackendIpcMode::Qwen3ToolPredict; - launch.payload_path = model_path; + launch.payload_path = resolved_model; launch.work_dir = work_dir; launch.args.push_back("--target-gpu=" + std::to_string(std::max(0, gpu))); launch.args.push_back("--max-ctx=" + std::to_string(max_ctx)); @@ -217,23 +159,24 @@ bool Qwen3ToolPredictorIpcClient::predict( return false; } - std::string path; - if (!write_private_prompt_file( - process_.work_dir(), prompt_ids, path)) { + std::string prompt_name; + if (!process_.write_private_file( + "tool_predictor_prompt", prompt_ids.data(), + prompt_ids.size() * sizeof(int32_t), prompt_name)) { error = "native_predictor_prompt_write_failed"; return false; } if (std::chrono::steady_clock::now() >= deadline) { - std::remove(path.c_str()); + (void)process_.remove_private_file(prompt_name); error = "native_predictor_timeout"; return false; } if (std::fprintf( - command, "predict %d %s\n", max_tokens, path.c_str()) < 0 || + command, "predict %d %s\n", max_tokens, prompt_name.c_str()) < 0 || std::fflush(command) != 0) { - std::remove(path.c_str()); + (void)process_.remove_private_file(prompt_name); error = "native_predictor_command_write_failed"; process_.terminate(); active_ = false; @@ -241,7 +184,7 @@ bool Qwen3ToolPredictorIpcClient::predict( } const auto response_started = std::chrono::steady_clock::now(); if (response_started >= deadline) { - std::remove(path.c_str()); + (void)process_.remove_private_file(prompt_name); error = "native_predictor_timeout"; process_.terminate(); active_ = false; @@ -252,7 +195,7 @@ bool Qwen3ToolPredictorIpcClient::predict( deadline - response_started).count())); const bool ok = read_qwen3_tool_predictor_response( stream_fd, max_tokens, response_timeout_ms, output_ids, error); - std::remove(path.c_str()); + (void)process_.remove_private_file(prompt_name); if (!ok) { output_ids.clear(); process_.terminate(); diff --git a/server/src/server/http_server.cpp b/server/src/server/http_server.cpp index 1adc717b7..e91b6c63b 100644 --- a/server/src/server/http_server.cpp +++ b/server/src/server/http_server.cpp @@ -2490,29 +2490,25 @@ void HttpServer::start_automatic_tool_speculation(ParsedRequest & req) const { !config_.tool_speculation.enabled() || !config_.semantic_tool_predictor.enabled() || http_detail::tool_choice_disables_tool_calls(req.tool_choice) || - req.tools.empty() || !req.raw_body.is_object()) return; + req.tools.empty()) return; req.automatic_tool_speculation.emplace(); auto & launch = *req.automatic_tool_speculation; const SemanticToolPredictorConfig predictor = config_.semantic_tool_predictor; - json semantic_request = req.raw_body; - // Endpoint parsers normalize Anthropic/Responses dialogue into req.messages. - // Supplying it here gives both transports one OpenAI-shaped semantic view. - semantic_request["messages"] = req.messages; - semantic_request["tools"] = req.tools; - if (!req.tool_choice.is_null()) { - semantic_request["tool_choice"] = req.tool_choice; - } + std::string payload_error; const json payload = build_semantic_tool_predictor_request( - semantic_request, + req.messages, req.tools, req.tool_choice, predictor.model.empty() ? "native-qwen3" : predictor.model, - predictor.max_tokens); - const json tools = req.tools; + predictor.max_tokens, payload_error); + if (!payload_error.empty()) { + launch.predictor_error = std::move(payload_error); + return; + } const auto native = native_semantic_predictor_; try { const SemanticToolPrediction semantic = predict_semantic_tool_call( - predictor, payload, tools, native); + predictor, payload, req.tools, native); launch.predictor_wall_ms = semantic.wall_ms; launch.prediction_source = semantic.source; if (!semantic.ok) { diff --git a/server/src/server/semantic_tool_hint.cpp b/server/src/server/semantic_tool_hint.cpp index b18641357..9dc69bd65 100644 --- a/server/src/server/semantic_tool_hint.cpp +++ b/server/src/server/semantic_tool_hint.cpp @@ -4,6 +4,7 @@ #include #include +#include #include namespace dflash::common { @@ -155,6 +156,90 @@ bool semantic_deadline_expired( return deadline && std::chrono::steady_clock::now() >= *deadline; } +constexpr size_t kMaxNativePredictorRequestBytes = 256U * 1024U; +constexpr size_t kMaxNativePredictorJsonDepth = 64U; + +bool semantic_take_budget(size_t bytes, size_t & remaining) { + if (bytes > remaining) return false; + remaining -= bytes; + return true; +} + +bool semantic_string_within_budget( + const std::string & value, + const std::chrono::steady_clock::time_point * deadline, + size_t & remaining, + size_t & operations, + bool & timed_out) { + if (!semantic_take_budget(2, remaining)) return false; + for (const unsigned char character : value) { + if ((operations++ & 255U) == 0U && + semantic_deadline_expired(deadline)) { + timed_out = true; + return false; + } + const size_t encoded_bytes = character < 0x20U + ? 6U : (character == '"' || character == '\\' ? 2U : 1U); + if (!semantic_take_budget(encoded_bytes, remaining)) return false; + } + return true; +} + +bool semantic_json_within_budget( + const json & value, + const std::chrono::steady_clock::time_point * deadline, + size_t & remaining, + size_t depth, + size_t & operations, + bool & timed_out) { + if (depth > kMaxNativePredictorJsonDepth) return false; + if ((operations++ & 255U) == 0U && + semantic_deadline_expired(deadline)) { + timed_out = true; + return false; + } + if (value.is_null()) return semantic_take_budget(4, remaining); + if (value.is_boolean()) return semantic_take_budget(5, remaining); + if (value.is_number()) return semantic_take_budget(64, remaining); + if (value.is_string()) { + return semantic_string_within_budget( + value.get_ref(), deadline, + remaining, operations, timed_out); + } + if (value.is_array()) { + if (!semantic_take_budget(2, remaining)) return false; + bool first = true; + for (const auto & element : value) { + if (!first && !semantic_take_budget(1, remaining)) return false; + first = false; + if (!semantic_json_within_budget( + element, deadline, remaining, depth + 1, + operations, timed_out)) { + return false; + } + } + return true; + } + if (value.is_object()) { + if (!semantic_take_budget(2, remaining)) return false; + bool first = true; + for (const auto & item : value.items()) { + if (!first && !semantic_take_budget(1, remaining)) return false; + first = false; + if (!semantic_string_within_budget( + item.key(), deadline, remaining, operations, timed_out) || + !semantic_take_budget(1, remaining) || + !semantic_json_within_budget( + item.value(), deadline, remaining, depth + 1, + operations, timed_out)) { + return false; + } + } + return true; + } + return false; +} + bool semantic_message_content( const json & message, const std::chrono::steady_clock::time_point * deadline, @@ -300,20 +385,44 @@ bool materialize_declared_tool_defaults( } json build_semantic_tool_predictor_request( - const json & target_request, + const json & messages, + const json & tools, + const json & tool_choice, const std::string & sidecar_model, - int max_tokens) { + int max_tokens, + std::string & error) { + error.clear(); + size_t request_budget = kMaxNativePredictorRequestBytes; + size_t budget_operations = 0; + bool budget_timed_out = false; + const json default_choice = "auto"; + const json & effective_choice = tool_choice.is_null() + ? default_choice : tool_choice; + if (!semantic_take_budget(512, request_budget) || + !semantic_string_within_budget( + sidecar_model, nullptr, request_budget, + budget_operations, budget_timed_out) || + !semantic_json_within_budget( + messages, nullptr, request_budget, 0, + budget_operations, budget_timed_out) || + !semantic_json_within_budget( + tools, nullptr, request_budget, 0, + budget_operations, budget_timed_out) || + !semantic_json_within_budget( + effective_choice, nullptr, request_budget, 0, + budget_operations, budget_timed_out)) { + error = "predictor_request_too_large"; + return nullptr; + } json request = { {"model", sidecar_model}, {"stream", false}, {"temperature", 0}, {"max_tokens", max_tokens}, + {"messages", messages}, + {"tools", tools}, + {"tool_choice", effective_choice}, }; - for (const char * key : {"messages", "tools", "tool_choice"}) { - const auto it = target_request.find(key); - if (it != target_request.end()) request[key] = *it; - } - if (!request.contains("tool_choice")) request["tool_choice"] = "auto"; return request; } @@ -326,6 +435,17 @@ std::string build_native_semantic_tool_predictor_prompt( error = "native_predictor_timeout"; return {}; } + size_t request_budget = kMaxNativePredictorRequestBytes; + size_t budget_operations = 0; + bool budget_timed_out = false; + if (!semantic_json_within_budget( + predictor_request, deadline, request_budget, 0, + budget_operations, budget_timed_out)) { + error = budget_timed_out + ? "native_predictor_timeout" + : "native_predictor_request_too_large"; + return {}; + } const json choice = predictor_request.value("tool_choice", json("auto")); if ((choice.is_string() && choice.get() == "none") || (choice.is_object() && choice.value("type", "") == "none")) { diff --git a/server/src/server/semantic_tool_hint.h b/server/src/server/semantic_tool_hint.h index 254b6367d..7a7f02186 100644 --- a/server/src/server/semantic_tool_hint.h +++ b/server/src/server/semantic_tool_hint.h @@ -66,12 +66,16 @@ bool materialize_declared_tool_defaults( SemanticToolCall & call, std::string & error); -// Build the small OpenAI-compatible request sent to the predictor. Only -// dialogue/tool semantics are forwarded; target-only extensions are omitted. +// Build the bounded OpenAI-compatible request sent to the predictor. Only +// normalized dialogue/tool semantics are copied; target-only extensions are +// omitted. Oversized inputs fail before any full-field copy. json build_semantic_tool_predictor_request( - const json & target_request, + const json & messages, + const json & tools, + const json & tool_choice, const std::string & sidecar_model, - int max_tokens); + int max_tokens, + std::string & error); // Native predictor bridge. The prompt uses the Qwen tool template and the // decoded response is parsed semantically before any target token IDs exist. diff --git a/server/src/server/tokenizer.cpp b/server/src/server/tokenizer.cpp index ebc74e112..519cb248c 100644 --- a/server/src/server/tokenizer.cpp +++ b/server/src/server/tokenizer.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include namespace dflash::common { @@ -30,6 +31,44 @@ bool preprocessing_deadline_expired( return true; } +bool preprocessing_find_until( + const std::string & text, + const std::string & needle, + size_t start, + const std::chrono::steady_clock::time_point * deadline, + bool & timed_out, + size_t & found) { + found = std::string::npos; + if (needle.empty()) { + found = start <= text.size() ? start : std::string::npos; + return true; + } + if (start > text.size() || needle.size() > text.size() - start) { + return true; + } + + constexpr size_t kSearchPositionsPerDeadlineCheck = 4096; + const size_t last_start = text.size() - needle.size(); + size_t cursor = start; + while (cursor <= last_start) { + if (deadline && std::chrono::steady_clock::now() >= *deadline) { + timed_out = true; + return false; + } + const size_t positions = (std::min)( + kSearchPositionsPerDeadlineCheck, last_start - cursor + 1); + const size_t view_bytes = positions + needle.size() - 1; + const std::string_view window(text.data() + cursor, view_bytes); + const size_t local = window.find(needle); + if (local != std::string_view::npos && local < positions) { + found = cursor + local; + return true; + } + cursor += positions; + } + return true; +} + } // namespace // ─── Unicode helpers ──────────────────────────────────────────────────── @@ -744,7 +783,11 @@ std::vector Tokenizer::encode_impl( for (const auto & [tok_str, tok_id] : added_tokens_) { if (preprocessing_deadline_expired( deadline, operations, timed_out)) return {}; - size_t found = text.find(tok_str, pos); + size_t found = std::string::npos; + if (!preprocessing_find_until( + text, tok_str, pos, deadline, timed_out, found)) { + return {}; + } if (found != std::string::npos && found < next_special) { next_special = found; } diff --git a/server/src/server/tool_speculation.h b/server/src/server/tool_speculation.h index 01544f36d..fd1d890ee 100644 --- a/server/src/server/tool_speculation.h +++ b/server/src/server/tool_speculation.h @@ -153,10 +153,13 @@ struct ToolSpeculationConfig { bool allows(const std::string & name) const; }; +// Report whether this build can close every non-protocol descriptor before +// executing an untrusted tool child. Unsupported platforms fail closed. +bool tool_speculation_executor_isolation_supported(); + // Capture the model process affinity and fail closed unless it is physically // disjoint from the configured child executor CPUs. No-op when no CPU lane is // requested. -bool tool_speculation_executor_isolation_supported(); bool qualify_tool_speculation_cpu_affinity(ToolSpeculationConfig & config, std::string & error); diff --git a/server/test/test_semantic_tool_hint.cpp b/server/test/test_semantic_tool_hint.cpp index 0d7ee9cfd..31df6eebd 100644 --- a/server/test/test_semantic_tool_hint.cpp +++ b/server/test/test_semantic_tool_hint.cpp @@ -175,8 +175,11 @@ TEST_CASE(SemanticToolHintFixture, predictor_request_forwards_only_semantics) { {"tool_speculation", {{"name", "unsafe"}}}, {"prefix_cache", {{"scope", "full"}}}, }; + std::string error; const json request = build_semantic_tool_predictor_request( - target, "Qwen3-0.6B", 32); + target["messages"], target["tools"], target["tool_choice"], + "Qwen3-0.6B", 32, error); + CHECK(error.empty()); CHECK(request["model"] == "Qwen3-0.6B"); CHECK(request["max_tokens"] == 32); CHECK(request["tool_choice"] == "required"); @@ -184,6 +187,18 @@ TEST_CASE(SemanticToolHintFixture, predictor_request_forwards_only_semantics) { CHECK(!request.contains("prefix_cache")); } +TEST_CASE(SemanticToolHintFixture, predictor_request_rejects_oversized_fields_before_copy) { + const json messages = json::array({{ + {"role", "user"}, + {"content", std::string(300U * 1024U, 'x')}, + }}); + std::string error; + const json request = build_semantic_tool_predictor_request( + messages, weather_tools(), nullptr, "Qwen3-0.6B", 32, error); + CHECK(request.is_null()); + CHECK(error == "predictor_request_too_large"); +} + TEST_CASE(SemanticToolHintFixture, native_predictor_config_is_independent_of_http) { SemanticToolPredictorConfig config; config.native_model_path = "/models/qwen3-0.6b.gguf"; @@ -244,6 +259,28 @@ TEST_CASE(SemanticToolHintFixture, native_prompt_honors_preprocessing_deadline) CHECK(error == "native_predictor_timeout"); } +TEST_CASE(SemanticToolHintFixture, native_prompt_rejects_oversized_input_before_rendering) { + json request = { + {"messages", json::array({{{"role", "user"}, {"content", "weather"}}})}, + {"tools", weather_tools()}, + }; + const std::string oversized(300U * 1024U, 'x'); + const auto deadline = std::chrono::steady_clock::now() + + std::chrono::seconds(5); + std::string error; + + request["messages"][0]["content"] = oversized; + CHECK(build_native_semantic_tool_predictor_prompt( + request, error, &deadline).empty()); + CHECK(error == "native_predictor_request_too_large"); + + request["messages"][0]["content"] = "weather"; + request["tools"][0]["function"]["description"] = oversized; + CHECK(build_native_semantic_tool_predictor_prompt( + request, error, &deadline).empty()); + CHECK(error == "native_predictor_request_too_large"); +} + TEST_CASE(SemanticToolHintFixture, parses_native_qwen_xml_semantics) { const std::string generated = "\n" diff --git a/server/test/test_server_unit.cpp b/server/test/test_server_unit.cpp index eba9fa9ed..05118f2db 100644 --- a/server/test/test_server_unit.cpp +++ b/server/test/test_server_unit.cpp @@ -2002,6 +2002,21 @@ TEST_CASE(ServerUnitFixture, test_resolve_deepseek_chat_markers) { unlink(path.c_str()); } +TEST_CASE(ServerUnitFixture, test_tokenizer_added_token_search_obeys_deadline) { + const std::string path = write_deepseek_marker_tokenizer_fixture(); + Tokenizer tokenizer; + TEST_ASSERT(tokenizer.load_from_gguf(path.c_str())); + const std::string long_input(32U * 1024U * 1024U, 'x'); + const auto started = std::chrono::steady_clock::now(); + const auto deadline = started + std::chrono::milliseconds(1); + std::vector output; + TEST_ASSERT(!tokenizer.encode_until(long_input, deadline, output)); + const auto elapsed = std::chrono::steady_clock::now() - started; + TEST_ASSERT(elapsed < std::chrono::seconds(1)); + TEST_ASSERT(output.empty()); + unlink(path.c_str()); +} + TEST_CASE(ServerUnitFixture, test_hash_prefix_deterministic) { std::vector ids = {100, 200, 300, 400, 500}; auto h1 = hash_prefix(ids.data(), (int)ids.size()); From 50ac165acb9809191fe7ad763bd496e12ea2cb44 Mon Sep 17 00:00:00 2001 From: mrciffa <49000955+davide221@users.noreply.github.com> Date: Tue, 18 Aug 2026 20:14:02 +0200 Subject: [PATCH 09/11] fix(server): resolve remaining tool speculation review --- .../benchmark_cpu_tool_speculation.py | 33 ++-- .../test_benchmark_cpu_tool_speculation.py | 44 +++++ server/src/common/backend_ipc.cpp | 167 ++++++++++++------ server/src/common/io_utils.h | 16 +- .../src/common/qwen3_tool_predictor_ipc.cpp | 5 + server/src/server/http_server.cpp | 83 +++++++-- server/src/server/http_server.h | 10 ++ .../server/native_semantic_tool_predictor.cpp | 6 +- .../server/native_semantic_tool_predictor.h | 3 +- server/src/server/semantic_tool_hint.cpp | 71 ++++++-- server/src/server/semantic_tool_hint.h | 26 ++- server/src/server/tokenizer.cpp | 9 + server/src/server/tool_speculation.cpp | 7 + .../test/smoke_qwen3_tool_predictor_ipc.cpp | 39 ++-- server/test/test_semantic_tool_hint.cpp | 113 ++++++------ server/test/test_server_unit.cpp | 52 ++++++ 16 files changed, 507 insertions(+), 177 deletions(-) diff --git a/optimizations/ooo_spec_lucebox5_cpu/benchmark_cpu_tool_speculation.py b/optimizations/ooo_spec_lucebox5_cpu/benchmark_cpu_tool_speculation.py index 4e62052cd..ac70881b2 100755 --- a/optimizations/ooo_spec_lucebox5_cpu/benchmark_cpu_tool_speculation.py +++ b/optimizations/ooo_spec_lucebox5_cpu/benchmark_cpu_tool_speculation.py @@ -372,26 +372,33 @@ def group_exists() -> bool: except PermissionError: return True + def wait_for_group_exit(timeout: float) -> bool: + deadline = time.monotonic() + timeout + while group_exists(): + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + if process.poll() is None: + try: + process.wait(timeout=min(0.01, remaining)) + except subprocess.TimeoutExpired: + pass + else: + time.sleep(min(0.01, remaining)) + return True + try: os.killpg(pgid, signal.SIGTERM) - except ProcessLookupError: + except (ProcessLookupError, PermissionError): pass - grace_deadline = time.monotonic() + 1.0 - while group_exists() and time.monotonic() < grace_deadline: - if process.poll() is None: - try: - process.wait(timeout=0.01) - except subprocess.TimeoutExpired: - pass - else: - time.sleep(0.01) - - if group_exists(): + if not wait_for_group_exit(1.0): try: os.killpg(pgid, signal.SIGKILL) - except ProcessLookupError: + except (ProcessLookupError, PermissionError): pass + if not wait_for_group_exit(5.0): + raise RuntimeError("CPU executor process group did not stop") if process.poll() is None: try: diff --git a/optimizations/ooo_spec_lucebox5_cpu/test_benchmark_cpu_tool_speculation.py b/optimizations/ooo_spec_lucebox5_cpu/test_benchmark_cpu_tool_speculation.py index 9d497e8ab..92377a2f7 100644 --- a/optimizations/ooo_spec_lucebox5_cpu/test_benchmark_cpu_tool_speculation.py +++ b/optimizations/ooo_spec_lucebox5_cpu/test_benchmark_cpu_tool_speculation.py @@ -52,6 +52,50 @@ def kill_group(_pgid: int, sent_signal: int) -> None: with self.assertRaisesRegex(RuntimeError, "did not stop"): stop_executor({"process": process, "pgid": 4321}) + def test_stop_executor_waits_for_group_after_sigkill(self) -> None: + process = Mock(pid=4321) + process.poll.return_value = 0 + alive = True + + def kill_group(_pgid: int, sent_signal: int) -> None: + nonlocal alive + if sent_signal == signal.SIGKILL: + alive = False + elif sent_signal == 0 and not alive: + raise ProcessLookupError + + with ( + patch( + "benchmark_cpu_tool_speculation.os.killpg", + side_effect=kill_group, + ) as killpg, + patch( + "benchmark_cpu_tool_speculation.time.monotonic", + side_effect=[0.0, 2.0, 3.0], + ), + ): + stop_executor({"process": process, "pgid": 4321}) + + calls = [call.args[1] for call in killpg.call_args_list] + self.assertEqual(calls, [signal.SIGTERM, 0, signal.SIGKILL, 0]) + + def test_stop_executor_reports_permission_failure_cleanly(self) -> None: + process = Mock(pid=4321) + process.poll.return_value = 0 + + with ( + patch( + "benchmark_cpu_tool_speculation.os.killpg", + side_effect=PermissionError, + ), + patch( + "benchmark_cpu_tool_speculation.time.monotonic", + side_effect=[0.0, 2.0, 3.0, 9.0], + ), + ): + with self.assertRaisesRegex(RuntimeError, "did not stop"): + stop_executor({"process": process, "pgid": 4321}) + def test_cpu_list_parser_canonicalizes_ranges(self) -> None: self.assertEqual(parse_cpu_list("30-31,15,14-15"), [14, 15, 30, 31]) with self.assertRaises(argparse.ArgumentTypeError): diff --git a/server/src/common/backend_ipc.cpp b/server/src/common/backend_ipc.cpp index ca765a21b..3551f184d 100644 --- a/server/src/common/backend_ipc.cpp +++ b/server/src/common/backend_ipc.cpp @@ -4,6 +4,7 @@ #include "io_utils.h" #include +#include #include #include #include @@ -12,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -22,7 +22,6 @@ # include # include # include -# include # include # include # if defined(__linux__) @@ -36,24 +35,7 @@ namespace dflash::common { namespace { #if !defined(_WIN32) -unsigned int descriptor_scan_limit() { - struct rlimit limit {}; - if (::getrlimit(RLIMIT_NOFILE, &limit) == 0 && - limit.rlim_cur != RLIM_INFINITY && limit.rlim_cur > 0) { - return static_cast((std::min)( - static_cast(limit.rlim_cur), - static_cast(INT_MAX))); - } - const long open_max = ::sysconf(_SC_OPEN_MAX); - if (open_max <= 0) return 0; - return static_cast((std::min)( - static_cast(open_max), - static_cast(INT_MAX))); -} - -bool close_descriptor_range(unsigned int first, - unsigned int last, - unsigned int scan_limit) { +bool close_descriptor_range(unsigned int first, unsigned int last) { if (first > last) return true; #if defined(__linux__) && defined(SYS_close_range) int rc = -1; @@ -62,44 +44,131 @@ bool close_descriptor_range(unsigned int first, } while (rc != 0 && errno == EINTR); if (rc == 0) return true; #endif + return false; +} - // close_range is Linux-specific and may also be blocked by an older - // kernel or seccomp policy. Fall back to the POSIX descriptor space that - // was captured before fork, and verify every ambiguous close failure. - if (scan_limit == 0 || first >= scan_limit) return scan_limit != 0; - const unsigned int upper = (std::min)(last, scan_limit - 1U); - for (unsigned int fd = first; fd <= upper; ++fd) { - if (::close(static_cast(fd)) == 0 || errno == EBADF) continue; - const int close_error = errno; - errno = 0; - if (::fcntl(static_cast(fd), F_GETFD) < 0 && errno == EBADF) { - continue; +bool descriptor_is_kept(const std::array & keep, int fd) { + return fd == keep[0] || fd == keep[1] || fd == keep[2]; +} + +// close_range can be unavailable on an older kernel or denied by seccomp. +// Enumerate the descriptors that actually exist instead of trusting the +// current RLIMIT_NOFILE: a process may have opened a high descriptor before +// lowering its soft limit. Raw getdents64 keeps this post-fork path free of +// libc directory-stream allocation and turns an unavailable /proc into a +// fail-closed launch. +bool close_unlisted_descriptors(const std::array & keep) { +#if defined(__linux__) && defined(SYS_getdents64) + struct LinuxDirent64 { + uint64_t inode; + int64_t offset; + unsigned short record_bytes; + unsigned char type; + char name[1]; + }; + + const int directory_fd = ::open( + "/proc/self/fd", O_RDONLY | O_DIRECTORY | O_CLOEXEC | O_NOFOLLOW); + if (directory_fd < 0) return false; + + alignas(long) char entries[4096]; + bool ok = true; + int failure = 0; + while (ok) { + long bytes = -1; + do { + bytes = ::syscall( + SYS_getdents64, directory_fd, entries, sizeof(entries)); + } while (bytes < 0 && errno == EINTR); + if (bytes == 0) break; + if (bytes < 0) { + ok = false; + failure = errno; + break; + } + + size_t cursor = 0; + while (cursor < static_cast(bytes)) { + auto * entry = reinterpret_cast(entries + cursor); + constexpr size_t name_offset = offsetof(LinuxDirent64, name); + if (static_cast(bytes) - cursor <= name_offset || + entry->record_bytes <= name_offset || + entry->record_bytes > static_cast(bytes) - cursor) { + ok = false; + failure = EIO; + break; + } + const size_t name_bytes = entry->record_bytes - name_offset; + const char * terminator = static_cast( + std::memchr(entry->name, '\0', name_bytes)); + if (!terminator) { + ok = false; + failure = EIO; + break; + } + + uint64_t parsed = 0; + bool numeric = terminator != entry->name; + for (const char * digit = entry->name; digit < terminator; ++digit) { + if (*digit < '0' || *digit > '9') { + numeric = false; + break; + } + parsed = parsed * 10U + static_cast(*digit - '0'); + if (parsed > static_cast(INT_MAX)) { + numeric = false; + break; + } + } + if (numeric) { + const int fd = static_cast(parsed); + if (fd > STDERR_FILENO && fd != directory_fd && + !descriptor_is_kept(keep, fd) && + ::close(fd) != 0 && errno != EBADF) { + const int close_error = errno; + errno = 0; + if (!(::fcntl(fd, F_GETFD) < 0 && errno == EBADF)) { + ok = false; + failure = close_error; + break; + } + } + } + cursor += entry->record_bytes; } - errno = close_error; - return false; } - return true; + + if (::close(directory_fd) != 0 && ok) { + ok = false; + failure = errno; + } + if (!ok) errno = failure == 0 ? EIO : failure; + return ok; +#else + (void)keep; + errno = ENOTSUP; + return false; +#endif } -bool isolate_child_descriptors(int payload_fd, - int stream_fd, - int shared_fd, - unsigned int scan_limit) { +bool isolate_child_descriptors( + int payload_fd, int stream_fd, int shared_fd) { std::array keep{payload_fd, stream_fd, shared_fd}; std::sort(keep.begin(), keep.end()); unsigned int first = STDERR_FILENO + 1; int previous = -1; + bool ranges_closed = true; for (const int fd : keep) { if (fd < static_cast(first) || fd == previous) continue; - if (!close_descriptor_range( - first, static_cast(fd - 1), scan_limit)) { - return false; + if (!close_descriptor_range(first, static_cast(fd - 1))) { + ranges_closed = false; + break; } first = static_cast(fd) + 1U; previous = fd; } - return close_descriptor_range( - first, (std::numeric_limits::max)(), scan_limit); + if (ranges_closed && close_descriptor_range(first, UINT_MAX)) return true; + return close_unlisted_descriptors(keep); } #endif @@ -191,15 +260,6 @@ bool BackendIpcProcess::start(const BackendIpcLaunchConfig & cfg) { close(); if (cfg.bin.empty() || cfg.payload_path.empty()) return false; if (!init_work_dir(cfg.work_dir, cfg.require_private_work_dir)) return false; - const unsigned int descriptor_limit = cfg.isolate_inherited_fds - ? descriptor_scan_limit() : 0; - if (cfg.isolate_inherited_fds && descriptor_limit == 0) { - std::fprintf(stderr, - "backend-ipc cannot determine descriptor scan limit\n"); - close(); - return false; - } - int cmd_pipe[2] = {-1, -1}; int payload_pipe[2] = {-1, -1}; int stream_pipe[2] = {-1, -1}; @@ -295,8 +355,7 @@ bool BackendIpcProcess::start(const BackendIpcLaunchConfig & cfg) { argv.push_back(nullptr); if (cfg.isolate_inherited_fds && !isolate_child_descriptors( - payload_pipe[0], stream_pipe[1], shared_payload_fd_, - descriptor_limit)) { + payload_pipe[0], stream_pipe[1], shared_payload_fd_)) { std::fprintf(stderr, "backend-ipc descriptor isolation failed: %s\n", std::strerror(errno)); diff --git a/server/src/common/io_utils.h b/server/src/common/io_utils.h index f66b69ed4..6097c3616 100644 --- a/server/src/common/io_utils.h +++ b/server/src/common/io_utils.h @@ -137,17 +137,27 @@ static inline bool read_exact_fd_until( deadline - now).count(); pollfd descriptor{fd, POLLIN | POLLHUP, 0}; const int wait_ms = static_cast((std::min)( - int64_t{INT_MAX}, (std::max)(int64_t{1}, remaining))); + int64_t{INT_MAX}, (std::max)(int64_t{0}, remaining))); const int polled = ::poll(&descriptor, 1, wait_ms); if (polled == 0) { - timed_out = true; - return false; + if (std::chrono::steady_clock::now() >= deadline) { + timed_out = true; + return false; + } + continue; } if (polled < 0) { if (errno == EINTR) continue; return false; } if (descriptor.revents & (POLLERR | POLLNVAL)) return false; + // Readiness does not extend the wall-clock budget. In particular, a + // descriptor that becomes readable during poll's final millisecond + // must not be consumed after the deadline. + if (std::chrono::steady_clock::now() >= deadline) { + timed_out = true; + return false; + } const ssize_t count = ::read( fd, cursor + received, bytes - received); if (count == 0) return false; diff --git a/server/src/common/qwen3_tool_predictor_ipc.cpp b/server/src/common/qwen3_tool_predictor_ipc.cpp index 9040fea1b..643b363bc 100644 --- a/server/src/common/qwen3_tool_predictor_ipc.cpp +++ b/server/src/common/qwen3_tool_predictor_ipc.cpp @@ -202,6 +202,11 @@ bool Qwen3ToolPredictorIpcClient::predict( active_ = false; return false; } + if (std::chrono::steady_clock::now() >= deadline) { + output_ids.clear(); + error = "native_predictor_timeout"; + return false; + } return true; #endif } diff --git a/server/src/server/http_server.cpp b/server/src/server/http_server.cpp index e91b6c63b..1efeddbea 100644 --- a/server/src/server/http_server.cpp +++ b/server/src/server/http_server.cpp @@ -553,6 +553,7 @@ bool semantic_sidecar_send_all( (descriptor.revents & (POLLERR | POLLHUP | POLLNVAL))) { return false; } + if (std::chrono::steady_clock::now() >= deadline) return false; #if defined(_WIN32) const int n = ::send( fd, data + sent, @@ -629,7 +630,8 @@ bool semantic_sidecar_connect( } int socket_error = 0; socklen_t error_size = static_cast(sizeof(socket_error)); - if (polled <= 0 || !(descriptor.revents & POLLOUT) || + if (std::chrono::steady_clock::now() >= deadline || + polled <= 0 || !(descriptor.revents & POLLOUT) || getsockopt(fd, SOL_SOCKET, SO_ERROR, reinterpret_cast(&socket_error), &error_size) != 0 || socket_error != 0) { @@ -749,6 +751,11 @@ SemanticToolPrediction request_semantic_tool_prediction( prediction.error = "predictor_receive_failed"; return finish(); } + if (std::chrono::steady_clock::now() >= deadline) { + socket_close(fd); + prediction.error = "predictor_timeout"; + return finish(); + } #if defined(_WIN32) const int received = recv( fd, buffer.data(), static_cast(buffer.size()), 0); @@ -827,13 +834,13 @@ SemanticToolPrediction request_semantic_tool_prediction( SemanticToolPrediction predict_semantic_tool_call( const SemanticToolPredictorConfig & config, - const json & payload, + const SemanticToolPredictorRequest & request, const json & request_tools, const std::shared_ptr & native) { const auto started = std::chrono::steady_clock::now(); SemanticToolPrediction native_result; if (native && native->active()) { - native_result = native->predict(payload, request_tools); + native_result = native->predict(request, request_tools); if (native_result.ok || !config.http_enabled()) { return native_result; } @@ -862,7 +869,7 @@ SemanticToolPrediction predict_semantic_tool_call( SemanticToolPredictorConfig fallback_config = config; fallback_config.timeout_ms = remaining_ms; SemanticToolPrediction fallback = request_semantic_tool_prediction( - fallback_config, payload, request_tools); + fallback_config, request.payload(), request_tools); if (!fallback.ok && !native_result.error.empty()) { fallback.error = "native=" + native_result.error + ";http=" + fallback.error; @@ -1482,6 +1489,25 @@ std::vector normalize_chat_messages( return chat_msgs; } +namespace http_detail { + +json canonical_predictor_messages(std::vector messages) { + json canonical = json::array(); + for (ChatMessage & message : messages) { + json item = { + {"role", std::move(message.role)}, + {"content", std::move(message.content)}, + }; + if (!message.tool_call_id.empty()) { + item["tool_call_id"] = std::move(message.tool_call_id); + } + canonical.push_back(std::move(item)); + } + return canonical; +} + +} // namespace http_detail + // ─── Disk-cache identity salt ─────────────────────────────────────────── // Compute a 16-byte salt from inputs that affect KV cache validity: // model path + stat(size + mtime) [covers rope/yarn — GGUF-derived], @@ -2496,19 +2522,36 @@ void HttpServer::start_automatic_tool_speculation(ParsedRequest & req) const { auto & launch = *req.automatic_tool_speculation; const SemanticToolPredictorConfig predictor = config_.semantic_tool_predictor; + const auto native = native_semantic_predictor_; + auto log_completion = [&]() { + std::fprintf(stderr, + "[tool-hint] predictor complete transport=%s%s tools=%zu " + "execute=%s wall_ms=%.1f\n", + native ? "native-qwen3" : "http", + native && predictor.http_enabled() ? "+http-fallback" : "", + json_array_size(req.tools), launch.attempt ? "true" : "false", + launch.predictor_wall_ms); + }; + const auto preflight_started = std::chrono::steady_clock::now(); std::string payload_error; - const json payload = build_semantic_tool_predictor_request( - req.messages, req.tools, req.tool_choice, + const json & predictor_messages = req.predictor_messages.is_array() + ? req.predictor_messages : req.messages; + const auto payload = build_semantic_tool_predictor_request( + predictor_messages, req.tools, req.tool_choice, predictor.model.empty() ? "native-qwen3" : predictor.model, predictor.max_tokens, payload_error); - if (!payload_error.empty()) { - launch.predictor_error = std::move(payload_error); + if (!payload.has_value()) { + launch.predictor_error = payload_error.empty() + ? "predictor_request_invalid" : std::move(payload_error); + launch.prediction_source = "request-preflight"; + launch.predictor_wall_ms = std::chrono::duration( + std::chrono::steady_clock::now() - preflight_started).count(); + log_completion(); return; } - const auto native = native_semantic_predictor_; try { const SemanticToolPrediction semantic = predict_semantic_tool_call( - predictor, payload, req.tools, native); + predictor, *payload, req.tools, native); launch.predictor_wall_ms = semantic.wall_ms; launch.prediction_source = semantic.source; if (!semantic.ok) { @@ -2537,13 +2580,7 @@ void HttpServer::start_automatic_tool_speculation(ParsedRequest & req) const { launch.predictor_error = "automatic_prediction_failed: unknown error"; } - std::fprintf(stderr, - "[tool-hint] predictor complete transport=%s%s tools=%zu " - "execute=%s wall_ms=%.1f\n", - native ? "native-qwen3" : "http", - native && predictor.http_enabled() ? "+http-fallback" : "", - json_array_size(req.tools), launch.attempt ? "true" : "false", - launch.predictor_wall_ms); + log_completion(); } void HttpServer::enqueue_request_and_wait(SocketHandle fd, ParsedRequest req) { @@ -2627,6 +2664,18 @@ bool HttpServer::route_request(SocketHandle fd, const HttpRequest & hr) { if (!render_and_tokenize_request(fd, render_messages, req)) return true; + if (config_.semantic_tool_predictor.enabled() && + config_.tool_speculation.enabled() && + config_.pflash_upstream_base.empty() && + req.automatic_tool_speculation_enabled && + !req.tool_speculation.has_value() && + !http_detail::tool_choice_disables_tool_calls(req.tool_choice) && + !req.tools.empty()) { + req.predictor_messages = + http_detail::canonical_predictor_messages( + std::move(render_messages)); + } + // count_tokens: short-circuit after tokenization. Skip generation // entirely — Anthropic's contract is just {"input_tokens": N}. if (count_tokens_only) { diff --git a/server/src/server/http_server.h b/server/src/server/http_server.h index 6c4d4d602..900c7757e 100644 --- a/server/src/server/http_server.h +++ b/server/src/server/http_server.h @@ -264,6 +264,11 @@ bool should_clamp_flowkv_disk_cache( // True for API dialects that explicitly prohibit a tool call. Used before // either caller-supplied or automatic speculative execution can start. bool tool_choice_disables_tool_calls(const json & tool_choice); +// Convert the exact normalized dialogue rendered for the target into the +// OpenAI-compatible message shape consumed by semantic predictors. The input +// is taken by value so callers can move the rendered message storage into the +// predictor payload without another full content copy. +json canonical_predictor_messages(std::vector messages); } // namespace http_detail @@ -282,6 +287,11 @@ struct ParsedRequest { json tool_choice; // Original messages (for response formatting) json messages; + // Canonical dialogue used by the semantic predictor. Responses API + // function_call/function_call_output items are represented here as the + // assistant/tool turns seen by the target instead of disappearing during + // predictor prompt construction. + json predictor_messages; // Original request body (for upstream proxy forwarding) json raw_body; // Concrete invocation predicted by a caller or future semantic sidecar. diff --git a/server/src/server/native_semantic_tool_predictor.cpp b/server/src/server/native_semantic_tool_predictor.cpp index 0975388a2..3e7f55085 100644 --- a/server/src/server/native_semantic_tool_predictor.cpp +++ b/server/src/server/native_semantic_tool_predictor.cpp @@ -38,7 +38,7 @@ NativeSemanticToolPredictor::create( } SemanticToolPrediction NativeSemanticToolPredictor::predict( - const json & predictor_request, + const SemanticToolPredictorRequest & predictor_request, const json & request_tools, std::string * generated_text) { const auto started = std::chrono::steady_clock::now(); @@ -91,6 +91,10 @@ SemanticToolPrediction NativeSemanticToolPredictor::predict( output_ids, prediction.error)) { return finish(); } + if (std::chrono::steady_clock::now() >= deadline) { + prediction.error = "native_predictor_timeout"; + return finish(); + } const std::string generated = tokenizer_.decode(output_ids); if (generated_text) *generated_text = generated; if (!parse_native_semantic_tool_prediction( diff --git a/server/src/server/native_semantic_tool_predictor.h b/server/src/server/native_semantic_tool_predictor.h index 088db4344..125ac38cd 100644 --- a/server/src/server/native_semantic_tool_predictor.h +++ b/server/src/server/native_semantic_tool_predictor.h @@ -22,7 +22,8 @@ class NativeSemanticToolPredictor { NativeSemanticToolPredictor & operator=( const NativeSemanticToolPredictor &) = delete; - SemanticToolPrediction predict(const json & predictor_request, + SemanticToolPrediction predict( + const SemanticToolPredictorRequest & predictor_request, const json & request_tools, std::string * generated_text = nullptr); diff --git a/server/src/server/semantic_tool_hint.cpp b/server/src/server/semantic_tool_hint.cpp index 9dc69bd65..d65dabdc1 100644 --- a/server/src/server/semantic_tool_hint.cpp +++ b/server/src/server/semantic_tool_hint.cpp @@ -281,6 +281,41 @@ std::string forced_tool_name(const json & choice) { return choice.value("name", ""); } +json canonical_semantic_tools(const json & tools) { + json canonical = json::array(); + if (!tools.is_array()) return canonical; + for (const auto & tool : tools) { + if (!tool.is_object()) continue; + const json * source = &tool; + const auto wrapped = tool.find("function"); + if (wrapped != tool.end() && wrapped->is_object()) source = &*wrapped; + const std::string name = source->value("name", ""); + if (name.empty()) continue; + + json function = {{"name", name}}; + const auto description = source->find("description"); + if (description != source->end() && description->is_string()) { + function["description"] = *description; + } + for (const char * schema_key : {"parameters", "input_schema"}) { + const auto schema = source->find(schema_key); + if (schema != source->end() && schema->is_object()) { + function["parameters"] = *schema; + break; + } + } + const auto strict = source->find("strict"); + if (strict != source->end() && strict->is_boolean()) { + function["strict"] = *strict; + } + canonical.push_back({ + {"type", "function"}, + {"function", std::move(function)}, + }); + } + return canonical; +} + } // namespace bool parse_semantic_tool_prediction( @@ -384,7 +419,8 @@ bool materialize_declared_tool_defaults( return true; } -json build_semantic_tool_predictor_request( +std::optional +build_semantic_tool_predictor_request( const json & messages, const json & tools, const json & tool_choice, @@ -412,7 +448,7 @@ json build_semantic_tool_predictor_request( effective_choice, nullptr, request_budget, 0, budget_operations, budget_timed_out)) { error = "predictor_request_too_large"; - return nullptr; + return std::nullopt; } json request = { {"model", sidecar_model}, @@ -420,14 +456,27 @@ json build_semantic_tool_predictor_request( {"temperature", 0}, {"max_tokens", max_tokens}, {"messages", messages}, - {"tools", tools}, + {"tools", canonical_semantic_tools(tools)}, {"tool_choice", effective_choice}, }; - return request; + // Tool-schema normalization can add OpenAI wrapper objects. Validate the + // canonical result once before granting the bounded-request type; native + // rendering then relies on that type instead of scanning the same payload + // for a third time. + size_t canonical_budget = kMaxNativePredictorRequestBytes; + size_t canonical_operations = 0; + bool canonical_timed_out = false; + if (!semantic_json_within_budget( + request, nullptr, canonical_budget, 0, + canonical_operations, canonical_timed_out)) { + error = "predictor_request_too_large"; + return std::nullopt; + } + return SemanticToolPredictorRequest(std::move(request)); } std::string build_native_semantic_tool_predictor_prompt( - const json & predictor_request, + const SemanticToolPredictorRequest & bounded_request, std::string & error, const std::chrono::steady_clock::time_point * deadline) { error.clear(); @@ -435,17 +484,7 @@ std::string build_native_semantic_tool_predictor_prompt( error = "native_predictor_timeout"; return {}; } - size_t request_budget = kMaxNativePredictorRequestBytes; - size_t budget_operations = 0; - bool budget_timed_out = false; - if (!semantic_json_within_budget( - predictor_request, deadline, request_budget, 0, - budget_operations, budget_timed_out)) { - error = budget_timed_out - ? "native_predictor_timeout" - : "native_predictor_request_too_large"; - return {}; - } + const json & predictor_request = bounded_request.payload(); const json choice = predictor_request.value("tool_choice", json("auto")); if ((choice.is_string() && choice.get() == "none") || (choice.is_object() && choice.value("type", "") == "none")) { diff --git a/server/src/server/semantic_tool_hint.h b/server/src/server/semantic_tool_hint.h index 7a7f02186..418b3d96d 100644 --- a/server/src/server/semantic_tool_hint.h +++ b/server/src/server/semantic_tool_hint.h @@ -5,7 +5,9 @@ #include #include +#include #include +#include namespace dflash::common { @@ -48,6 +50,25 @@ struct SemanticToolPrediction { double wall_ms = 0.0; }; +// A predictor payload that has passed the recursive byte/depth admission gate. +// Keeping construction private makes the native prompt renderer's bounded-input +// invariant explicit and prevents accidental duplicate validation work. +class SemanticToolPredictorRequest { +public: + const json & payload() const { return payload_; } + +private: + explicit SemanticToolPredictorRequest(json payload) + : payload_(std::move(payload)) {} + + friend std::optional + build_semantic_tool_predictor_request( + const json &, const json &, const json &, const std::string &, int, + std::string &); + + json payload_; +}; + // Parse one OpenAI-compatible sidecar response and reject calls whose // function name is absent from the request schema. Arguments remain decoded // JSON values; sidecar token IDs are never accepted by the target. @@ -69,7 +90,8 @@ bool materialize_declared_tool_defaults( // Build the bounded OpenAI-compatible request sent to the predictor. Only // normalized dialogue/tool semantics are copied; target-only extensions are // omitted. Oversized inputs fail before any full-field copy. -json build_semantic_tool_predictor_request( +std::optional +build_semantic_tool_predictor_request( const json & messages, const json & tools, const json & tool_choice, @@ -80,7 +102,7 @@ json build_semantic_tool_predictor_request( // Native predictor bridge. The prompt uses the Qwen tool template and the // decoded response is parsed semantically before any target token IDs exist. std::string build_native_semantic_tool_predictor_prompt( - const json & predictor_request, + const SemanticToolPredictorRequest & predictor_request, std::string & error, const std::chrono::steady_clock::time_point * deadline = nullptr); diff --git a/server/src/server/tokenizer.cpp b/server/src/server/tokenizer.cpp index 519cb248c..a350d3ff3 100644 --- a/server/src/server/tokenizer.cpp +++ b/server/src/server/tokenizer.cpp @@ -21,6 +21,8 @@ namespace dflash::common { namespace { +constexpr size_t kMaxAddedTokenBytes = 4096; + bool preprocessing_deadline_expired( const std::chrono::steady_clock::time_point * deadline, size_t & operations, @@ -666,6 +668,13 @@ bool Tokenizer::load_from_gguf(const char * model_path) { if (ttype == 3 || ttype == 4) { const std::string & tok = id_to_token_[i]; if (!tok.empty()) { + if (tok.size() > kMaxAddedTokenBytes) { + std::fprintf(stderr, + "[tokenizer] special token %d exceeds %zu-byte limit\n", + i, kMaxAddedTokenBytes); + gguf_free(gctx); + return false; + } added_tokens_.push_back({tok, (int32_t)i}); } } diff --git a/server/src/server/tool_speculation.cpp b/server/src/server/tool_speculation.cpp index 40c43d944..0f3ddae9e 100644 --- a/server/src/server/tool_speculation.cpp +++ b/server/src/server/tool_speculation.cpp @@ -1003,6 +1003,13 @@ bool ToolSpeculationAttempt::collect_executor_result( return false; } if (polled == 0) continue; + if (std::chrono::steady_clock::now() >= deadline) { + error = "executor_timeout"; + terminate_executor(); + wait_ms = std::chrono::duration( + std::chrono::steady_clock::now() - wait_started).count(); + return false; + } if (descriptor.revents & (POLLERR | POLLNVAL)) { error = "executor_stdout_failed"; terminate_executor(); diff --git a/server/test/smoke_qwen3_tool_predictor_ipc.cpp b/server/test/smoke_qwen3_tool_predictor_ipc.cpp index 41512d44a..6af577032 100644 --- a/server/test/smoke_qwen3_tool_predictor_ipc.cpp +++ b/server/test/smoke_qwen3_tool_predictor_ipc.cpp @@ -32,19 +32,17 @@ json production_tools() { )json"); } -json make_request(const std::string & prompt, const json & tools, - int32_t max_tokens) { - return { - {"model", "native-qwen3"}, - {"messages", json::array({{ +std::optional make_request( + const std::string & prompt, + const json & tools, + int32_t max_tokens, + std::string & error) { + return build_semantic_tool_predictor_request( + json::array({{ {"role", "user"}, {"content", prompt}, - }})}, - {"tools", tools}, - {"tool_choice", "required"}, - {"temperature", 0}, - {"max_tokens", max_tokens}, - }; + }}), + tools, "required", "native-qwen3", max_tokens, error); } std::vector production_cases() { @@ -141,8 +139,13 @@ int main(int argc, char ** argv) { if (argc > 4) { const std::string prompt = argv[4]; std::string generated; - const auto prediction = predictor->predict( - make_request(prompt, tools, config.max_tokens), tools, &generated); + const auto request = make_request( + prompt, tools, config.max_tokens, error); + if (!request.has_value()) { + std::fprintf(stderr, "request rejected: %s\n", error.c_str()); + return 1; + } + const auto prediction = predictor->predict(*request, tools, &generated); print_prediction("custom", prediction, "", ordered_json::object(), generated); return prediction.ok ? 0 : 1; @@ -156,9 +159,13 @@ int main(int argc, char ** argv) { std::vector walls; for (const Case & test_case : cases) { std::string generated; - const auto prediction = predictor->predict( - make_request(test_case.prompt, tools, config.max_tokens), tools, - &generated); + const auto request = make_request( + test_case.prompt, tools, config.max_tokens, error); + if (!request.has_value()) { + std::fprintf(stderr, "request rejected: %s\n", error.c_str()); + return 1; + } + const auto prediction = predictor->predict(*request, tools, &generated); print_prediction(test_case.id, prediction, test_case.expected_name, test_case.expected_arguments, generated); valid += prediction.ok ? 1 : 0; diff --git a/server/test/test_semantic_tool_hint.cpp b/server/test/test_semantic_tool_hint.cpp index 31df6eebd..9552b3906 100644 --- a/server/test/test_semantic_tool_hint.cpp +++ b/server/test/test_semantic_tool_hint.cpp @@ -176,15 +176,17 @@ TEST_CASE(SemanticToolHintFixture, predictor_request_forwards_only_semantics) { {"prefix_cache", {{"scope", "full"}}}, }; std::string error; - const json request = build_semantic_tool_predictor_request( + const auto request = build_semantic_tool_predictor_request( target["messages"], target["tools"], target["tool_choice"], "Qwen3-0.6B", 32, error); CHECK(error.empty()); - CHECK(request["model"] == "Qwen3-0.6B"); - CHECK(request["max_tokens"] == 32); - CHECK(request["tool_choice"] == "required"); - CHECK(!request.contains("tool_speculation")); - CHECK(!request.contains("prefix_cache")); + REQUIRE(request.has_value()); + const json & payload = request->payload(); + CHECK(payload["model"] == "Qwen3-0.6B"); + CHECK(payload["max_tokens"] == 32); + CHECK(payload["tool_choice"] == "required"); + CHECK(!payload.contains("tool_speculation")); + CHECK(!payload.contains("prefix_cache")); } TEST_CASE(SemanticToolHintFixture, predictor_request_rejects_oversized_fields_before_copy) { @@ -193,12 +195,44 @@ TEST_CASE(SemanticToolHintFixture, predictor_request_rejects_oversized_fields_be {"content", std::string(300U * 1024U, 'x')}, }}); std::string error; - const json request = build_semantic_tool_predictor_request( + const auto request = build_semantic_tool_predictor_request( messages, weather_tools(), nullptr, "Qwen3-0.6B", 32, error); - CHECK(request.is_null()); + CHECK(!request.has_value()); + CHECK(error == "predictor_request_too_large"); + + const json small_messages = json::array({{ + {"role", "user"}, {"content", "weather"}, + }}); + json oversized_tools = weather_tools(); + oversized_tools[0]["function"]["description"] = + std::string(300U * 1024U, 'x'); + const auto oversized_tool_request = build_semantic_tool_predictor_request( + small_messages, oversized_tools, nullptr, "Qwen3-0.6B", 32, error); + CHECK(!oversized_tool_request.has_value()); CHECK(error == "predictor_request_too_large"); } +TEST_CASE(SemanticToolHintFixture, predictor_request_canonicalizes_anthropic_tools) { + const json tools = json::array({{ + {"name", "get_weather"}, + {"description", "Read weather"}, + {"input_schema", { + {"type", "object"}, + {"properties", {{"city", {{"type", "string"}}}}}, + }}, + }}); + std::string error; + const auto request = build_semantic_tool_predictor_request( + json::array({{{"role", "user"}, {"content", "weather"}}}), + tools, nullptr, "Qwen3-0.6B", 32, error); + REQUIRE(request.has_value()); + const json & canonical = request->payload()["tools"][0]; + CHECK(canonical["type"] == "function"); + CHECK(canonical["function"]["name"] == "get_weather"); + CHECK(canonical["function"].contains("parameters")); + CHECK(!canonical["function"].contains("input_schema")); +} + TEST_CASE(SemanticToolHintFixture, native_predictor_config_is_independent_of_http) { SemanticToolPredictorConfig config; config.native_model_path = "/models/qwen3-0.6b.gguf"; @@ -209,31 +243,24 @@ TEST_CASE(SemanticToolHintFixture, native_predictor_config_is_independent_of_htt } TEST_CASE(SemanticToolHintFixture, native_prompt_honors_tool_choice_none) { - const json request = { - {"messages", json::array({{ - {"role", "user"}, - {"content", "Do not call a tool"}, - }})}, - {"tools", weather_tools()}, - {"tool_choice", "none"}, - }; std::string error; - CHECK(build_native_semantic_tool_predictor_prompt(request, error).empty()); + const auto request = build_semantic_tool_predictor_request( + json::array({{{"role", "user"}, {"content", "Do not call a tool"}}}), + weather_tools(), "none", "Qwen3-0.6B", 32, error); + REQUIRE(request.has_value()); + CHECK(build_native_semantic_tool_predictor_prompt(*request, error).empty()); CHECK(error == "native_predictor_tool_choice_none"); } TEST_CASE(SemanticToolHintFixture, native_prompt_uses_qwen_tool_contract) { - const json request = { - {"messages", json::array({{ - {"role", "user"}, - {"content", "What is the weather in Rome?"}, - }})}, - {"tools", weather_tools()}, - {"tool_choice", "required"}, - }; std::string error; + const auto request = build_semantic_tool_predictor_request( + json::array({{{"role", "user"}, + {"content", "What is the weather in Rome?"}}}), + weather_tools(), "required", "Qwen3-0.6B", 32, error); + REQUIRE(request.has_value()); const std::string prompt = - build_native_semantic_tool_predictor_prompt(request, error); + build_native_semantic_tool_predictor_prompt(*request, error); CHECK(error.empty()); CHECK(prompt.find("You must call exactly one available function.") != std::string::npos); @@ -247,40 +274,18 @@ TEST_CASE(SemanticToolHintFixture, native_prompt_uses_qwen_tool_contract) { } TEST_CASE(SemanticToolHintFixture, native_prompt_honors_preprocessing_deadline) { - const json request = { - {"messages", json::array({{{"role", "user"}, {"content", "weather"}}})}, - {"tools", weather_tools()}, - }; + std::string error; + const auto request = build_semantic_tool_predictor_request( + json::array({{{"role", "user"}, {"content", "weather"}}}), + weather_tools(), nullptr, "Qwen3-0.6B", 32, error); + REQUIRE(request.has_value()); const auto expired = std::chrono::steady_clock::now() - std::chrono::milliseconds(1); - std::string error; CHECK(build_native_semantic_tool_predictor_prompt( - request, error, &expired).empty()); + *request, error, &expired).empty()); CHECK(error == "native_predictor_timeout"); } -TEST_CASE(SemanticToolHintFixture, native_prompt_rejects_oversized_input_before_rendering) { - json request = { - {"messages", json::array({{{"role", "user"}, {"content", "weather"}}})}, - {"tools", weather_tools()}, - }; - const std::string oversized(300U * 1024U, 'x'); - const auto deadline = std::chrono::steady_clock::now() + - std::chrono::seconds(5); - std::string error; - - request["messages"][0]["content"] = oversized; - CHECK(build_native_semantic_tool_predictor_prompt( - request, error, &deadline).empty()); - CHECK(error == "native_predictor_request_too_large"); - - request["messages"][0]["content"] = "weather"; - request["tools"][0]["function"]["description"] = oversized; - CHECK(build_native_semantic_tool_predictor_prompt( - request, error, &deadline).empty()); - CHECK(error == "native_predictor_request_too_large"); -} - TEST_CASE(SemanticToolHintFixture, parses_native_qwen_xml_semantics) { const std::string generated = "\n" diff --git a/server/test/test_server_unit.cpp b/server/test/test_server_unit.cpp index 05118f2db..a66dfa128 100644 --- a/server/test/test_server_unit.cpp +++ b/server/test/test_server_unit.cpp @@ -2017,6 +2017,25 @@ TEST_CASE(ServerUnitFixture, test_tokenizer_added_token_search_obeys_deadline) { unlink(path.c_str()); } +TEST_CASE(ServerUnitFixture, test_tokenizer_rejects_oversized_added_token) { + gguf_context * fixture = gguf_init_empty(); + const std::string oversized(4097U, 'x'); + const char * tokens[] = {"x", oversized.c_str()}; + const uint32_t token_types[] = {1, 3}; + gguf_set_arr_str(fixture, "tokenizer.ggml.tokens", tokens, 2); + gguf_set_arr_data(fixture, "tokenizer.ggml.token_type", GGUF_TYPE_UINT32, + token_types, sizeof(token_types)); + gguf_set_val_str(fixture, "tokenizer.ggml.model", "gpt2"); + gguf_set_val_str(fixture, "tokenizer.ggml.pre", "qwen35"); + const std::string path = "/tmp/dflash_test_oversized_special_token.gguf"; + gguf_write_to_file(fixture, path.c_str(), /*only_meta=*/false); + gguf_free(fixture); + + Tokenizer tokenizer; + TEST_ASSERT(!tokenizer.load_from_gguf(path.c_str())); + unlink(path.c_str()); +} + TEST_CASE(ServerUnitFixture, test_hash_prefix_deterministic) { std::vector ids = {100, 200, 300, 400, 500}; auto h1 = hash_prefix(ids.data(), (int)ids.size()); @@ -2907,6 +2926,39 @@ TEST_CASE(ServerUnitFixture, test_normalize_responses_tool_followup_messages) { TEST_ASSERT(chat_msgs[3].tool_call_id == call_id); TEST_ASSERT(chat_msgs[3].content == "Process exited with code 0"); } + const json predictor_messages = + http_detail::canonical_predictor_messages(std::move(chat_msgs)); + TEST_ASSERT(predictor_messages.size() == 4); + TEST_ASSERT(predictor_messages[2]["role"] == "assistant"); + TEST_ASSERT(predictor_messages[2]["content"] == raw_tool_call); + TEST_ASSERT(predictor_messages[3]["role"] == "tool"); + TEST_ASSERT(predictor_messages[3]["content"] == + "Process exited with code 0"); + TEST_ASSERT(predictor_messages[3]["tool_call_id"] == call_id); + + const json predictor_tools = json::array({{ + {"type", "function"}, + {"function", { + {"name", "exec_command"}, + {"parameters", { + {"type", "object"}, + {"properties", {{"cmd", {{"type", "string"}}}}}, + }}, + }}, + }}); + std::string predictor_error; + const auto predictor_request = build_semantic_tool_predictor_request( + predictor_messages, predictor_tools, "required", "native-qwen3", 32, + predictor_error); + TEST_ASSERT(predictor_request.has_value()); + const std::string predictor_prompt = + build_native_semantic_tool_predictor_prompt( + *predictor_request, predictor_error); + TEST_ASSERT(predictor_error.empty()); + TEST_ASSERT(predictor_prompt.find(raw_tool_call) != std::string::npos); + TEST_ASSERT(predictor_prompt.find( + "\nProcess exited with code 0\n") != + std::string::npos); } // ═══════════════════════════════════════════════════════════════════════ From 55c9dd73ebddff563d833af6563f88d74d8be046 Mon Sep 17 00:00:00 2001 From: mrciffa <49000955+davide221@users.noreply.github.com> Date: Tue, 18 Aug 2026 20:36:41 +0200 Subject: [PATCH 10/11] fix(server): handle malformed predictor tools portably --- .../src/common/qwen3_tool_predictor_ipc.cpp | 4 +- server/src/server/semantic_tool_hint.cpp | 45 ++++++++++++------- server/test/test_semantic_tool_hint.cpp | 25 +++++++++++ 3 files changed, 56 insertions(+), 18 deletions(-) diff --git a/server/src/common/qwen3_tool_predictor_ipc.cpp b/server/src/common/qwen3_tool_predictor_ipc.cpp index 643b363bc..d9bb586af 100644 --- a/server/src/common/qwen3_tool_predictor_ipc.cpp +++ b/server/src/common/qwen3_tool_predictor_ipc.cpp @@ -75,11 +75,11 @@ bool Qwen3ToolPredictorIpcClient::start( int max_ctx, const std::string & work_dir, int readiness_timeout_ms) { -#if defined(_WIN32) +#if !defined(__linux__) (void)bin; (void)model_path; (void)gpu; (void)max_ctx; (void)work_dir; (void)readiness_timeout_ms; std::fprintf(stderr, - "Qwen3 tool-predictor IPC is only implemented on POSIX hosts\n"); + "Qwen3 tool-predictor IPC is only implemented on Linux hosts\n"); return false; #else std::lock_guard lock(mutex_); diff --git a/server/src/server/semantic_tool_hint.cpp b/server/src/server/semantic_tool_hint.cpp index d65dabdc1..43c9c7036 100644 --- a/server/src/server/semantic_tool_hint.cpp +++ b/server/src/server/semantic_tool_hint.cpp @@ -11,14 +11,21 @@ namespace dflash::common { namespace { +std::string string_member(const json & object, const char * key) { + if (!object.is_object()) return {}; + const auto member = object.find(key); + return member != object.end() && member->is_string() + ? member->get() : std::string{}; +} + bool request_has_function(const json & tools, const std::string & name) { if (!tools.is_array() || name.empty()) return false; for (const auto & tool : tools) { if (!tool.is_object()) continue; - if (tool.value("name", "") == name) return true; + if (string_member(tool, "name") == name) return true; const auto function = tool.find("function"); if (function != tool.end() && function->is_object() && - function->value("name", "") == name) { + string_member(*function, "name") == name) { return true; } } @@ -30,10 +37,10 @@ std::string sole_request_function(const json & tools) { std::string sole; for (const auto & tool : tools) { if (!tool.is_object()) continue; - std::string name = tool.value("name", ""); + std::string name = string_member(tool, "name"); const auto function = tool.find("function"); if (name.empty() && function != tool.end() && function->is_object()) { - name = function->value("name", ""); + name = string_member(*function, "name"); } if (name.empty()) continue; if (!sole.empty() && sole != name) return {}; @@ -59,8 +66,8 @@ bool parse_arguments(const json & value, ordered_json & out) { bool parse_call_object(const json & value, SemanticToolCall & out) { if (!value.is_object()) return false; - const std::string name = value.value( - "name", value.value("function", std::string{})); + std::string name = string_member(value, "name"); + if (name.empty()) name = string_member(value, "function"); if (name.empty()) return false; const json * arguments = nullptr; @@ -263,10 +270,10 @@ bool semantic_message_content( continue; } if (!part.is_object()) continue; - const std::string type = part.value("type", ""); + const std::string type = string_member(part, "type"); if (type == "text" || type == "input_text" || type == "output_text") { - text += part.value("text", ""); + text += string_member(part, "text"); } } return !semantic_deadline_expired(deadline); @@ -276,9 +283,9 @@ std::string forced_tool_name(const json & choice) { if (!choice.is_object()) return {}; const auto function = choice.find("function"); if (function != choice.end() && function->is_object()) { - return function->value("name", ""); + return string_member(*function, "name"); } - return choice.value("name", ""); + return string_member(choice, "name"); } json canonical_semantic_tools(const json & tools) { @@ -289,7 +296,7 @@ json canonical_semantic_tools(const json & tools) { const json * source = &tool; const auto wrapped = tool.find("function"); if (wrapped != tool.end() && wrapped->is_object()) source = &*wrapped; - const std::string name = source->value("name", ""); + const std::string name = string_member(*source, "name"); if (name.empty()) continue; json function = {{"name", name}}; @@ -385,7 +392,7 @@ bool materialize_declared_tool_defaults( const json & candidate = tool.contains("function") && tool["function"].is_object() ? tool["function"] : tool; - if (candidate.value("name", "") == call.name) { + if (string_member(candidate, "name") == call.name) { function = &candidate; break; } @@ -450,13 +457,18 @@ build_semantic_tool_predictor_request( error = "predictor_request_too_large"; return std::nullopt; } + json canonical_tools = canonical_semantic_tools(tools); + if (canonical_tools.empty()) { + error = "predictor_request_has_no_valid_tools"; + return std::nullopt; + } json request = { {"model", sidecar_model}, {"stream", false}, {"temperature", 0}, {"max_tokens", max_tokens}, {"messages", messages}, - {"tools", canonical_semantic_tools(tools)}, + {"tools", std::move(canonical_tools)}, {"tool_choice", effective_choice}, }; // Tool-schema normalization can add OpenAI wrapper objects. Validate the @@ -487,7 +499,7 @@ std::string build_native_semantic_tool_predictor_prompt( const json & predictor_request = bounded_request.payload(); const json choice = predictor_request.value("tool_choice", json("auto")); if ((choice.is_string() && choice.get() == "none") || - (choice.is_object() && choice.value("type", "") == "none")) { + (choice.is_object() && string_member(choice, "type") == "none")) { error = "native_predictor_tool_choice_none"; return {}; } @@ -516,7 +528,8 @@ std::string build_native_semantic_tool_predictor_prompt( return {}; } if (!message.is_object()) continue; - std::string role = message.value("role", "user"); + std::string role = string_member(message, "role"); + if (role.empty()) role = "user"; if (role == "developer") role = "system"; std::string content; if (!semantic_message_content(message, deadline, content)) { @@ -614,7 +627,7 @@ std::string build_native_semantic_tool_predictor_prompt( const json & call = raw_call.contains("function") && raw_call["function"].is_object() ? raw_call["function"] : raw_call; - const std::string name = call.value("name", ""); + const std::string name = string_member(call, "name"); if (name.empty() || !call.contains("arguments")) continue; if (!message.content.empty()) rendered += "\n"; rendered += "\n{\"name\": \"" + name + diff --git a/server/test/test_semantic_tool_hint.cpp b/server/test/test_semantic_tool_hint.cpp index 9552b3906..2ae4d25a6 100644 --- a/server/test/test_semantic_tool_hint.cpp +++ b/server/test/test_semantic_tool_hint.cpp @@ -233,6 +233,31 @@ TEST_CASE(SemanticToolHintFixture, predictor_request_canonicalizes_anthropic_too CHECK(!canonical["function"].contains("input_schema")); } +TEST_CASE(SemanticToolHintFixture, predictor_request_handles_non_string_tool_names) { + json tools = json::array({ + {{"type", "function"}, {"function", {{"name", 42}}}}, + {{"name", false}, {"input_schema", {{"type", "object"}}}}, + weather_tools()[0], + }); + std::string error; + const auto request = build_semantic_tool_predictor_request( + json::array({{{"role", "user"}, {"content", "weather"}}}), + tools, nullptr, "Qwen3-0.6B", 32, error); + REQUIRE(request.has_value()); + CHECK(error.empty()); + REQUIRE(request->payload()["tools"].size() == 1); + CHECK(request->payload()["tools"][0]["function"]["name"] == + "get_weather"); + + tools = json::array({ + {{"type", "function"}, {"function", {{"name", 42}}}}, + }); + CHECK(!build_semantic_tool_predictor_request( + json::array({{{"role", "user"}, {"content", "weather"}}}), + tools, nullptr, "Qwen3-0.6B", 32, error).has_value()); + CHECK(error == "predictor_request_has_no_valid_tools"); +} + TEST_CASE(SemanticToolHintFixture, native_predictor_config_is_independent_of_http) { SemanticToolPredictorConfig config; config.native_model_path = "/models/qwen3-0.6b.gguf"; From 4b46709bd4388863c0d6719aaad152fa6d7ed084 Mon Sep 17 00:00:00 2001 From: mrciffa <49000955+davide221@users.noreply.github.com> Date: Tue, 18 Aug 2026 20:42:25 +0200 Subject: [PATCH 11/11] fix(server): align native predictor platform guards --- server/src/common/qwen3_tool_predictor_ipc.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/common/qwen3_tool_predictor_ipc.cpp b/server/src/common/qwen3_tool_predictor_ipc.cpp index d9bb586af..421f82b9f 100644 --- a/server/src/common/qwen3_tool_predictor_ipc.cpp +++ b/server/src/common/qwen3_tool_predictor_ipc.cpp @@ -18,7 +18,7 @@ bool read_qwen3_tool_predictor_response( std::string & error) { output_ids.clear(); error.clear(); -#if defined(_WIN32) +#if !defined(__linux__) (void)stream_fd; (void)max_tokens; (void)timeout_ms; @@ -135,7 +135,7 @@ bool Qwen3ToolPredictorIpcClient::predict( std::string & error) { output_ids.clear(); error.clear(); -#if defined(_WIN32) +#if !defined(__linux__) (void)prompt_ids; (void)max_tokens; (void)timeout_ms; error = "native_predictor_ipc_unsupported"; return false;