From 13721a45fcd85eb4291605cff5d37fdb1e26f837 Mon Sep 17 00:00:00 2001 From: Rahul-Ganesan Date: Thu, 3 Sep 2026 19:17:52 -0400 Subject: [PATCH 1/8] traffic: replay real production traces and join the result back to the workload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `gitm/traffic/` — canonical request schema, two adapters, a regime tagger, replay through vLLM's native `bench serve`, and the join that ties a result back to the trace that produced it. No custom load generator. Replay goes through `--dataset-name timed_trace --self-timed`, which schedules every request at its own timestamp. The `[vllm]` extra floors at 0.23.0 because that dataset landed in bfb9ebc21 (2026-05-28), one day after v0.22.0 shipped — checked at the tags, not inferred from a changelog. Below the floor the run dies on an argparse complaint about an unknown dataset name, which reads like a typo in our command rather than a missing feature. Three real traces, pinned by sha256 through the existing gitm.bench.manifest. BurstGPT_1 383/400 rows, in p50/p95 353/1638, D=1.01. BurstGPT_3 399/400, 309/1497 — its `Session ID` and `Elapsed time` are inserted at positions 1 and 2, not appended, so a positional reader takes the session id as the model; columns are read by name. Mooncake 400/400, 9075/49904, D=6.74. The two axes that matter separate cleanly on real data, which is what makes them axes rather than decoration. Real data handling is enforced, not logged: 7 named drop reasons, each firing exactly once against a CSV and a JSONL dirty fixture, and a trace whose counts do not reconcile cannot be constructed. A caller's own filter counts apart from bad data and still reconciles. Fidelity is checked against the file vLLM will read, not an in-memory copy — every statistic exactly 0.0 on both adapters. Fired end to end against real `vllm bench serve` 0.28.0: 40/40 completed, paced to 12.008s against a 12.000s trace span, confirmed by two independent clocks. The failure this exists to catch: `--timed-trace-chunk-hash-size` defaults to 16 against Mooncake's 512-token blocks, making every prompt 32x short while completed, duration, throughput and every percentile still read perfectly. Block size verified over 1,546 rows rather than assumed. The emitter refuses to write such a file; the joiner catches one fired from a plan built elsewhere. Two result fields are wrong rather than missing: under `--self-timed` vLLM still records the CLI's `request_rate` ("inf", a string) and `burstiness` (1.0) against the trace's real 2.837 rps and D 6.74 — exactly the two axes a playbook row keys on. Dropped with a stated reason, values kept visible. `unjoined_keys()` fails on any new vLLM key neither kept nor deliberately dropped; it caught `rtfx` on its first run. Also `gitm/_banner.py` and its wiring into `gitm/cli.py` — a dependency of the new CLI. stderr, and only when stdout is a TTY: the gate asks about stdout because the question is whether a human is watching, and a banner on stdout surfaces as a JSON parse error in a CI step, nowhere near the banner. 24 selftest checks, the same assertions as 25 pytest cases, ruff clean. Knob and environment fields are left pending until the shared config-capture schema exists. --- .../fixtures/benchserve_result.json | 36 + .../fixtures/burstgpt3_dirty.csv | 7 + .../fixtures/burstgpt3_slice.csv | 401 ++++++++++ .../fixtures/burstgpt_dirty.csv | 10 + .../fixtures/burstgpt_slice.csv | 401 ++++++++++ .../fixtures/mooncake_dirty.jsonl | 9 + .../fixtures/mooncake_slice.jsonl | 400 ++++++++++ benchmarks/traffic_replay/manifest.yaml | 28 + benchmarks/traffic_replay/spec.md | 237 ++++++ gitm/_banner.py | 56 ++ gitm/cli.py | 6 + gitm/traffic/__init__.py | 73 ++ gitm/traffic/__main__.py | 147 ++++ gitm/traffic/_selftest.py | 682 ++++++++++++++++++ gitm/traffic/adapters.py | 407 +++++++++++ gitm/traffic/gui.py | 391 ++++++++++ gitm/traffic/parameterize.py | 216 ++++++ gitm/traffic/regime.py | 183 +++++ gitm/traffic/replay.py | 262 +++++++ gitm/traffic/results.py | 308 ++++++++ gitm/traffic/runner.py | 325 +++++++++ gitm/traffic/schema.py | 204 ++++++ gitm/traffic/validate.py | 275 +++++++ pyproject.toml | 21 +- tests/test_traffic.py | 30 + 25 files changed, 5113 insertions(+), 2 deletions(-) create mode 100644 benchmarks/traffic_replay/fixtures/benchserve_result.json create mode 100644 benchmarks/traffic_replay/fixtures/burstgpt3_dirty.csv create mode 100644 benchmarks/traffic_replay/fixtures/burstgpt3_slice.csv create mode 100644 benchmarks/traffic_replay/fixtures/burstgpt_dirty.csv create mode 100644 benchmarks/traffic_replay/fixtures/burstgpt_slice.csv create mode 100644 benchmarks/traffic_replay/fixtures/mooncake_dirty.jsonl create mode 100644 benchmarks/traffic_replay/fixtures/mooncake_slice.jsonl create mode 100644 benchmarks/traffic_replay/manifest.yaml create mode 100644 benchmarks/traffic_replay/spec.md create mode 100644 gitm/_banner.py create mode 100644 gitm/traffic/__init__.py create mode 100644 gitm/traffic/__main__.py create mode 100644 gitm/traffic/_selftest.py create mode 100644 gitm/traffic/adapters.py create mode 100644 gitm/traffic/gui.py create mode 100644 gitm/traffic/parameterize.py create mode 100644 gitm/traffic/regime.py create mode 100644 gitm/traffic/replay.py create mode 100644 gitm/traffic/results.py create mode 100644 gitm/traffic/runner.py create mode 100644 gitm/traffic/schema.py create mode 100644 gitm/traffic/validate.py create mode 100644 tests/test_traffic.py diff --git a/benchmarks/traffic_replay/fixtures/benchserve_result.json b/benchmarks/traffic_replay/fixtures/benchserve_result.json new file mode 100644 index 0000000..54ceb53 --- /dev/null +++ b/benchmarks/traffic_replay/fixtures/benchserve_result.json @@ -0,0 +1,36 @@ +{ + "date": "20260902-173307", + "endpoint_type": "openai", + "backend": "openai", + "label": null, + "model_id": "STUB", + "tokenizer_id": "gpt2", + "num_prompts": 40, + "request_rate": "inf", + "burstiness": 1.0, + "max_concurrency": null, + "duration": 12.00791824299995, + "completed": 40, + "failed": 0, + "total_input_tokens": 506280, + "total_output_tokens": 40, + "request_throughput": 3.3311352717876903, + "request_goodput": null, + "output_throughput": 3.3311352717876903, + "total_token_throughput": 42165.510270288585, + "max_output_tokens_per_s": 108.0, + "max_concurrent_requests": 16, + "rtfx": 0.0, + "mean_ttft_ms": 192.18192199998612, + "median_ttft_ms": 13.159716000018307, + "std_ttft_ms": 388.61726254662835, + "p99_ttft_ms": 1040.9596590099, + "mean_tpot_ms": 0.0, + "median_tpot_ms": 0.0, + "std_tpot_ms": 0.0, + "p99_tpot_ms": 0.0, + "mean_itl_ms": 0.14320468181949764, + "median_itl_ms": 0.004474500087781053, + "std_itl_ms": 0.3848804774613307, + "p99_itl_ms": 1.9144619700205112 +} diff --git a/benchmarks/traffic_replay/fixtures/burstgpt3_dirty.csv b/benchmarks/traffic_replay/fixtures/burstgpt3_dirty.csv new file mode 100644 index 0000000..83bfba1 --- /dev/null +++ b/benchmarks/traffic_replay/fixtures/burstgpt3_dirty.csv @@ -0,0 +1,7 @@ +Timestamp,Session ID,Elapsed time,Model,Request tokens,Response tokens,Total tokens,Log Type +10,3f2a7c11-0000-4000-8000-000000000001,5,ChatGPT,100,50,150,Conversation log +20,,3,ChatGPT,200,60,260,API log +30,3f2a7c11-0000-4000-8000-000000000001,zzz,ChatGPT,120,40,160,Conversation log +40,3f2a7c11-0000-4000-8000-000000000002,7,ChatGPT,0,0,0,Conversation log +50,3f2a7c11-0000-4000-8000-000000000002 +35,3f2a7c11-0000-4000-8000-000000000003,2,ChatGPT,80,20,100,Conversation log diff --git a/benchmarks/traffic_replay/fixtures/burstgpt3_slice.csv b/benchmarks/traffic_replay/fixtures/burstgpt3_slice.csv new file mode 100644 index 0000000..2a2c222 --- /dev/null +++ b/benchmarks/traffic_replay/fixtures/burstgpt3_slice.csv @@ -0,0 +1,401 @@ +Timestamp,Session ID,Elapsed time,Model,Request tokens,Response tokens,Total tokens,Log Type +19440110.0,1722ac82-0a46-4bf0-aa08-89794e7a2b3f,43,GPT-4,906,446,1352,Conversation log +19440161.0,d5983bf9-4b48-497b-892b-a58995247443,2,ChatGPT,36,29,65,Conversation log +19440192.0,a5c69b35-4fbc-45e9-955e-18715e376d74,8,GPT-4,1779,123,1902,Conversation log +19440254.0,8b74c7d8-1643-4bba-8c69-91cb4548c506,3,ChatGPT,935,178,1113,Conversation log +19440301.0,1722ac82-0a46-4bf0-aa08-89794e7a2b3f,26,GPT-4,1631,282,1913,Conversation log +19440371.0,c6f7a15f-4716-4671-b66f-720888a22e68,2,ChatGPT,728,126,854,Conversation log +19440529.0,8b74c7d8-1643-4bba-8c69-91cb4548c506,3,ChatGPT,1136,175,1311,Conversation log +19440537.0,8219e6e1-b261-4978-bf78-0e8703a5fcc1,3,ChatGPT,15,118,133,Conversation log +19440574.0,8219e6e1-b261-4978-bf78-0e8703a5fcc1,2,ChatGPT,147,125,272,Conversation log +19440640.0,8219e6e1-b261-4978-bf78-0e8703a5fcc1,3,ChatGPT,291,163,454,Conversation log +19440655.0,8b74c7d8-1643-4bba-8c69-91cb4548c506,3,ChatGPT,1325,152,1477,Conversation log +19440692.0,8b74c7d8-1643-4bba-8c69-91cb4548c506,2,ChatGPT,1221,185,1406,Conversation log +19440735.0,4c5aaf07-bc43-45eb-a3c3-2c77f12f9ee7,3,ChatGPT,18,141,159,Conversation log +19440845.0,de1987f7-d609-4a7c-a4e4-0deeb4ac66b0,3,ChatGPT,21,103,124,Conversation log +19440980.0,de1987f7-d609-4a7c-a4e4-0deeb4ac66b0,3,ChatGPT,137,152,289,Conversation log +19441216.0,9cfed687-bda4-455b-a400-837e08fce913,2,ChatGPT,42,63,105,Conversation log +19441238.0,7117fb3a-4b85-400b-a429-45adad4fce56,2,ChatGPT,66,7,73,Conversation log +19441263.0,7117fb3a-4b85-400b-a429-45adad4fce56,0,ChatGPT,93,6,99,Conversation log +19441654.0,216d88ba-f25e-47d1-a4b0-b9ee6dda5a58,4,ChatGPT,29,339,368,Conversation log +19441735.0,8c89ec7b-cb6f-4121-9704-2837255d9744,25,GPT-4,37,472,509,Conversation log +19442826.0,56e38018-bb73-4f4f-b233-57edbf97d91d,26,GPT-4,5299,541,5840,Conversation log +19443125.0,977bfbdf-a273-469f-a4be-762f5047c843,3,ChatGPT,16,109,125,Conversation log +19443156.0,977bfbdf-a273-469f-a4be-762f5047c843,2,ChatGPT,157,277,434,Conversation log +19443224.0,aa037ed8-0ee9-4567-af72-25c0fcc67d96,9,GPT-4,12,87,99,Conversation log +19443267.0,aa037ed8-0ee9-4567-af72-25c0fcc67d96,44,GPT-4,151,617,768,Conversation log +19443443.0,aa037ed8-0ee9-4567-af72-25c0fcc67d96,40,GPT-4,812,534,1346,Conversation log +19443733.0,aa037ed8-0ee9-4567-af72-25c0fcc67d96,10,GPT-4,1431,98,1529,Conversation log +19443781.0,aa037ed8-0ee9-4567-af72-25c0fcc67d96,6,GPT-4,1609,79,1688,Conversation log +19443876.0,56e38018-bb73-4f4f-b233-57edbf97d91d,18,GPT-4,5866,359,6225,Conversation log +19443878.0,c404e0bd-29b4-4b8e-b61b-02a47cea83a1,5,ChatGPT,42,340,382,Conversation log +19444826.0,a5722132-b491-41c2-9fda-c0e6e0ad009b,23,GPT-4,19,410,429,Conversation log +19445110.0,0c601e0d-7cd3-40a0-8c76-316913fd02d9,25,GPT-4,984,377,1361,Conversation log +19445406.0,0c601e0d-7cd3-40a0-8c76-316913fd02d9,30,GPT-4,1460,259,1719,Conversation log +19447288.0,712de7cf-7c1c-444b-bf68-33408132b420,70,GPT-4,25,879,904,Conversation log +19447641.0,56e38018-bb73-4f4f-b233-57edbf97d91d,25,GPT-4,5805,403,6208,Conversation log +19448735.0,712de7cf-7c1c-444b-bf68-33408132b420,94,GPT-4,930,787,1717,Conversation log +19448801.0,072315f3-c0d9-4be5-94f3-7767861ff34c,111,GPT-4,25,837,862,Conversation log +19448828.0,a3b80941-f129-4f60-a709-aa9ed5d8c7c2,47,GPT-4,31,477,508,Conversation log +19448842.0,38d2f7f8-3384-40cc-a77a-e0a0f7ee80bc,123,GPT-4,35,862,897,Conversation log +19448853.0,f011f24f-692d-4555-bf84-f1caa1f7db50,101,GPT-4,30,711,741,Conversation log +19448987.0,f011f24f-692d-4555-bf84-f1caa1f7db50,4,ChatGPT,772,394,1166,Conversation log +19449072.0,f011f24f-692d-4555-bf84-f1caa1f7db50,65,GPT-4,1197,406,1603,Conversation log +19450093.0,bda1996c-a6b2-43e4-b9bc-cf02ebb47f78,2,ChatGPT,36,34,70,Conversation log +19450216.0,bda1996c-a6b2-43e4-b9bc-cf02ebb47f78,2,ChatGPT,104,26,130,Conversation log +19450676.0,bda1996c-a6b2-43e4-b9bc-cf02ebb47f78,3,ChatGPT,173,33,206,Conversation log +19450738.0,bda1996c-a6b2-43e4-b9bc-cf02ebb47f78,0,ChatGPT,252,37,289,Conversation log +19450780.0,bda1996c-a6b2-43e4-b9bc-cf02ebb47f78,1,ChatGPT,328,28,356,Conversation log +19450849.0,bda1996c-a6b2-43e4-b9bc-cf02ebb47f78,0,ChatGPT,309,15,324,Conversation log +19451325.0,f6afa413-2768-41d4-8377-f1b5ff823b4f,47,GPT-4,38,537,575,Conversation log +19451346.0,0931bfd2-a988-418e-9f84-5e9e13c0bc8e,67,GPT-4,27,645,672,Conversation log +19452654.0,bda1996c-a6b2-43e4-b9bc-cf02ebb47f78,2,ChatGPT,364,76,440,Conversation log +19452763.0,bda1996c-a6b2-43e4-b9bc-cf02ebb47f78,2,ChatGPT,388,9,397,Conversation log +19453134.0,bda1996c-a6b2-43e4-b9bc-cf02ebb47f78,2,ChatGPT,384,61,445,Conversation log +19453146.0,313b905a-2252-45ca-bf14-46e87f6233dd,5,ChatGPT,69,62,131,Conversation log +19454781.0,,21,GPT-4,264,171,435,API log +19454805.0,,38,GPT-4,265,245,510,API log +19454846.0,,14,GPT-4,283,298,581,API log +19456925.0,313b905a-2252-45ca-bf14-46e87f6233dd,3,ChatGPT,249,98,347,Conversation log +19457291.0,313b905a-2252-45ca-bf14-46e87f6233dd,4,ChatGPT,400,45,445,Conversation log +19457368.0,d0c6aa1e-c64e-4919-9395-12c14c2a71b8,2,ChatGPT,247,50,297,Conversation log +19457398.0,d0c6aa1e-c64e-4919-9395-12c14c2a71b8,2,ChatGPT,560,112,672,Conversation log +19457404.0,313b905a-2252-45ca-bf14-46e87f6233dd,3,ChatGPT,498,43,541,Conversation log +19457413.0,d0c6aa1e-c64e-4919-9395-12c14c2a71b8,3,ChatGPT,684,162,846,Conversation log +19457592.0,313b905a-2252-45ca-bf14-46e87f6233dd,3,ChatGPT,707,155,862,Conversation log +19457808.0,d0c6aa1e-c64e-4919-9395-12c14c2a71b8,4,ChatGPT,1240,124,1364,Conversation log +19457830.0,d0c6aa1e-c64e-4919-9395-12c14c2a71b8,1,ChatGPT,1378,91,1469,Conversation log +19460722.0,313b905a-2252-45ca-bf14-46e87f6233dd,2,ChatGPT,763,23,786,Conversation log +19460763.0,313b905a-2252-45ca-bf14-46e87f6233dd,1,ChatGPT,602,23,625,Conversation log +19460840.0,313b905a-2252-45ca-bf14-46e87f6233dd,0,ChatGPT,560,23,583,Conversation log +19460854.0,313b905a-2252-45ca-bf14-46e87f6233dd,3,GPT-4,520,23,543,Conversation log +19461009.0,313b905a-2252-45ca-bf14-46e87f6233dd,5,GPT-4,273,37,310,Conversation log +19461240.0,313b905a-2252-45ca-bf14-46e87f6233dd,3,GPT-4,289,28,317,Conversation log +19461337.0,313b905a-2252-45ca-bf14-46e87f6233dd,3,GPT-4,306,41,347,Conversation log +19461408.0,313b905a-2252-45ca-bf14-46e87f6233dd,4,GPT-4,316,15,331,Conversation log +19461651.0,70d0486e-d937-479b-ab8f-51b9b0e944a8,2,ChatGPT,82,57,139,Conversation log +19461670.0,70d0486e-d937-479b-ab8f-51b9b0e944a8,5,GPT-4,222,60,282,Conversation log +19461704.0,313b905a-2252-45ca-bf14-46e87f6233dd,14,GPT-4,354,69,423,Conversation log +19461751.0,313b905a-2252-45ca-bf14-46e87f6233dd,6,GPT-4,345,78,423,Conversation log +19461797.0,313b905a-2252-45ca-bf14-46e87f6233dd,3,ChatGPT,370,69,439,Conversation log +19461919.0,313b905a-2252-45ca-bf14-46e87f6233dd,2,ChatGPT,439,75,514,Conversation log +19461995.0,313b905a-2252-45ca-bf14-46e87f6233dd,11,GPT-4,484,82,566,Conversation log +19462283.0,313b905a-2252-45ca-bf14-46e87f6233dd,9,GPT-4,471,55,526,Conversation log +19462532.0,313b905a-2252-45ca-bf14-46e87f6233dd,11,GPT-4,495,60,555,Conversation log +19462658.0,313b905a-2252-45ca-bf14-46e87f6233dd,8,GPT-4,527,50,577,Conversation log +19462739.0,313b905a-2252-45ca-bf14-46e87f6233dd,11,GPT-4,427,61,488,Conversation log +19462794.0,313b905a-2252-45ca-bf14-46e87f6233dd,4,GPT-4,452,59,511,Conversation log +19462830.0,,2,ChatGPT,11,77,88,API log +19462868.0,313b905a-2252-45ca-bf14-46e87f6233dd,2,GPT-4,432,24,456,Conversation log +19462991.0,,2,ChatGPT,10,51,61,API log +19463035.0,,1,ChatGPT,11,112,123,API log +19463117.0,c29c0b68-41af-4e5d-bd23-182d153c7026,1,ChatGPT,129,38,167,Conversation log +19463150.0,c29c0b68-41af-4e5d-bd23-182d153c7026,1,ChatGPT,271,89,360,Conversation log +19463175.0,c29c0b68-41af-4e5d-bd23-182d153c7026,0,ChatGPT,376,34,410,Conversation log +19463219.0,c29c0b68-41af-4e5d-bd23-182d153c7026,3,GPT-4,422,51,473,Conversation log +19464436.0,70d0486e-d937-479b-ab8f-51b9b0e944a8,23,GPT-4,498,243,741,Conversation log +19465371.0,70d0486e-d937-479b-ab8f-51b9b0e944a8,14,GPT-4,819,84,903,Conversation log +19465879.0,70d0486e-d937-479b-ab8f-51b9b0e944a8,13,GPT-4,985,88,1073,Conversation log +19466031.0,70d0486e-d937-479b-ab8f-51b9b0e944a8,57,GPT-4,12149,501,12650,Conversation log +19466436.0,70d0486e-d937-479b-ab8f-51b9b0e944a8,101,GPT-4,12570,551,13121,Conversation log +19466694.0,ba445a59-328f-4f81-bc96-b33024b6f2c9,6,GPT-4,192,30,222,Conversation log +19469449.0,70d0486e-d937-479b-ab8f-51b9b0e944a8,52,GPT-4,13103,499,13602,Conversation log +19469801.0,70d0486e-d937-479b-ab8f-51b9b0e944a8,44,GPT-4,25085,288,25373,Conversation log +19470447.0,35cc1b41-1b59-41f0-a2fa-fcec2b634299,4,ChatGPT,439,234,673,Conversation log +19470521.0,5b5b9d0f-6525-4a12-9815-62822f9225a5,2,ChatGPT,69,318,387,Conversation log +19470605.0,5b5b9d0f-6525-4a12-9815-62822f9225a5,3,ChatGPT,504,326,830,Conversation log +19471220.0,e97a0e91-cb95-41ed-a57c-ec06bacfe694,3,ChatGPT,61,56,117,Conversation log +19471695.0,a0a1380b-0e3e-4977-a689-711747ef8a25,50,GPT-4,57,365,422,Conversation log +19471856.0,a0a1380b-0e3e-4977-a689-711747ef8a25,29,GPT-4,461,207,668,Conversation log +19471919.0,c29c0b68-41af-4e5d-bd23-182d153c7026,4,GPT-4,495,86,581,Conversation log +19472459.0,7e7d6f12-3649-4467-8624-c19a2e58aa9d,2,ChatGPT,577,45,622,Conversation log +19472810.0,e365d716-958f-444d-b446-f50b4d1cdf93,3,ChatGPT,198,282,480,Conversation log +19472958.0,10218b75-e89a-4787-84fe-53d7a1058556,5,ChatGPT,22,311,333,Conversation log +19473110.0,8555dc14-21ad-401d-b33c-816cbb086de5,1,ChatGPT,41,18,59,Conversation log +19473192.0,49d76cf0-2d00-4308-a798-3970f3b93f4f,2,GPT-4,38,15,53,Conversation log +19473219.0,7afcfc41-af8f-49cb-8cda-86acbdbb4ab6,2,ChatGPT,167,143,310,Conversation log +19473253.0,5433ddce-0bb7-4ffc-ab98-14fa1aaec544,0,ChatGPT,0,0,0,Conversation log +19473501.0,e70a0be6-879e-4079-a006-3584f43f23a1,2,ChatGPT,101,101,202,Conversation log +19473786.0,7e7d6f12-3649-4467-8624-c19a2e58aa9d,4,ChatGPT,599,107,706,Conversation log +19473831.0,7599245b-e0bf-49ad-b5d3-77148ccc9bc4,6,ChatGPT,363,536,899,Conversation log +19474039.0,bb0e0d1b-ca5d-49a3-999c-2cb33e7d5b20,6,GPT-4,64,59,123,Conversation log +19474118.0,96056b5c-31d5-4c74-9a3b-9c9b9050f87d,3,ChatGPT,58,142,200,Conversation log +19474155.0,2e0a0c9b-32a0-4ba8-b52f-6779cf9b9ad2,1,ChatGPT,40,29,69,Conversation log +19474230.0,b0410482-18b7-40b6-923e-b878361f2802,2,ChatGPT,20,138,158,Conversation log +19474307.0,b0410482-18b7-40b6-923e-b878361f2802,6,ChatGPT,178,296,474,Conversation log +19474352.0,b0410482-18b7-40b6-923e-b878361f2802,1,ChatGPT,500,156,656,Conversation log +19474363.0,8d3d977a-8acc-431e-a304-646f7ac6b561,2,ChatGPT,61,36,97,Conversation log +19474368.0,b0410482-18b7-40b6-923e-b878361f2802,0,ChatGPT,676,33,709,Conversation log +19474383.0,b0410482-18b7-40b6-923e-b878361f2802,4,ChatGPT,724,342,1066,Conversation log +19474413.0,2e0a0c9b-32a0-4ba8-b52f-6779cf9b9ad2,0,ChatGPT,96,14,110,Conversation log +19474481.0,2e0a0c9b-32a0-4ba8-b52f-6779cf9b9ad2,1,ChatGPT,142,18,160,Conversation log +19474499.0,2e0a0c9b-32a0-4ba8-b52f-6779cf9b9ad2,1,ChatGPT,178,4,182,Conversation log +19474507.0,2e0a0c9b-32a0-4ba8-b52f-6779cf9b9ad2,0,ChatGPT,206,7,213,Conversation log +19474576.0,2e0a0c9b-32a0-4ba8-b52f-6779cf9b9ad2,2,ChatGPT,190,29,219,Conversation log +19474622.0,859aa400-b904-4596-a781-1629f37437d7,1,ChatGPT,49,45,94,Conversation log +19474629.0,95ed3640-36ab-4253-9fad-4afde1b9d00a,145,GPT-4,829,843,1672,Conversation log +19474635.0,23ca3716-d01f-4440-9290-ba9c06c4e4fe,3,ChatGPT,15,101,116,Conversation log +19474640.0,23ca3716-d01f-4440-9290-ba9c06c4e4fe,3,ChatGPT,126,192,318,Conversation log +19474670.0,2e0a0c9b-32a0-4ba8-b52f-6779cf9b9ad2,3,ChatGPT,308,126,434,Conversation log +19474711.0,e1cd025b-2c39-465c-9317-50b5e9325ece,5,ChatGPT,17,420,437,Conversation log +19474721.0,e1cd025b-2c39-465c-9317-50b5e9325ece,5,ChatGPT,456,413,869,Conversation log +19474783.0,23ca3716-d01f-4440-9290-ba9c06c4e4fe,3,ChatGPT,349,189,538,Conversation log +19474836.0,859aa400-b904-4596-a781-1629f37437d7,2,ChatGPT,137,27,164,Conversation log +19474918.0,88941ff8-fdf7-4599-b5f8-fcee623ef50f,4,ChatGPT,30,186,216,Conversation log +19474920.0,7afcfc41-af8f-49cb-8cda-86acbdbb4ab6,3,ChatGPT,596,271,867,Conversation log +19474942.0,88941ff8-fdf7-4599-b5f8-fcee623ef50f,1,ChatGPT,247,120,367,Conversation log +19474983.0,88941ff8-fdf7-4599-b5f8-fcee623ef50f,2,ChatGPT,399,165,564,Conversation log +19474997.0,9286be0f-c848-46af-bcf1-f2ce2c1916b1,2,ChatGPT,30,19,49,Conversation log +19475017.0,88941ff8-fdf7-4599-b5f8-fcee623ef50f,2,ChatGPT,591,185,776,Conversation log +19475019.0,7e7d6f12-3649-4467-8624-c19a2e58aa9d,2,ChatGPT,610,115,725,Conversation log +19475022.0,b29284f7-349c-4002-9c4d-e40f11d1b634,0,ChatGPT,13,3,16,Conversation log +19475062.0,1a311690-1fd2-449d-9833-dbc9444275f0,0,ChatGPT,53,57,110,Conversation log +19475129.0,4ee90b9f-41f6-474b-908c-f4204b50d952,23,GPT-4,1417,515,1932,Conversation log +19475163.0,b3e86cba-e38d-4bb0-a797-e1a270aa6772,1,ChatGPT,30,7,37,Conversation log +19475174.0,00ecf1a2-1bc8-4078-9480-11cce1395f72,2,ChatGPT,24,161,185,Conversation log +19475218.0,6dce0737-5e96-4efe-81cd-7342ee46de64,2,ChatGPT,91,81,172,Conversation log +19475254.0,021a5e07-4d20-4615-9855-229182369c8f,2,ChatGPT,171,153,324,Conversation log +19475307.0,b3e86cba-e38d-4bb0-a797-e1a270aa6772,1,ChatGPT,69,9,78,Conversation log +19475357.0,6338841e-ad62-4ef0-b897-51246061ea19,8,ChatGPT,22,741,763,Conversation log +19475409.0,6dce0737-5e96-4efe-81cd-7342ee46de64,0,ChatGPT,248,29,277,Conversation log +19475452.0,c29c0b68-41af-4e5d-bd23-182d153c7026,8,GPT-4,439,167,606,Conversation log +19475456.0,fa1adb6b-a9a3-49ef-b87b-e13d7d6d1c9a,2,ChatGPT,75,88,163,Conversation log +19475457.0,90d7608a-6f5e-4d49-bd87-06a323bad8bf,0,ChatGPT,93,26,119,Conversation log +19475473.0,3650ec27-6d6a-4492-923b-f8c5b8625b35,0,ChatGPT,42,69,111,Conversation log +19475474.0,3639b4da-8871-4b98-b0cb-365f227d68a3,3,ChatGPT,52,194,246,Conversation log +19475481.0,6dce0737-5e96-4efe-81cd-7342ee46de64,0,ChatGPT,295,2,297,Conversation log +19475490.0,6dce0737-5e96-4efe-81cd-7342ee46de64,0,ChatGPT,314,3,317,Conversation log +19475521.0,c29c0b68-41af-4e5d-bd23-182d153c7026,2,GPT-4,434,58,492,Conversation log +19475609.0,021a5e07-4d20-4615-9855-229182369c8f,4,ChatGPT,563,220,783,Conversation log +19475685.0,e48d6c1f-787e-4d42-ab0d-f095302946aa,0,ChatGPT,49,39,88,Conversation log +19475689.0,021a5e07-4d20-4615-9855-229182369c8f,4,ChatGPT,834,402,1236,Conversation log +19475708.0,e70a0be6-879e-4079-a006-3584f43f23a1,2,ChatGPT,279,104,383,Conversation log +19475729.0,fa1adb6b-a9a3-49ef-b87b-e13d7d6d1c9a,2,ChatGPT,184,93,277,Conversation log +19475773.0,e48d6c1f-787e-4d42-ab0d-f095302946aa,2,ChatGPT,155,59,214,Conversation log +19475796.0,fa1adb6b-a9a3-49ef-b87b-e13d7d6d1c9a,0,ChatGPT,286,5,291,Conversation log +19475839.0,fa1adb6b-a9a3-49ef-b87b-e13d7d6d1c9a,1,ChatGPT,312,10,322,Conversation log +19475859.0,6dce0737-5e96-4efe-81cd-7342ee46de64,2,ChatGPT,352,31,383,Conversation log +19475906.0,b3e86cba-e38d-4bb0-a797-e1a270aa6772,2,ChatGPT,192,104,296,Conversation log +19475917.0,3650ec27-6d6a-4492-923b-f8c5b8625b35,5,ChatGPT,164,213,377,Conversation log +19475993.0,6338841e-ad62-4ef0-b897-51246061ea19,12,ChatGPT,788,1101,1889,Conversation log +19476074.0,926c155f-6cbc-4404-8848-55b7a859a54e,5,ChatGPT,22,5,27,Conversation log +19476077.0,021a5e07-4d20-4615-9855-229182369c8f,4,ChatGPT,1336,78,1414,Conversation log +19476106.0,b29284f7-349c-4002-9c4d-e40f11d1b634,3,ChatGPT,65,38,103,Conversation log +19476180.0,3dbd1af9-4e46-413d-b63c-ae52c55ec66e,155,GPT-4,22,757,779,Conversation log +19476193.0,1e3e8e87-3c87-4600-8ea8-794aabf3d198,3,ChatGPT,25,212,237,Conversation log +19476213.0,95ed3640-36ab-4253-9fad-4afde1b9d00a,108,GPT-4,2457,558,3015,Conversation log +19476216.0,9286be0f-c848-46af-bcf1-f2ce2c1916b1,6,ChatGPT,108,46,154,Conversation log +19476296.0,021a5e07-4d20-4615-9855-229182369c8f,4,ChatGPT,1515,84,1599,Conversation log +19476299.0,00ecf1a2-1bc8-4078-9480-11cce1395f72,2,ChatGPT,207,144,351,Conversation log +19476304.0,0f284f8b-add5-493e-8fda-a57620a580ee,4,ChatGPT,75,69,144,Conversation log +19476323.0,926c155f-6cbc-4404-8848-55b7a859a54e,5,ChatGPT,160,106,266,Conversation log +19476327.0,0f284f8b-add5-493e-8fda-a57620a580ee,1,ChatGPT,182,67,249,Conversation log +19476332.0,a76bb823-3f64-4b7a-8a15-f4eb1253ad7f,3,ChatGPT,471,46,517,Conversation log +19476342.0,0f284f8b-add5-493e-8fda-a57620a580ee,1,ChatGPT,269,44,313,Conversation log +19476366.0,0f284f8b-add5-493e-8fda-a57620a580ee,8,GPT-4,328,80,408,Conversation log +19476371.0,a76bb823-3f64-4b7a-8a15-f4eb1253ad7f,1,ChatGPT,482,26,508,Conversation log +19476481.0,e70a0be6-879e-4079-a006-3584f43f23a1,3,ChatGPT,516,140,656,Conversation log +19476485.0,021a5e07-4d20-4615-9855-229182369c8f,3,ChatGPT,1499,191,1690,Conversation log +19476490.0,00ecf1a2-1bc8-4078-9480-11cce1395f72,4,ChatGPT,370,216,586,Conversation log +19476519.0,90d7608a-6f5e-4d49-bd87-06a323bad8bf,1,ChatGPT,143,15,158,Conversation log +19476557.0,2f9dc6e6-62f8-47ed-bf3c-b55ea0219207,3,ChatGPT,14,221,235,Conversation log +19476581.0,90d7608a-6f5e-4d49-bd87-06a323bad8bf,0,ChatGPT,177,8,185,Conversation log +19476611.0,e7ab5f81-cbef-4543-94fc-226946f1fe0a,58,GPT-4,35,389,424,Conversation log +19476629.0,0a9088f6-3cd2-48fb-85df-d9b5525b812d,2,ChatGPT,116,111,227,Conversation log +19476631.0,6552716e-d54c-485e-9edc-ff9a4baf3273,1,ChatGPT,40,20,60,Conversation log +19476652.0,6552716e-d54c-485e-9edc-ff9a4baf3273,1,ChatGPT,97,22,119,Conversation log +19476659.0,84b8a44a-4b64-4ab6-b784-01a53f7c1126,18,GPT-4,417,48,465,Conversation log +19476680.0,1e3e8e87-3c87-4600-8ea8-794aabf3d198,2,ChatGPT,256,178,434,Conversation log +19476680.0,2f9dc6e6-62f8-47ed-bf3c-b55ea0219207,3,ChatGPT,257,288,545,Conversation log +19476752.0,021a5e07-4d20-4615-9855-229182369c8f,3,ChatGPT,1404,133,1537,Conversation log +19476766.0,021a5e07-4d20-4615-9855-229182369c8f,2,ChatGPT,1094,191,1285,Conversation log +19476789.0,6552716e-d54c-485e-9edc-ff9a4baf3273,2,ChatGPT,161,31,192,Conversation log +19476826.0,021a5e07-4d20-4615-9855-229182369c8f,1,ChatGPT,1134,136,1270,Conversation log +19476858.0,3f9670a6-3e3d-44b4-b3f0-ac1742d3843e,1,ChatGPT,33,34,67,Conversation log +19476868.0,3f9670a6-3e3d-44b4-b3f0-ac1742d3843e,3,ChatGPT,159,113,272,Conversation log +19476897.0,2f9dc6e6-62f8-47ed-bf3c-b55ea0219207,3,ChatGPT,571,113,684,Conversation log +19476930.0,2f9dc6e6-62f8-47ed-bf3c-b55ea0219207,2,ChatGPT,718,140,858,Conversation log +19476960.0,68201869-131c-4de7-b428-7cc34ca426cf,1,ChatGPT,67,55,122,Conversation log +19476997.0,2f9dc6e6-62f8-47ed-bf3c-b55ea0219207,1,ChatGPT,885,15,900,Conversation log +19477082.0,9286be0f-c848-46af-bcf1-f2ce2c1916b1,6,ChatGPT,276,114,390,Conversation log +19477093.0,6552716e-d54c-485e-9edc-ff9a4baf3273,2,ChatGPT,308,103,411,Conversation log +19477103.0,68201869-131c-4de7-b428-7cc34ca426cf,4,ChatGPT,196,35,231,Conversation log +19477153.0,6552716e-d54c-485e-9edc-ff9a4baf3273,2,ChatGPT,533,160,693,Conversation log +19477185.0,1ad91820-0b5f-437b-8e5f-53d86ba39fab,5,ChatGPT,715,435,1150,Conversation log +19477185.0,1a311690-1fd2-449d-9833-dbc9444275f0,3,ChatGPT,216,91,307,Conversation log +19477188.0,210705c1-e47f-42fe-80d8-239f78163ab0,60,GPT-4,15,524,539,Conversation log +19477195.0,0f9ad46b-9f44-4b06-bbb8-1c0fc44591a1,2,ChatGPT,63,36,99,Conversation log +19477207.0,a88af531-4376-4743-9a88-dc78f8b6b901,102,GPT-4,24,760,784,Conversation log +19477223.0,68201869-131c-4de7-b428-7cc34ca426cf,3,ChatGPT,298,47,345,Conversation log +19477281.0,68201869-131c-4de7-b428-7cc34ca426cf,2,ChatGPT,447,86,533,Conversation log +19477334.0,80676dea-b9d2-4347-ae11-fb3008926e06,3,ChatGPT,24,143,167,Conversation log +19477338.0,68201869-131c-4de7-b428-7cc34ca426cf,2,ChatGPT,674,134,808,Conversation log +19477358.0,80676dea-b9d2-4347-ae11-fb3008926e06,1,ChatGPT,180,55,235,Conversation log +19477402.0,701b2d51-cd8d-49e5-a5d8-ff3598cb49c7,4,ChatGPT,163,201,364,Conversation log +19477430.0,f93dadc5-1806-46e3-90f3-c53a79e64fc5,2,ChatGPT,86,56,142,Conversation log +19477458.0,1ad91820-0b5f-437b-8e5f-53d86ba39fab,6,ChatGPT,1272,292,1564,Conversation log +19477574.0,9286be0f-c848-46af-bcf1-f2ce2c1916b1,4,ChatGPT,675,277,952,Conversation log +19477599.0,9286be0f-c848-46af-bcf1-f2ce2c1916b1,5,ChatGPT,1241,235,1476,Conversation log +19477626.0,0f284f8b-add5-493e-8fda-a57620a580ee,30,GPT-4,432,15,447,Conversation log +19477736.0,bd73fbc3-25f5-4e8a-8efa-a63bfbdd7cf3,7,ChatGPT,573,635,1208,Conversation log +19477766.0,80676dea-b9d2-4347-ae11-fb3008926e06,1,ChatGPT,251,146,397,Conversation log +19477772.0,efe52725-b30d-4de8-8c73-56ad121666f0,2,ChatGPT,132,192,324,Conversation log +19477788.0,80676dea-b9d2-4347-ae11-fb3008926e06,3,ChatGPT,421,146,567,Conversation log +19477817.0,181e8a96-674e-44cd-b54d-78dea1b504c7,9,ChatGPT,282,258,540,Conversation log +19477869.0,efe52725-b30d-4de8-8c73-56ad121666f0,0,ChatGPT,348,13,361,Conversation log +19477881.0,b5defce4-fa6b-4b59-9615-bcc1f271e456,11,ChatGPT,84,495,579,Conversation log +19477941.0,efe52725-b30d-4de8-8c73-56ad121666f0,0,ChatGPT,380,11,391,Conversation log +19477964.0,b5defce4-fa6b-4b59-9615-bcc1f271e456,5,ChatGPT,684,418,1102,Conversation log +19477975.0,80676dea-b9d2-4347-ae11-fb3008926e06,3,ChatGPT,585,239,824,Conversation log +19477984.0,efe52725-b30d-4de8-8c73-56ad121666f0,1,ChatGPT,418,17,435,Conversation log +19478156.0,68201869-131c-4de7-b428-7cc34ca426cf,3,ChatGPT,790,77,867,Conversation log +19478181.0,68201869-131c-4de7-b428-7cc34ca426cf,2,ChatGPT,773,74,847,Conversation log +19478184.0,b5defce4-fa6b-4b59-9615-bcc1f271e456,9,ChatGPT,1133,340,1473,Conversation log +19478185.0,c29c0b68-41af-4e5d-bd23-182d153c7026,11,GPT-4,561,125,686,Conversation log +19478186.0,b0d071ef-8d89-47e2-9d3e-6561c8d34b0c,5,ChatGPT,128,158,286,Conversation log +19478223.0,f5c2c846-16b3-4610-978b-5783f80efb48,9,ChatGPT,401,694,1095,Conversation log +19478277.0,c29c0b68-41af-4e5d-bd23-182d153c7026,3,GPT-4,646,25,671,Conversation log +19478282.0,c29c0b68-41af-4e5d-bd23-182d153c7026,7,GPT-4,717,131,848,Conversation log +19478341.0,efe52725-b30d-4de8-8c73-56ad121666f0,2,ChatGPT,464,17,481,Conversation log +19478360.0,b0d071ef-8d89-47e2-9d3e-6561c8d34b0c,1,ChatGPT,341,24,365,Conversation log +19478364.0,c29c0b68-41af-4e5d-bd23-182d153c7026,33,GPT-4,999,316,1315,Conversation log +19478373.0,7e7d6f12-3649-4467-8624-c19a2e58aa9d,0,ChatGPT,738,48,786,Conversation log +19478411.0,68201869-131c-4de7-b428-7cc34ca426cf,2,ChatGPT,798,57,855,Conversation log +19478436.0,3c165d0f-0118-449a-94d5-ec59cb721be5,3,ChatGPT,794,89,883,Conversation log +19478527.0,3c165d0f-0118-449a-94d5-ec59cb721be5,1,ChatGPT,933,96,1029,Conversation log +19478580.0,3f9670a6-3e3d-44b4-b3f0-ac1742d3843e,4,ChatGPT,379,131,510,Conversation log +19478585.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,2,ChatGPT,82,46,128,Conversation log +19478600.0,a9e07ba1-e7c1-4360-a774-6d684273d5b5,1,ChatGPT,32,18,50,Conversation log +19478619.0,efe52725-b30d-4de8-8c73-56ad121666f0,2,ChatGPT,236,71,307,Conversation log +19478624.0,c29c0b68-41af-4e5d-bd23-182d153c7026,22,GPT-4,1581,337,1918,Conversation log +19478631.0,3d1ad40d-66cf-4b64-8c38-2a12326abadb,3,ChatGPT,15,328,343,Conversation log +19478636.0,b834913c-a77c-4360-8c3e-bba69ea65374,0,ChatGPT,79,63,142,Conversation log +19478637.0,3c165d0f-0118-449a-94d5-ec59cb721be5,3,ChatGPT,1078,167,1245,Conversation log +19478647.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,1,ChatGPT,191,34,225,Conversation log +19478699.0,d921e3c1-8cea-4faa-9c44-ea5557fbc116,4,ChatGPT,27,382,409,Conversation log +19478713.0,64fc0e1a-1001-4467-bc1b-5012c5eea2bf,45,GPT-4,26,553,579,Conversation log +19478717.0,1ad91820-0b5f-437b-8e5f-53d86ba39fab,5,ChatGPT,1600,471,2071,Conversation log +19478717.0,3f9670a6-3e3d-44b4-b3f0-ac1742d3843e,3,ChatGPT,528,106,634,Conversation log +19478757.0,b834913c-a77c-4360-8c3e-bba69ea65374,1,ChatGPT,220,68,288,Conversation log +19478778.0,64fc0e1a-1001-4467-bc1b-5012c5eea2bf,47,GPT-4,592,502,1094,Conversation log +19478796.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,2,ChatGPT,380,104,484,Conversation log +19478817.0,bba5d06c-a6ce-4b6e-94db-a2022253e428,1,ChatGPT,92,59,151,Conversation log +19478820.0,1ad91820-0b5f-437b-8e5f-53d86ba39fab,5,ChatGPT,2121,552,2673,Conversation log +19478837.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,1,ChatGPT,498,57,555,Conversation log +19478853.0,1a311690-1fd2-449d-9833-dbc9444275f0,4,ChatGPT,366,55,421,Conversation log +19478886.0,926c155f-6cbc-4404-8848-55b7a859a54e,0,ChatGPT,304,16,320,Conversation log +19478889.0,5589009a-91d4-401f-8de6-e9848eb39c03,5,ChatGPT,189,281,470,Conversation log +19478896.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,2,ChatGPT,572,103,675,Conversation log +19478920.0,68201869-131c-4de7-b428-7cc34ca426cf,2,ChatGPT,852,116,968,Conversation log +19478929.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,1,ChatGPT,565,100,665,Conversation log +19478949.0,7345154f-a066-4d61-8bb4-1a1879d1aa52,1,ChatGPT,149,87,236,Conversation log +19478975.0,0f284f8b-add5-493e-8fda-a57620a580ee,34,GPT-4,563,214,777,Conversation log +19478977.0,bba5d06c-a6ce-4b6e-94db-a2022253e428,0,ChatGPT,163,8,171,Conversation log +19479005.0,bba5d06c-a6ce-4b6e-94db-a2022253e428,1,ChatGPT,184,6,190,Conversation log +19479020.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,1,ChatGPT,653,57,710,Conversation log +19479029.0,35793c3d-a428-48af-b670-3be233737e09,1,ChatGPT,8,9,17,Conversation log +19479030.0,f93dadc5-1806-46e3-90f3-c53a79e64fc5,3,ChatGPT,169,14,183,Conversation log +19479031.0,68201869-131c-4de7-b428-7cc34ca426cf,4,ChatGPT,774,161,935,Conversation log +19479038.0,0f284f8b-add5-493e-8fda-a57620a580ee,50,GPT-4,682,297,979,Conversation log +19479045.0,f93dadc5-1806-46e3-90f3-c53a79e64fc5,1,ChatGPT,201,13,214,Conversation log +19479084.0,68201869-131c-4de7-b428-7cc34ca426cf,1,ChatGPT,930,77,1007,Conversation log +19479086.0,f93dadc5-1806-46e3-90f3-c53a79e64fc5,1,ChatGPT,235,13,248,Conversation log +19479109.0,0524d64c-d5fa-4387-bc4a-e379438ede44,5,ChatGPT,20,339,359,Conversation log +19479110.0,181e8a96-674e-44cd-b54d-78dea1b504c7,2,ChatGPT,609,47,656,Conversation log +19479113.0,f5c2c846-16b3-4610-978b-5783f80efb48,1,ChatGPT,1195,181,1376,Conversation log +19479127.0,3f9670a6-3e3d-44b4-b3f0-ac1742d3843e,1,ChatGPT,669,53,722,Conversation log +19479143.0,35793c3d-a428-48af-b670-3be233737e09,0,ChatGPT,46,16,62,Conversation log +19479182.0,68201869-131c-4de7-b428-7cc34ca426cf,8,ChatGPT,936,157,1093,Conversation log +19479237.0,60cf96c3-b9a3-425d-b8aa-5cb52a83e94c,5,ChatGPT,37,294,331,Conversation log +19479238.0,0f2378f5-84c3-48fe-9355-fb536585b45d,68,GPT-4,12,776,788,Conversation log +19479258.0,35793c3d-a428-48af-b670-3be233737e09,1,ChatGPT,75,15,90,Conversation log +19479271.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,2,ChatGPT,467,54,521,Conversation log +19479280.0,35793c3d-a428-48af-b670-3be233737e09,1,ChatGPT,105,15,120,Conversation log +19479288.0,e4fa8c3a-fe67-44c6-b021-1e0ce99809b9,3,ChatGPT,63,324,387,Conversation log +19479305.0,e4fa8c3a-fe67-44c6-b021-1e0ce99809b9,4,ChatGPT,453,233,686,Conversation log +19479343.0,bd73fbc3-25f5-4e8a-8efa-a63bfbdd7cf3,9,ChatGPT,1234,670,1904,Conversation log +19479381.0,6a13d58e-49da-49b5-829b-ab764f010d6f,3,ChatGPT,22,221,243,Conversation log +19479424.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,1,ChatGPT,467,54,521,Conversation log +19479433.0,6a13d58e-49da-49b5-829b-ab764f010d6f,2,ChatGPT,267,202,469,Conversation log +19479437.0,84b8a44a-4b64-4ab6-b784-01a53f7c1126,8,GPT-4,519,50,569,Conversation log +19479458.0,0f284f8b-add5-493e-8fda-a57620a580ee,16,GPT-4,943,22,965,Conversation log +19479460.0,bd73fbc3-25f5-4e8a-8efa-a63bfbdd7cf3,5,ChatGPT,1927,480,2407,Conversation log +19479474.0,0a4b4191-e4f5-4fde-9b54-9a4b9d0dca25,2,ChatGPT,56,123,179,Conversation log +19479517.0,6a13d58e-49da-49b5-829b-ab764f010d6f,2,ChatGPT,493,200,693,Conversation log +19479542.0,0f284f8b-add5-493e-8fda-a57620a580ee,40,GPT-4,1096,191,1287,Conversation log +19479544.0,6a13d58e-49da-49b5-829b-ab764f010d6f,2,ChatGPT,722,181,903,Conversation log +19479562.0,bcf6c528-e342-421a-b04e-a1991e272945,2,ChatGPT,21,10,31,Conversation log +19479564.0,b5defce4-fa6b-4b59-9615-bcc1f271e456,5,ChatGPT,1497,315,1812,Conversation log +19479577.0,11c0c733-3220-4deb-b5fd-d6366fe54bef,3,ChatGPT,28,272,300,Conversation log +19479584.0,ed956d87-4593-41ff-9270-37c02b668487,14,ChatGPT,1552,928,2480,Conversation log +19479596.0,48ebee3e-57a7-4411-aa64-309820af4510,2,ChatGPT,121,94,215,Conversation log +19479601.0,8f5263df-e41d-4360-8f9a-7423cd08950e,1,ChatGPT,167,96,263,Conversation log +19479633.0,11c0c733-3220-4deb-b5fd-d6366fe54bef,46,GPT-4,334,326,660,Conversation log +19479644.0,a9e07ba1-e7c1-4360-a774-6d684273d5b5,2,ChatGPT,116,66,182,Conversation log +19479647.0,160dfb66-bc81-4228-9902-cdf0203d0868,6,ChatGPT,19,297,316,Conversation log +19479648.0,6a13d58e-49da-49b5-829b-ab764f010d6f,3,ChatGPT,945,185,1130,Conversation log +19479670.0,0f284f8b-add5-493e-8fda-a57620a580ee,54,GPT-4,1271,195,1466,Conversation log +19479674.0,3dda1072-611f-4497-8afa-0cd4e30553fb,3,ChatGPT,23,331,354,Conversation log +19479725.0,7345154f-a066-4d61-8bb4-1a1879d1aa52,12,GPT-4,353,70,423,Conversation log +19479810.0,bd73fbc3-25f5-4e8a-8efa-a63bfbdd7cf3,3,ChatGPT,2573,159,2732,Conversation log +19479937.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,2,ChatGPT,462,29,491,Conversation log +19479963.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,0,ChatGPT,394,33,427,Conversation log +19479982.0,3dda1072-611f-4497-8afa-0cd4e30553fb,7,ChatGPT,366,385,751,Conversation log +19480016.0,bba5d06c-a6ce-4b6e-94db-a2022253e428,2,ChatGPT,264,41,305,Conversation log +19480116.0,8942a261-5744-4644-b6c3-b7cf3684bb89,4,ChatGPT,130,171,301,Conversation log +19480120.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,2,ChatGPT,363,49,412,Conversation log +19480183.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,1,ChatGPT,412,42,454,Conversation log +19480201.0,014d885c-28f4-46ac-853f-2dd376471a01,21,GPT-4,95,301,396,Conversation log +19480211.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,1,ChatGPT,456,45,501,Conversation log +19480244.0,75456ff6-769e-4f64-b987-4427f1acb11c,1,ChatGPT,25,15,40,Conversation log +19480245.0,f08fcfa0-365c-487d-a022-3ddbac088f33,3,ChatGPT,36,14,50,Conversation log +19480361.0,b9811a13-a9ab-4dcb-bd09-71561c50c901,5,ChatGPT,22,275,297,Conversation log +19480369.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,2,ChatGPT,498,56,554,Conversation log +19480389.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,2,ChatGPT,586,56,642,Conversation log +19480409.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,2,ChatGPT,530,42,572,Conversation log +19480480.0,75456ff6-769e-4f64-b987-4427f1acb11c,2,ChatGPT,81,29,110,Conversation log +19480521.0,053d1e8d-3601-4060-814d-44c056aa1df6,1,ChatGPT,81,77,158,Conversation log +19480534.0,053d1e8d-3601-4060-814d-44c056aa1df6,8,GPT-4,240,79,319,Conversation log +19480555.0,90c02ac1-d3fb-46b6-a7e3-cc4bc0f6cfe3,2,ChatGPT,62,106,168,Conversation log +19480590.0,27746ab4-f311-4cd3-8ef1-839cc0b7e8c4,13,ChatGPT,19,913,932,Conversation log +19480688.0,90c02ac1-d3fb-46b6-a7e3-cc4bc0f6cfe3,5,ChatGPT,200,326,526,Conversation log +19480702.0,90c02ac1-d3fb-46b6-a7e3-cc4bc0f6cfe3,3,ChatGPT,539,197,736,Conversation log +19480717.0,90c02ac1-d3fb-46b6-a7e3-cc4bc0f6cfe3,3,ChatGPT,750,203,953,Conversation log +19480744.0,a0a1380b-0e3e-4977-a689-711747ef8a25,24,GPT-4,694,92,786,Conversation log +19480786.0,0f284f8b-add5-493e-8fda-a57620a580ee,11,GPT-4,1064,44,1108,Conversation log +19480806.0,0f284f8b-add5-493e-8fda-a57620a580ee,12,GPT-4,825,61,886,Conversation log +19480904.0,bc2666a9-8581-4b83-9841-20a0b5a6cf7e,2,ChatGPT,104,53,157,Conversation log +19480949.0,cafa6119-e832-4ed5-834b-049c9ab362b0,2,ChatGPT,22,112,134,Conversation log +19480962.0,b9811a13-a9ab-4dcb-bd09-71561c50c901,7,ChatGPT,318,280,598,Conversation log +19480963.0,7afcfc41-af8f-49cb-8cda-86acbdbb4ab6,3,ChatGPT,950,81,1031,Conversation log +19480973.0,021a5e07-4d20-4615-9855-229182369c8f,2,ChatGPT,1164,146,1310,Conversation log +19480986.0,021a5e07-4d20-4615-9855-229182369c8f,1,ChatGPT,906,115,1021,Conversation log +19480994.0,9286be0f-c848-46af-bcf1-f2ce2c1916b1,2,ChatGPT,1477,35,1512,Conversation log +19480994.0,021a5e07-4d20-4615-9855-229182369c8f,2,ChatGPT,726,56,782,Conversation log +19481010.0,5a98fdec-ce4d-4b14-9506-3a5fbbd15c8b,4,ChatGPT,20,209,229,Conversation log +19481059.0,7f0d0ece-3f46-4e9c-a8ae-db8122db4a7b,5,ChatGPT,16,147,163,Conversation log +19481064.0,00ecf1a2-1bc8-4078-9480-11cce1395f72,59,GPT-4,614,452,1066,Conversation log +19481091.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,4,ChatGPT,526,45,571,Conversation log +19481109.0,cafa6119-e832-4ed5-834b-049c9ab362b0,1,ChatGPT,165,73,238,Conversation log +19481113.0,9ca5e263-a763-47a5-992a-f939b5e73a4d,6,GPT-4,93,42,135,Conversation log +19481123.0,1b78554b-528f-4cea-b8f2-235b32b2166d,3,ChatGPT,31,216,247,Conversation log +19481147.0,8f6cae9b-7fd5-48eb-a6f5-45934878a327,3,ChatGPT,17,213,230,Conversation log +19481191.0,cafa6119-e832-4ed5-834b-049c9ab362b0,3,ChatGPT,286,168,454,Conversation log +19481204.0,9ca5e263-a763-47a5-992a-f939b5e73a4d,8,GPT-4,165,116,281,Conversation log +19481210.0,b9811a13-a9ab-4dcb-bd09-71561c50c901,2,ChatGPT,673,48,721,Conversation log +19481223.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,2,ChatGPT,472,36,508,Conversation log +19481238.0,cafa6119-e832-4ed5-834b-049c9ab362b0,4,ChatGPT,472,261,733,Conversation log +19481239.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,2,ChatGPT,431,46,477,Conversation log +19481289.0,021a5e07-4d20-4615-9855-229182369c8f,3,ChatGPT,658,48,706,Conversation log +19481302.0,bba5d06c-a6ce-4b6e-94db-a2022253e428,0,ChatGPT,341,19,360,Conversation log +19481328.0,9ca5e263-a763-47a5-992a-f939b5e73a4d,2,GPT-4,304,5,309,Conversation log +19481337.0,bba5d06c-a6ce-4b6e-94db-a2022253e428,1,ChatGPT,246,34,280,Conversation log +19481366.0,469e228e-3e34-4311-b5af-2568f8e954b1,60,GPT-4,20,594,614,Conversation log +19481418.0,9286be0f-c848-46af-bcf1-f2ce2c1916b1,2,ChatGPT,1505,27,1532,Conversation log +19481424.0,9286be0f-c848-46af-bcf1-f2ce2c1916b1,1,ChatGPT,1309,15,1324,Conversation log +19481433.0,9286be0f-c848-46af-bcf1-f2ce2c1916b1,1,ChatGPT,868,68,936,Conversation log +19481442.0,355868d5-7817-4035-9fb3-97102e8433ec,3,ChatGPT,49,152,201,Conversation log +19481513.0,07fba814-8468-408d-9823-60c86fb9c385,2,ChatGPT,116,176,292,Conversation log +19481540.0,9286be0f-c848-46af-bcf1-f2ce2c1916b1,1,ChatGPT,425,55,480,Conversation log +19481552.0,9286be0f-c848-46af-bcf1-f2ce2c1916b1,1,ChatGPT,458,42,500,Conversation log +19481563.0,0f284f8b-add5-493e-8fda-a57620a580ee,3,GPT-4,870,11,881,Conversation log diff --git a/benchmarks/traffic_replay/fixtures/burstgpt_dirty.csv b/benchmarks/traffic_replay/fixtures/burstgpt_dirty.csv new file mode 100644 index 0000000..bcdcd3c --- /dev/null +++ b/benchmarks/traffic_replay/fixtures/burstgpt_dirty.csv @@ -0,0 +1,10 @@ +Timestamp,Model,Request tokens,Response tokens,Total tokens,Log Type +10,ChatGPT,100,50,150,Conversation log +20,ChatGPT,0,0,0,Conversation log +30,ChatGPT,120,0,120,Conversation log +40,ChatGPT,-5,10,5,Conversation log +50,ChatGPT,abc,10,10,Conversation log +55,ChatGPT,,50,50,Conversation log +60,ChatGPT,100 +45,ChatGPT,100,50,150,Conversation log +70,GPT-4,200,100,300,API log diff --git a/benchmarks/traffic_replay/fixtures/burstgpt_slice.csv b/benchmarks/traffic_replay/fixtures/burstgpt_slice.csv new file mode 100644 index 0000000..9ead040 --- /dev/null +++ b/benchmarks/traffic_replay/fixtures/burstgpt_slice.csv @@ -0,0 +1,401 @@ +Timestamp,Model,Request tokens,Response tokens,Total tokens,Log Type +5,ChatGPT,472,18,490,Conversation log +45,ChatGPT,1087,230,1317,Conversation log +118,GPT-4,417,276,693,Conversation log +185,ChatGPT,1360,647,2007,Conversation log +214,ChatGPT,185,215,400,Conversation log +233,GPT-4,586,293,879,Conversation log +261,ChatGPT,37,1656,1693,Conversation log +267,ChatGPT,54,503,557,Conversation log +410,ChatGPT,1528,414,1942,Conversation log +535,ChatGPT,89,370,459,Conversation log +560,GPT-4,549,362,911,Conversation log +638,ChatGPT,172,69,241,Conversation log +686,GPT-4,969,206,1175,Conversation log +741,ChatGPT,97,137,234,Conversation log +771,GPT-4,1574,501,2075,Conversation log +821,ChatGPT,253,239,492,Conversation log +966,ChatGPT,23,344,367,Conversation log +974,ChatGPT,509,207,716,Conversation log +1028,ChatGPT,387,349,736,Conversation log +1072,ChatGPT,733,180,913,Conversation log +1106,ChatGPT,0,0,0,Conversation log +1129,ChatGPT,1305,477,1782,Conversation log +1278,ChatGPT,17,382,399,Conversation log +1449,ChatGPT,601,557,1158,Conversation log +1611,GPT-4,1327,239,1566,Conversation log +1780,ChatGPT,1437,181,1618,Conversation log +2069,ChatGPT,1336,245,1581,Conversation log +2173,ChatGPT,19,440,459,Conversation log +2273,GPT-4,92,95,187,Conversation log +2293,ChatGPT,507,448,955,Conversation log +2579,ChatGPT,938,263,1201,Conversation log +2584,ChatGPT,1544,608,2152,Conversation log +2608,GPT-4,1840,378,2218,Conversation log +2684,ChatGPT,991,462,1453,Conversation log +2826,ChatGPT,1109,329,1438,Conversation log +3009,ChatGPT,1476,443,1919,Conversation log +3091,ChatGPT,14,11,25,Conversation log +3232,GPT-4,1741,428,2169,Conversation log +4525,GPT-4,0,0,0,Conversation log +4574,ChatGPT,1306,267,1573,Conversation log +4608,GPT-4,0,0,0,Conversation log +4725,GPT-4,0,0,0,Conversation log +4798,GPT-4,614,323,937,Conversation log +5812,ChatGPT,1960,136,2096,Conversation log +6049,ChatGPT,1698,627,2325,Conversation log +6263,ChatGPT,1907,616,2523,Conversation log +6499,ChatGPT,137,225,362,Conversation log +6882,GPT-4,20,86,106,Conversation log +6998,GPT-4,127,268,395,Conversation log +7442,GPT-4,665,167,832,Conversation log +7982,GPT-4,855,297,1152,Conversation log +8052,GPT-4,1191,30,1221,Conversation log +8126,GPT-4,18,67,85,Conversation log +8208,GPT-4,101,148,249,Conversation log +8211,ChatGPT,112,336,448,Conversation log +8258,GPT-4,264,76,340,Conversation log +8681,ChatGPT,82,343,425,Conversation log +8782,ChatGPT,506,343,849,Conversation log +8989,ChatGPT,530,339,869,Conversation log +9015,ChatGPT,942,396,1338,Conversation log +9418,ChatGPT,1364,416,1780,Conversation log +10205,ChatGPT,122,496,618,Conversation log +10414,ChatGPT,24,63,87,Conversation log +10443,ChatGPT,698,258,956,Conversation log +10577,ChatGPT,103,76,179,Conversation log +10600,ChatGPT,1073,129,1202,Conversation log +10603,ChatGPT,200,95,295,Conversation log +10636,ChatGPT,1282,188,1470,Conversation log +10669,ChatGPT,1762,418,2180,Conversation log +10740,ChatGPT,1625,268,1893,Conversation log +10760,ChatGPT,25,326,351,Conversation log +10760,ChatGPT,1639,188,1827,Conversation log +10812,ChatGPT,42,197,239,Conversation log +10825,ChatGPT,1637,231,1868,Conversation log +10949,ChatGPT,1701,381,2082,Conversation log +10995,ChatGPT,383,319,702,Conversation log +11096,ChatGPT,266,246,512,Conversation log +11411,ChatGPT,1450,497,1947,Conversation log +11517,ChatGPT,1631,422,2053,Conversation log +11539,ChatGPT,540,285,825,Conversation log +11642,ChatGPT,1797,577,2374,Conversation log +11738,ChatGPT,0,0,0,Conversation log +11817,ChatGPT,20,310,330,Conversation log +11853,ChatGPT,352,298,650,Conversation log +12560,ChatGPT,726,334,1060,Conversation log +14618,ChatGPT,870,269,1139,Conversation log +19020,ChatGPT,66,342,408,Conversation log +19108,ChatGPT,478,296,774,Conversation log +19499,ChatGPT,837,292,1129,Conversation log +19601,ChatGPT,1152,237,1389,Conversation log +20288,ChatGPT,42,17,59,Conversation log +26939,ChatGPT,29,69,98,Conversation log +27037,ChatGPT,136,74,210,Conversation log +27162,ChatGPT,229,69,298,Conversation log +27307,ChatGPT,326,66,392,Conversation log +27894,ChatGPT,408,322,730,Conversation log +27948,ChatGPT,667,79,746,Conversation log +28389,ChatGPT,657,192,849,Conversation log +28436,ChatGPT,788,292,1080,Conversation log +28571,ChatGPT,1035,227,1262,Conversation log +28765,GPT-4,930,83,1013,API log +28829,GPT-4,929,80,1009,API log +29450,ChatGPT,36,14,50,Conversation log +29549,ChatGPT,103,23,126,Conversation log +29568,ChatGPT,209,242,451,Conversation log +29648,ChatGPT,505,166,671,Conversation log +29822,ChatGPT,166,34,200,Conversation log +29832,ChatGPT,19,326,345,Conversation log +29848,ChatGPT,239,30,269,Conversation log +29870,ChatGPT,357,559,916,Conversation log +29907,ChatGPT,320,28,348,Conversation log +29987,ChatGPT,333,35,368,Conversation log +29992,ChatGPT,342,11,353,Conversation log +30024,ChatGPT,353,62,415,Conversation log +30096,ChatGPT,26,205,231,Conversation log +30324,ChatGPT,950,67,1017,Conversation log +30342,ChatGPT,249,229,478,Conversation log +30348,ChatGPT,926,73,999,Conversation log +30368,ChatGPT,495,270,765,Conversation log +30371,ChatGPT,807,73,880,Conversation log +30393,ChatGPT,587,96,683,Conversation log +30402,ChatGPT,0,0,0,Conversation log +30414,ChatGPT,0,0,0,Conversation log +30418,ChatGPT,0,0,0,Conversation log +30422,ChatGPT,1539,82,1621,Conversation log +30424,ChatGPT,428,194,622,Conversation log +30476,ChatGPT,434,1018,1452,Conversation log +30528,ChatGPT,561,79,640,Conversation log +30601,ChatGPT,565,90,655,Conversation log +30612,ChatGPT,31,144,175,Conversation log +30649,ChatGPT,21,16,37,Conversation log +30654,ChatGPT,578,300,878,Conversation log +30677,ChatGPT,68,232,300,Conversation log +30795,ChatGPT,14,423,437,Conversation log +30852,ChatGPT,839,465,1304,Conversation log +30878,ChatGPT,447,381,828,Conversation log +30899,ChatGPT,838,436,1274,Conversation log +31163,ChatGPT,1107,288,1395,Conversation log +31244,ChatGPT,1306,245,1551,Conversation log +31293,ChatGPT,1458,366,1824,Conversation log +31390,ChatGPT,1524,427,1951,Conversation log +31458,ChatGPT,1435,462,1897,Conversation log +31790,ChatGPT,1618,161,1779,Conversation log +31810,ChatGPT,1536,208,1744,Conversation log +31826,ChatGPT,1385,150,1535,Conversation log +31841,ChatGPT,1113,197,1310,Conversation log +31906,ChatGPT,109,1413,1522,Conversation log +31985,ChatGPT,45,972,1017,Conversation log +31994,ChatGPT,36,307,343,Conversation log +32004,GPT-4,380,210,590,Conversation log +32034,ChatGPT,28,62,90,Conversation log +32119,ChatGPT,42,954,996,Conversation log +32185,ChatGPT,246,130,376,Conversation log +32247,ChatGPT,395,787,1182,Conversation log +32311,ChatGPT,1254,312,1566,Conversation log +32387,ChatGPT,840,246,1086,Conversation log +32488,ChatGPT,1169,845,2014,Conversation log +32561,ChatGPT,1304,343,1647,Conversation log +32582,ChatGPT,19,56,75,Conversation log +32599,ChatGPT,92,252,344,Conversation log +32601,ChatGPT,69,381,450,Conversation log +32625,ChatGPT,10,318,328,Conversation log +32656,ChatGPT,531,40,571,Conversation log +32663,ChatGPT,23,426,449,Conversation log +32674,ChatGPT,49,11,60,Conversation log +32678,ChatGPT,40,541,581,Conversation log +32680,GPT-4,99,470,569,Conversation log +32698,ChatGPT,134,55,189,Conversation log +32820,ChatGPT,42,1362,1404,Conversation log +32915,ChatGPT,1877,455,2332,Conversation log +32946,ChatGPT,12,391,403,Conversation log +33017,ChatGPT,302,119,421,Conversation log +33035,ChatGPT,447,639,1086,Conversation log +33042,ChatGPT,10,3,13,Conversation log +33068,ChatGPT,26,67,93,Conversation log +33072,ChatGPT,103,16,119,Conversation log +33099,ChatGPT,0,0,0,Conversation log +33111,ChatGPT,1737,300,2037,Conversation log +33206,ChatGPT,22,416,438,Conversation log +33312,ChatGPT,111,1158,1269,Conversation log +33313,ChatGPT,100,104,204,Conversation log +33324,ChatGPT,1416,586,2002,Conversation log +33371,ChatGPT,39,1238,1277,Conversation log +33407,ChatGPT,22,287,309,Conversation log +33411,ChatGPT,13,4,17,Conversation log +33413,ChatGPT,405,256,661,Conversation log +33437,ChatGPT,38,19,57,Conversation log +33454,ChatGPT,708,44,752,Conversation log +33481,ChatGPT,963,109,1072,Conversation log +33509,ChatGPT,1276,266,1542,Conversation log +33510,ChatGPT,1065,524,1589,Conversation log +33527,ChatGPT,271,54,325,Conversation log +33581,ChatGPT,1314,1903,3217,Conversation log +33602,ChatGPT,337,356,693,Conversation log +33632,ChatGPT,123,261,384,Conversation log +33632,ChatGPT,387,58,445,Conversation log +33637,ChatGPT,102,36,138,Conversation log +33696,ChatGPT,854,350,1204,Conversation log +33747,ChatGPT,522,65,587,Conversation log +33784,ChatGPT,916,484,1400,Conversation log +33810,ChatGPT,603,6,609,Conversation log +33850,ChatGPT,628,495,1123,Conversation log +33866,ChatGPT,31,26,57,Conversation log +34015,ChatGPT,16,421,437,Conversation log +34016,ChatGPT,1640,664,2304,Conversation log +34035,ChatGPT,1232,438,1670,Conversation log +34050,ChatGPT,489,92,581,Conversation log +34068,ChatGPT,40,1092,1132,Conversation log +34069,ChatGPT,461,496,957,Conversation log +34076,ChatGPT,362,155,517,Conversation log +34087,ChatGPT,149,3,152,Conversation log +34100,ChatGPT,1540,35,1575,Conversation log +34102,GPT-4,974,513,1487,Conversation log +34105,ChatGPT,541,201,742,Conversation log +34117,ChatGPT,954,51,1005,Conversation log +34126,ChatGPT,930,92,1022,Conversation log +34129,ChatGPT,34,15,49,Conversation log +34139,ChatGPT,1169,2019,3188,Conversation log +34146,GPT-4,1512,368,1880,Conversation log +34173,ChatGPT,1994,570,2564,Conversation log +34173,ChatGPT,489,338,827,Conversation log +34193,ChatGPT,72,19,91,Conversation log +34246,ChatGPT,1692,401,2093,Conversation log +34257,ChatGPT,849,5,854,Conversation log +34271,ChatGPT,490,27,517,Conversation log +34273,ChatGPT,205,144,349,Conversation log +34278,ChatGPT,1390,260,1650,Conversation log +34291,ChatGPT,547,302,849,Conversation log +34294,GPT-4,32,262,294,Conversation log +34320,ChatGPT,1021,335,1356,Conversation log +34323,ChatGPT,122,376,498,Conversation log +34347,ChatGPT,1844,233,2077,Conversation log +34352,ChatGPT,98,33,131,Conversation log +34384,ChatGPT,436,31,467,Conversation log +34405,ChatGPT,30,294,324,Conversation log +34420,ChatGPT,55,34,89,Conversation log +34444,ChatGPT,16,199,215,Conversation log +34525,ChatGPT,704,331,1035,Conversation log +34534,ChatGPT,71,731,802,Conversation log +34542,ChatGPT,47,32,79,Conversation log +34560,ChatGPT,1298,630,1928,Conversation log +34601,GPT-4,15,115,130,Conversation log +34615,GPT-4,145,314,459,Conversation log +34640,ChatGPT,0,0,0,Conversation log +34680,GPT-4,477,297,774,Conversation log +34746,GPT-4,189,253,442,Conversation log +34758,ChatGPT,418,95,513,Conversation log +34770,GPT-4,585,26,611,Conversation log +34802,GPT-4,757,211,968,Conversation log +34818,ChatGPT,705,299,1004,Conversation log +34824,GPT-4,203,200,403,Conversation log +34832,ChatGPT,17,114,131,Conversation log +34868,ChatGPT,22,29,51,Conversation log +34876,ChatGPT,210,416,626,Conversation log +34890,GPT-4,978,197,1175,Conversation log +34896,GPT-4,188,181,369,Conversation log +34911,GPT-4,226,362,588,Conversation log +34919,ChatGPT,16,18,34,Conversation log +34928,ChatGPT,45,14,59,Conversation log +34943,GPT-4,163,19,182,Conversation log +34976,ChatGPT,0,0,0,Conversation log +34986,ChatGPT,1258,384,1642,Conversation log +34988,ChatGPT,1956,353,2309,Conversation log +34995,GPT-4,389,221,610,Conversation log +35016,ChatGPT,1046,395,1441,Conversation log +35056,ChatGPT,867,30,897,Conversation log +35087,ChatGPT,145,618,763,Conversation log +35096,GPT-4,603,339,942,Conversation log +35132,GPT-4,19,380,399,Conversation log +35134,ChatGPT,870,80,950,Conversation log +35138,ChatGPT,72,13,85,Conversation log +35142,ChatGPT,963,90,1053,Conversation log +35149,GPT-4,33,82,115,Conversation log +35156,GPT-4,342,231,573,Conversation log +35161,GPT-4,161,22,183,Conversation log +35168,GPT-4,419,385,804,Conversation log +35178,ChatGPT,244,149,393,Conversation log +35184,ChatGPT,66,36,102,Conversation log +35220,ChatGPT,162,14,176,Conversation log +35227,GPT-4,210,17,227,Conversation log +35235,ChatGPT,862,214,1076,Conversation log +35244,ChatGPT,263,17,280,Conversation log +35246,ChatGPT,168,35,203,Conversation log +35258,ChatGPT,1122,59,1181,Conversation log +35260,ChatGPT,1036,13,1049,Conversation log +35266,ChatGPT,353,15,368,Conversation log +35293,ChatGPT,354,342,696,Conversation log +35319,ChatGPT,302,887,1189,Conversation log +35369,ChatGPT,723,541,1264,Conversation log +35370,ChatGPT,1202,157,1359,Conversation log +35374,GPT-4,466,60,526,Conversation log +35403,ChatGPT,217,241,458,Conversation log +35410,ChatGPT,25,57,82,Conversation log +35415,ChatGPT,1360,702,2062,Conversation log +35426,ChatGPT,70,82,152,Conversation log +35429,ChatGPT,10,111,121,Conversation log +35453,ChatGPT,140,98,238,Conversation log +35459,ChatGPT,20,556,576,Conversation log +35520,ChatGPT,590,93,683,Conversation log +35541,ChatGPT,308,151,459,Conversation log +35576,GPT-4,58,26,84,Conversation log +35611,ChatGPT,35,12,47,Conversation log +35647,ChatGPT,74,19,93,Conversation log +35746,ChatGPT,15,38,53,Conversation log +35761,ChatGPT,0,0,0,Conversation log +35776,ChatGPT,396,333,729,Conversation log +35788,ChatGPT,247,506,753,Conversation log +35796,ChatGPT,757,356,1113,Conversation log +35816,GPT-4,45,32,77,Conversation log +35837,GPT-4,273,25,298,Conversation log +35861,ChatGPT,274,442,716,Conversation log +35876,ChatGPT,1130,812,1942,Conversation log +35894,GPT-4,820,26,846,Conversation log +35898,GPT-4,788,188,976,Conversation log +35921,ChatGPT,18,412,430,Conversation log +35954,ChatGPT,179,329,508,Conversation log +35975,ChatGPT,19,38,57,Conversation log +36021,ChatGPT,23,27,50,Conversation log +36025,ChatGPT,1286,16,1302,Conversation log +36043,ChatGPT,535,297,832,Conversation log +36055,ChatGPT,17,7,24,Conversation log +36069,ChatGPT,36,24,60,Conversation log +36074,ChatGPT,992,15,1007,Conversation log +36097,GPT-4,695,16,711,Conversation log +36114,ChatGPT,36,238,274,Conversation log +36116,GPT-4,146,299,445,Conversation log +36121,GPT-4,149,256,405,Conversation log +36165,ChatGPT,73,273,346,Conversation log +36173,GPT-4,985,74,1059,Conversation log +36195,ChatGPT,18,457,475,Conversation log +36246,GPT-4,1958,737,2695,Conversation log +36282,GPT-4,36,505,541,Conversation log +36286,ChatGPT,66,17,83,Conversation log +36308,ChatGPT,88,300,388,Conversation log +36309,ChatGPT,203,285,488,Conversation log +36343,ChatGPT,232,514,746,Conversation log +36352,GPT-4,74,404,478,Conversation log +36362,ChatGPT,537,249,786,Conversation log +36371,ChatGPT,0,0,0,Conversation log +36385,GPT-4,567,416,983,Conversation log +36435,ChatGPT,14,203,217,Conversation log +36447,GPT-4,995,454,1449,Conversation log +36471,ChatGPT,836,701,1537,Conversation log +36479,ChatGPT,81,999,1080,Conversation log +36516,ChatGPT,477,163,640,Conversation log +36591,ChatGPT,1098,721,1819,Conversation log +36607,GPT-4,29,2,31,Conversation log +36617,ChatGPT,24,14,38,Conversation log +36626,GPT-4,1475,485,1960,Conversation log +36638,GPT-4,1025,405,1430,Conversation log +36646,GPT-4,391,25,416,Conversation log +36647,ChatGPT,48,11,59,Conversation log +36660,ChatGPT,69,14,83,Conversation log +36665,ChatGPT,36,370,406,Conversation log +36673,GPT-4,347,147,494,Conversation log +36700,ChatGPT,1847,1362,3209,Conversation log +36701,GPT-4,510,599,1109,Conversation log +36716,ChatGPT,20,606,626,Conversation log +36738,ChatGPT,20,171,191,Conversation log +36754,ChatGPT,0,0,0,Conversation log +36758,ChatGPT,0,0,0,Conversation log +36789,ChatGPT,26,23,49,Conversation log +36791,ChatGPT,299,20,319,Conversation log +36804,GPT-4,1995,663,2658,Conversation log +36805,ChatGPT,59,31,90,Conversation log +36836,ChatGPT,1365,214,1579,Conversation log +36850,ChatGPT,1595,354,1949,Conversation log +36868,GPT-4,500,238,738,Conversation log +36883,ChatGPT,51,1442,1493,Conversation log +36915,GPT-4,2126,379,2505,Conversation log +36922,ChatGPT,33,81,114,Conversation log +36968,ChatGPT,79,59,138,Conversation log +36989,GPT-4,145,45,190,Conversation log +36990,ChatGPT,118,24,142,Conversation log +36993,ChatGPT,222,43,265,Conversation log +36997,ChatGPT,21,387,408,Conversation log +37000,GPT-4,752,280,1032,Conversation log +37006,ChatGPT,158,104,262,Conversation log +37008,ChatGPT,170,14,184,Conversation log +37053,ChatGPT,290,90,380,Conversation log +37081,ChatGPT,601,327,928,Conversation log +37106,GPT-4,29,213,242,Conversation log +37117,GPT-4,112,110,222,Conversation log +37140,ChatGPT,50,962,1012,Conversation log +37148,ChatGPT,733,311,1044,Conversation log +37178,ChatGPT,9,9,18,Conversation log +37182,ChatGPT,40,11,51,Conversation log +37183,GPT-4,41,552,593,Conversation log +37189,ChatGPT,1544,1421,2965,Conversation log +37198,GPT-4,28,231,259,Conversation log +37201,ChatGPT,24,443,467,Conversation log +37218,ChatGPT,657,1011,1668,Conversation log +37228,ChatGPT,60,46,106,Conversation log +37235,GPT-4,275,225,500,Conversation log +37236,ChatGPT,0,0,0,Conversation log +37243,ChatGPT,0,0,0,Conversation log +37255,ChatGPT,50,1143,1193,Conversation log +37272,GPT-4,251,23,274,Conversation log +37274,GPT-4,18,70,88,Conversation log +37274,ChatGPT,67,58,125,Conversation log diff --git a/benchmarks/traffic_replay/fixtures/mooncake_dirty.jsonl b/benchmarks/traffic_replay/fixtures/mooncake_dirty.jsonl new file mode 100644 index 0000000..c90273b --- /dev/null +++ b/benchmarks/traffic_replay/fixtures/mooncake_dirty.jsonl @@ -0,0 +1,9 @@ +{"timestamp": 0, "input_length": 1024, "output_length": 64, "hash_ids": [1, 2]} +{"timestamp": 1000, "input_length": 0, "output_length": 64, "hash_ids": []} +{"timestamp": 2000, "input_length": 512, "output_length": 0, "hash_ids": [3]} +{"timestamp": 3000, "input_length": -5, "output_length": 8, "hash_ids": [4]} +{"timestamp": 4000, "input_length": "x", "output_length": 8, "hash_ids": [5]} +{"timestamp": 5000, "output_length": 8, "hash_ids": [6]} +this line is not json at all +{"timestamp": 4500, "input_length": 512, "output_length": 8, "hash_ids": [7]} +{"timestamp": 6000, "input_length": 1536, "output_length": 32, "hash_ids": [1, 2, 8]} diff --git a/benchmarks/traffic_replay/fixtures/mooncake_slice.jsonl b/benchmarks/traffic_replay/fixtures/mooncake_slice.jsonl new file mode 100644 index 0000000..20ebead --- /dev/null +++ b/benchmarks/traffic_replay/fixtures/mooncake_slice.jsonl @@ -0,0 +1,400 @@ +{"timestamp": 0, "input_length": 6758, "output_length": 500, "hash_ids": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]} +{"timestamp": 0, "input_length": 7322, "output_length": 490, "hash_ids": [0, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]} +{"timestamp": 0, "input_length": 7236, "output_length": 794, "hash_ids": [0, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41]} +{"timestamp": 0, "input_length": 2290, "output_length": 316, "hash_ids": [0, 42, 43, 44, 45]} +{"timestamp": 0, "input_length": 6760, "output_length": 3, "hash_ids": [0, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58]} +{"timestamp": 0, "input_length": 4834, "output_length": 173, "hash_ids": [0, 59, 60, 61, 62, 63, 64, 65, 66, 67]} +{"timestamp": 0, "input_length": 23141, "output_length": 453, "hash_ids": [0, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112]} +{"timestamp": 0, "input_length": 26888, "output_length": 458, "hash_ids": [0, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164]} +{"timestamp": 0, "input_length": 10498, "output_length": 402, "hash_ids": [0, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184]} +{"timestamp": 0, "input_length": 17450, "output_length": 610, "hash_ids": [0, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218]} +{"timestamp": 3000, "input_length": 13544, "output_length": 71, "hash_ids": [0, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244]} +{"timestamp": 3000, "input_length": 87169, "output_length": 402, "hash_ids": [0, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414]} +{"timestamp": 3000, "input_length": 6324, "output_length": 548, "hash_ids": [0, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426]} +{"timestamp": 3000, "input_length": 2012, "output_length": 354, "hash_ids": [0, 427, 428, 429]} +{"timestamp": 3000, "input_length": 7324, "output_length": 14, "hash_ids": [0, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443]} +{"timestamp": 3000, "input_length": 9418, "output_length": 145, "hash_ids": [0, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, 456, 457, 458, 459, 460, 461]} +{"timestamp": 3000, "input_length": 915, "output_length": 355, "hash_ids": [0, 462]} +{"timestamp": 3000, "input_length": 12846, "output_length": 466, "hash_ids": [0, 463, 464, 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, 487]} +{"timestamp": 3000, "input_length": 20506, "output_length": 929, "hash_ids": [0, 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527]} +{"timestamp": 3000, "input_length": 16609, "output_length": 349, "hash_ids": [0, 528, 529, 530, 531, 532, 533, 534, 535, 536, 537, 538, 539, 540, 541, 542, 543, 544, 545, 546, 547, 548, 549, 550, 551, 552, 553, 554, 555, 556, 557, 558, 559]} +{"timestamp": 3000, "input_length": 26353, "output_length": 370, "hash_ids": [0, 560, 561, 562, 563, 564, 565, 566, 567, 568, 569, 570, 571, 572, 573, 574, 575, 576, 577, 578, 579, 580, 581, 582, 583, 584, 585, 586, 587, 588, 589, 590, 591, 592, 593, 594, 595, 596, 597, 598, 599, 600, 601, 602, 603, 604, 605, 606, 607, 608, 609, 610]} +{"timestamp": 3000, "input_length": 6059, "output_length": 475, "hash_ids": [0, 611, 612, 613, 614, 615, 616, 617, 618, 619, 620, 621]} +{"timestamp": 3000, "input_length": 5954, "output_length": 420, "hash_ids": [0, 622, 623, 624, 625, 626, 627, 628, 629, 630, 631, 632]} +{"timestamp": 3000, "input_length": 11339, "output_length": 848, "hash_ids": [0, 633, 634, 635, 636, 637, 638, 639, 640, 641, 642, 643, 644, 645, 646, 647, 648, 649, 650, 651, 652, 653, 654]} +{"timestamp": 3000, "input_length": 15172, "output_length": 80, "hash_ids": [0, 655, 656, 657, 658, 659, 660, 661, 662, 663, 664, 665, 666, 667, 668, 669, 670, 671, 672, 673, 674, 675, 676, 677, 678, 679, 680, 681, 682, 683]} +{"timestamp": 3000, "input_length": 45922, "output_length": 265, "hash_ids": [0, 684, 685, 686, 687, 688, 689, 690, 691, 692, 693, 694, 695, 696, 697, 698, 699, 700, 701, 702, 703, 704, 705, 706, 707, 708, 709, 710, 711, 712, 713, 714, 715, 716, 717, 718, 719, 720, 721, 722, 723, 724, 725, 726, 727, 728, 729, 730, 731, 732, 733, 734, 735, 736, 737, 738, 739, 740, 741, 742, 743, 744, 745, 746, 747, 748, 749, 750, 751, 752, 753, 754, 755, 756, 757, 758, 759, 760, 761, 762, 763, 764, 765, 766, 767, 768, 769, 770, 771, 772]} +{"timestamp": 5999, "input_length": 1053, "output_length": 26, "hash_ids": [0, 773, 774]} +{"timestamp": 5999, "input_length": 5710, "output_length": 745, "hash_ids": [0, 775, 776, 777, 778, 779, 780, 781, 782, 783, 784, 785]} +{"timestamp": 5999, "input_length": 7238, "output_length": 11, "hash_ids": [0, 786, 787, 788, 789, 790, 791, 792, 793, 794, 795, 796, 797, 798, 799]} +{"timestamp": 9000, "input_length": 10355, "output_length": 584, "hash_ids": [0, 800, 801, 802, 803, 804, 805, 806, 807, 808, 809, 810, 811, 812, 813, 814, 815, 816, 817, 818, 819]} +{"timestamp": 9000, "input_length": 1477, "output_length": 615, "hash_ids": [0, 820, 821]} +{"timestamp": 9000, "input_length": 15366, "output_length": 344, "hash_ids": [0, 822, 823, 824, 825, 826, 827, 828, 829, 830, 831, 832, 833, 834, 835, 836, 837, 838, 839, 840, 841, 842, 843, 844, 845, 846, 847, 848, 849, 850, 851]} +{"timestamp": 9000, "input_length": 3806, "output_length": 309, "hash_ids": [0, 852, 853, 854, 855, 856, 857, 858]} +{"timestamp": 9000, "input_length": 8772, "output_length": 1, "hash_ids": [0, 859, 860, 861, 862, 863, 864, 865, 866, 867, 868, 869, 870, 871, 872, 873, 874, 875]} +{"timestamp": 9000, "input_length": 26156, "output_length": 238, "hash_ids": [0, 876, 877, 878, 879, 880, 881, 882, 883, 884, 885, 886, 887, 888, 889, 890, 891, 892, 893, 894, 895, 896, 897, 898, 899, 900, 901, 902, 903, 904, 905, 906, 907, 908, 909, 910, 911, 912, 913, 914, 915, 916, 917, 918, 919, 920, 921, 922, 923, 924, 925, 926]} +{"timestamp": 9000, "input_length": 16635, "output_length": 449, "hash_ids": [0, 927, 928, 929, 930, 931, 932, 933, 934, 935, 936, 937, 938, 939, 940, 941, 942, 943, 944, 945, 946, 947, 948, 949, 950, 951, 952, 953, 954, 955, 956, 957, 958]} +{"timestamp": 9000, "input_length": 2293, "output_length": 31, "hash_ids": [0, 959, 960, 961, 962]} +{"timestamp": 9000, "input_length": 1110, "output_length": 240, "hash_ids": [0, 963, 964]} +{"timestamp": 12000, "input_length": 3628, "output_length": 555, "hash_ids": [0, 965, 966, 967, 968, 969, 970, 971]} +{"timestamp": 12000, "input_length": 2038, "output_length": 524, "hash_ids": [0, 972, 973, 974]} +{"timestamp": 12000, "input_length": 1902, "output_length": 587, "hash_ids": [0, 975, 976, 977]} +{"timestamp": 12000, "input_length": 14041, "output_length": 227, "hash_ids": [0, 978, 979, 980, 981, 982, 983, 984, 985, 986, 987, 988, 989, 990, 991, 992, 993, 994, 995, 996, 997, 998, 999, 1000, 1001, 1002, 1003, 1004]} +{"timestamp": 12000, "input_length": 12095, "output_length": 239, "hash_ids": [0, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1023, 1024, 1025, 1026, 1027]} +{"timestamp": 12000, "input_length": 1066, "output_length": 324, "hash_ids": [0, 1028, 1029]} +{"timestamp": 12000, "input_length": 9615, "output_length": 203, "hash_ids": [0, 1030, 1031, 1032, 1033, 1034, 1035, 1036, 1037, 1038, 1039, 1040, 1041, 1042, 1043, 1044, 1045, 1046, 1047]} +{"timestamp": 12000, "input_length": 19976, "output_length": 105, "hash_ids": [0, 1048, 1049, 1050, 1051, 1052, 1053, 1054, 1055, 1056, 1057, 1058, 1059, 1060, 1061, 1062, 1063, 1064, 1065, 1066, 1067, 1068, 1069, 1070, 1071, 1072, 1073, 1074, 1075, 1076, 1077, 1078, 1079, 1080, 1081, 1082, 1083, 1084, 1085, 1086]} +{"timestamp": 15000, "input_length": 6525, "output_length": 481, "hash_ids": [0, 1087, 1088, 1089, 1090, 1091, 1092, 1093, 1094, 1095, 1096, 1097, 1098]} +{"timestamp": 15000, "input_length": 898, "output_length": 324, "hash_ids": [0, 1099]} +{"timestamp": 15000, "input_length": 5391, "output_length": 361, "hash_ids": [0, 1100, 1101, 1102, 1103, 1104, 1105, 1106, 1107, 1108, 1109]} +{"timestamp": 15000, "input_length": 23631, "output_length": 362, "hash_ids": [0, 1110, 1111, 1112, 1113, 1114, 1115, 1116, 1117, 1118, 1119, 1120, 1121, 1122, 1123, 1124, 1125, 1126, 1127, 1128, 1129, 1130, 1131, 1132, 1133, 1134, 1135, 1136, 1137, 1138, 1139, 1140, 1141, 1142, 1143, 1144, 1145, 1146, 1147, 1148, 1149, 1150, 1151, 1152, 1153, 1154, 1155]} +{"timestamp": 15000, "input_length": 38383, "output_length": 589, "hash_ids": [0, 1156, 1157, 1158, 1159, 1160, 1161, 1162, 1163, 1164, 1165, 1166, 1167, 1168, 1169, 1170, 1171, 1172, 1173, 1174, 1175, 1176, 1177, 1178, 1179, 1180, 1181, 1182, 1183, 1184, 1185, 1186, 1187, 1188, 1189, 1190, 1191, 1192, 1193, 1194, 1195, 1196, 1197, 1198, 1199, 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211, 1212, 1213, 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227, 1228, 1229]} +{"timestamp": 15000, "input_length": 7543, "output_length": 415, "hash_ids": [0, 1230, 1231, 1232, 1233, 1234, 1235, 1236, 1237, 1238, 1239, 1240, 1241, 1242, 1243]} +{"timestamp": 15000, "input_length": 8974, "output_length": 219, "hash_ids": [0, 1244, 1245, 1246, 1247, 1248, 1249, 1250, 1251, 1252, 1253, 1254, 1255, 1256, 1257, 1258, 1259, 1260]} +{"timestamp": 15000, "input_length": 24246, "output_length": 587, "hash_ids": [0, 1261, 1262, 1263, 1264, 1265, 1266, 1267, 1268, 1269, 1270, 1271, 1272, 1273, 1274, 1275, 1276, 1277, 1278, 1279, 1280, 1281, 1282, 1283, 1284, 1285, 1286, 1287, 1288, 1289, 1290, 1291, 1292, 1293, 1294, 1295, 1296, 1297, 1298, 1299, 1300, 1301, 1302, 1303, 1304, 1305, 1306, 1307]} +{"timestamp": 15000, "input_length": 15761, "output_length": 417, "hash_ids": [0, 1308, 1309, 1310, 1311, 1312, 1313, 1314, 1315, 1316, 1317, 1318, 1319, 1320, 1321, 1322, 1323, 1324, 1325, 1326, 1327, 1328, 1329, 1330, 1331, 1332, 1333, 1334, 1335, 1336, 1337]} +{"timestamp": 15000, "input_length": 27701, "output_length": 459, "hash_ids": [0, 1338, 1339, 1340, 1341, 1342, 1343, 1344, 1345, 1346, 1347, 1348, 1349, 1350, 1351, 1352, 1353, 1354, 1355, 1356, 1357, 1358, 1359, 1360, 1361, 1362, 1363, 1364, 1365, 1366, 1367, 1368, 1369, 1370, 1371, 1372, 1373, 1374, 1375, 1376, 1377, 1378, 1379, 1380, 1381, 1382, 1383, 1384, 1385, 1386, 1387, 1388, 1389, 1390, 1391]} +{"timestamp": 15000, "input_length": 11041, "output_length": 98, "hash_ids": [0, 1392, 1393, 1394, 1395, 1396, 1397, 1398, 1399, 1400, 1401, 1402, 1403, 1404, 1405, 1406, 1407, 1408, 1409, 1410, 1411, 1412]} +{"timestamp": 15000, "input_length": 4836, "output_length": 24, "hash_ids": [0, 1413, 1414, 1415, 1416, 1417, 1418, 1419, 1420, 1421]} +{"timestamp": 15000, "input_length": 2350, "output_length": 355, "hash_ids": [0, 1422, 1423, 1424, 1425]} +{"timestamp": 15000, "input_length": 934, "output_length": 421, "hash_ids": [0, 1426]} +{"timestamp": 15000, "input_length": 14081, "output_length": 522, "hash_ids": [0, 1427, 1428, 1429, 1430, 1431, 1432, 1433, 1434, 1435, 1436, 1437, 1438, 1439, 1440, 1441, 1442, 1443, 1444, 1445, 1446, 1447, 1448, 1449, 1450, 1451, 1452, 1453]} +{"timestamp": 18000, "input_length": 7935, "output_length": 375, "hash_ids": [0, 1454, 1455, 1456, 1457, 1458, 1459, 1460, 1461, 1462, 1463, 1464, 1465, 1466, 1467, 1468]} +{"timestamp": 18000, "input_length": 13886, "output_length": 534, "hash_ids": [0, 1469, 1470, 1471, 1472, 1473, 1474, 1475, 1476, 1477, 1478, 1479, 1480, 1481, 1482, 1483, 1484, 1485, 1486, 1487, 1488, 1489, 1490, 1491, 1492, 1493, 1494, 1495]} +{"timestamp": 18000, "input_length": 898, "output_length": 57, "hash_ids": [0, 1496]} +{"timestamp": 18000, "input_length": 19694, "output_length": 178, "hash_ids": [0, 1497, 1498, 1499, 1500, 1501, 1502, 1503, 1504, 1505, 1506, 1507, 1508, 1509, 1510, 1511, 1512, 1513, 1514, 1515, 1516, 1517, 1518, 1519, 1520, 1521, 1522, 1523, 1524, 1525, 1526, 1527, 1528, 1529, 1530, 1531, 1532, 1533, 1534]} +{"timestamp": 21000, "input_length": 3184, "output_length": 195, "hash_ids": [0, 1535, 1536, 1537, 1538, 1539, 1540]} +{"timestamp": 21000, "input_length": 2651, "output_length": 366, "hash_ids": [0, 1541, 1542, 1543, 1544, 1545]} +{"timestamp": 21000, "input_length": 23143, "output_length": 26, "hash_ids": [0, 1546, 1547, 1548, 1549, 1550, 1551, 1552, 1553, 1554, 1555, 1556, 1557, 1558, 1559, 1560, 1561, 1562, 1563, 1564, 1565, 1566, 1567, 1568, 1569, 1570, 1571, 1572, 1573, 1574, 1575, 1576, 1577, 1578, 1579, 1580, 1581, 1582, 1583, 1584, 1585, 1586, 1587, 1588, 1589, 1590]} +{"timestamp": 21000, "input_length": 5922, "output_length": 542, "hash_ids": [0, 1591, 1592, 1593, 1594, 1595, 1596, 1597, 1598, 1599, 1600, 1601]} +{"timestamp": 21000, "input_length": 8429, "output_length": 439, "hash_ids": [0, 1602, 1603, 1604, 1605, 1606, 1607, 1608, 1609, 1610, 1611, 1612, 1613, 1614, 1615, 1616, 1617]} +{"timestamp": 21000, "input_length": 8352, "output_length": 387, "hash_ids": [0, 1618, 1619, 1620, 1621, 1622, 1623, 1624, 1625, 1626, 1627, 1628, 1629, 1630, 1631, 1632, 1633]} +{"timestamp": 24000, "input_length": 12159, "output_length": 237, "hash_ids": [0, 1634, 1635, 1636, 1637, 1638, 1639, 1640, 1641, 1642, 1643, 1644, 1645, 1646, 1647, 1648, 1649, 1650, 1651, 1652, 1653, 1654, 1655, 1656]} +{"timestamp": 24000, "input_length": 28214, "output_length": 603, "hash_ids": [0, 1657, 1658, 1659, 1660, 1661, 1662, 1663, 1664, 1665, 1666, 1667, 1668, 1669, 1670, 1671, 1672, 1673, 1674, 1675, 1676, 1677, 1678, 1679, 1680, 1681, 1682, 1683, 1684, 1685, 1686, 1687, 1688, 1689, 1690, 1691, 1692, 1693, 1694, 1695, 1696, 1697, 1698, 1699, 1700, 1701, 1702, 1703, 1704, 1705, 1706, 1707, 1708, 1709, 1710, 1711]} +{"timestamp": 24000, "input_length": 26891, "output_length": 3, "hash_ids": [0, 1712, 1713, 1714, 1715, 1716, 1717, 1718, 1719, 1720, 1721, 1722, 1723, 1724, 1725, 1726, 1727, 1728, 1729, 1730, 1731, 1732, 1733, 1734, 1735, 1736, 1737, 1738, 1739, 1740, 1741, 1742, 1743, 1744, 1745, 1746, 1747, 1748, 1749, 1750, 1751, 1752, 1753, 1754, 1755, 1756, 1757, 1758, 1759, 1760, 1761, 1762, 1763]} +{"timestamp": 24000, "input_length": 2133, "output_length": 347, "hash_ids": [0, 1764, 1765, 1766, 1767]} +{"timestamp": 24000, "input_length": 36422, "output_length": 255, "hash_ids": [0, 1768, 1769, 1770, 1771, 1772, 1773, 1774, 1775, 1776, 1777, 1778, 1779, 1780, 1781, 1782, 1783, 1784, 1785, 1786, 1787, 1788, 1789, 1790, 1791, 1792, 1793, 1794, 1795, 1796, 1797, 1798, 1799, 1800, 1801, 1802, 1803, 1804, 1805, 1806, 1807, 1808, 1809, 1810, 1811, 1812, 1813, 1814, 1815, 1816, 1817, 1818, 1819, 1820, 1821, 1822, 1823, 1824, 1825, 1826, 1827, 1828, 1829, 1830, 1831, 1832, 1833, 1834, 1835, 1836, 1837, 1838]} +{"timestamp": 27000, "input_length": 1424, "output_length": 290, "hash_ids": [0, 1839, 1840]} +{"timestamp": 27000, "input_length": 18092, "output_length": 387, "hash_ids": [0, 1841, 1842, 1843, 1844, 1845, 1846, 1847, 1848, 1849, 1850, 1851, 1852, 1853, 1854, 1855, 1856, 1857, 1858, 1859, 1860, 1861, 1862, 1863, 1864, 1865, 1866, 1867, 1868, 1869, 1870, 1871, 1872, 1873, 1874, 1875]} +{"timestamp": 27000, "input_length": 35126, "output_length": 538, "hash_ids": [0, 1876, 1877, 1878, 1879, 1880, 1881, 1882, 1883, 1884, 1885, 1886, 1887, 1888, 1889, 1890, 1891, 1892, 1893, 1894, 1895, 1896, 1897, 1898, 1899, 1900, 1901, 1902, 1903, 1904, 1905, 1906, 1907, 1908, 1909, 1910, 1911, 1912, 1913, 1914, 1915, 1916, 1917, 1918, 1919, 1920, 1921, 1922, 1923, 1924, 1925, 1926, 1927, 1928, 1929, 1930, 1931, 1932, 1933, 1934, 1935, 1936, 1937, 1938, 1939, 1940, 1941, 1942, 1943]} +{"timestamp": 27000, "input_length": 954, "output_length": 48, "hash_ids": [0, 1944]} +{"timestamp": 27000, "input_length": 15852, "output_length": 629, "hash_ids": [0, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952, 1953, 1954, 1955, 1956, 1957, 1958, 1959, 1960, 1961, 1962, 1963, 1964, 1965, 1966, 1967, 1968, 1969, 1970, 1971, 1972, 1973, 1974]} +{"timestamp": 27000, "input_length": 9242, "output_length": 632, "hash_ids": [0, 1975, 1976, 1977, 1978, 1979, 1980, 1981, 1982, 1983, 1984, 1985, 1986, 1987, 1988, 1989, 1990, 1991, 1992]} +{"timestamp": 27000, "input_length": 12927, "output_length": 460, "hash_ids": [0, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017]} +{"timestamp": 27000, "input_length": 17799, "output_length": 345, "hash_ids": [0, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025, 2026, 2027, 2028, 2029, 2030, 2031, 2032, 2033, 2034, 2035, 2036, 2037, 2038, 2039, 2040, 2041, 2042, 2043, 2044, 2045, 2046, 2047, 2048, 2049, 2050, 2051]} +{"timestamp": 27000, "input_length": 8034, "output_length": 1, "hash_ids": [0, 2052, 2053, 2054, 2055, 2056, 2057, 2058, 2059, 2060, 2061, 2062, 2063, 2064, 2065, 2066]} +{"timestamp": 27000, "input_length": 10753, "output_length": 596, "hash_ids": [0, 2067, 2068, 2069, 2070, 2071, 2072, 2073, 2074, 2075, 2076, 2077, 2078, 2079, 2080, 2081, 2082, 2083, 2084, 2085, 2086, 2087]} +{"timestamp": 27000, "input_length": 4541, "output_length": 362, "hash_ids": [0, 2088, 2089, 2090, 2091, 2092, 2093, 2094, 2095]} +{"timestamp": 30000, "input_length": 42722, "output_length": 464, "hash_ids": [0, 2096, 2097, 2098, 2099, 2100, 2101, 2102, 2103, 2104, 2105, 2106, 2107, 2108, 2109, 2110, 2111, 2112, 2113, 2114, 2115, 2116, 2117, 2118, 2119, 2120, 2121, 2122, 2123, 2124, 2125, 2126, 2127, 2128, 2129, 2130, 2131, 2132, 2133, 2134, 2135, 2136, 2137, 2138, 2139, 2140, 2141, 2142, 2143, 2144, 2145, 2146, 2147, 2148, 2149, 2150, 2151, 2152, 2153, 2154, 2155, 2156, 2157, 2158, 2159, 2160, 2161, 2162, 2163, 2164, 2165, 2166, 2167, 2168, 2169, 2170, 2171, 2172, 2173, 2174, 2175, 2176, 2177, 2178]} +{"timestamp": 30000, "input_length": 10500, "output_length": 34, "hash_ids": [0, 2179, 2180, 2181, 2182, 2183, 2184, 2185, 2186, 2187, 2188, 2189, 2190, 2191, 2192, 2193, 2194, 2195, 2196, 2197, 2198]} +{"timestamp": 30000, "input_length": 13616, "output_length": 492, "hash_ids": [0, 2199, 2200, 2201, 2202, 2203, 2204, 2205, 2206, 2207, 2208, 2209, 2210, 2211, 2212, 2213, 2214, 2215, 2216, 2217, 2218, 2219, 2220, 2221, 2222, 2223, 2224]} +{"timestamp": 30000, "input_length": 49902, "output_length": 549, "hash_ids": [0, 2225, 2226, 2227, 2228, 2229, 2230, 2231, 2232, 2233, 2234, 2235, 2236, 2237, 2238, 2239, 2240, 2241, 2242, 2243, 2244, 2245, 2246, 2247, 2248, 2249, 2250, 2251, 2252, 2253, 2254, 2255, 2256, 2257, 2258, 2259, 2260, 2261, 2262, 2263, 2264, 2265, 2266, 2267, 2268, 2269, 2270, 2271, 2272, 2273, 2274, 2275, 2276, 2277, 2278, 2279, 2280, 2281, 2282, 2283, 2284, 2285, 2286, 2287, 2288, 2289, 2290, 2291, 2292, 2293, 2294, 2295, 2296, 2297, 2298, 2299, 2300, 2301, 2302, 2303, 2304, 2305, 2306, 2307, 2308, 2309, 2310, 2311, 2312, 2313, 2314, 2315, 2316, 2317, 2318, 2319, 2320, 2321]} +{"timestamp": 30000, "input_length": 17071, "output_length": 440, "hash_ids": [0, 2322, 2323, 2324, 2325, 2326, 2327, 2328, 2329, 2330, 2331, 2332, 2333, 2334, 2335, 2336, 2337, 2338, 2339, 2340, 2341, 2342, 2343, 2344, 2345, 2346, 2347, 2348, 2349, 2350, 2351, 2352, 2353, 2354]} +{"timestamp": 33000, "input_length": 21094, "output_length": 563, "hash_ids": [0, 2355, 2356, 2357, 2358, 2359, 2360, 2361, 2362, 2363, 2364, 2365, 2366, 2367, 2368, 2369, 2370, 2371, 2372, 2373, 2374, 2375, 2376, 2377, 2378, 2379, 2380, 2381, 2382, 2383, 2384, 2385, 2386, 2387, 2388, 2389, 2390, 2391, 2392, 2393, 2394, 2395]} +{"timestamp": 33000, "input_length": 29265, "output_length": 448, "hash_ids": [0, 2396, 2397, 2398, 2399, 2400, 2401, 2402, 2403, 2404, 2405, 2406, 2407, 2408, 2409, 2410, 2411, 2412, 2413, 2414, 2415, 2416, 2417, 2418, 2419, 2420, 2421, 2422, 2423, 2424, 2425, 2426, 2427, 2428, 2429, 2430, 2431, 2432, 2433, 2434, 2435, 2436, 2437, 2438, 2439, 2440, 2441, 2442, 2443, 2444, 2445, 2446, 2447, 2448, 2449, 2450, 2451, 2452]} +{"timestamp": 33000, "input_length": 17453, "output_length": 35, "hash_ids": [0, 2453, 2454, 2455, 2456, 2457, 2458, 2459, 2460, 2461, 2462, 2463, 2464, 2465, 2466, 2467, 2468, 2469, 2470, 2471, 2472, 2473, 2474, 2475, 2476, 2477, 2478, 2479, 2480, 2481, 2482, 2483, 2484, 2485, 2486]} +{"timestamp": 33000, "input_length": 82276, "output_length": 683, "hash_ids": [0, 2487, 2488, 2489, 2490, 2491, 2492, 2493, 2494, 2495, 2496, 2497, 2498, 2499, 2500, 2501, 2502, 2503, 2504, 2505, 2506, 2507, 2508, 2509, 2510, 2511, 2512, 2513, 2514, 2515, 2516, 2517, 2518, 2519, 2520, 2521, 2522, 2523, 2524, 2525, 2526, 2527, 2528, 2529, 2530, 2531, 2532, 2533, 2534, 2535, 2536, 2537, 2538, 2539, 2540, 2541, 2542, 2543, 2544, 2545, 2546, 2547, 2548, 2549, 2550, 2551, 2552, 2553, 2554, 2555, 2556, 2557, 2558, 2559, 2560, 2561, 2562, 2563, 2564, 2565, 2566, 2567, 2568, 2569, 2570, 2571, 2572, 2573, 2574, 2575, 2576, 2577, 2578, 2579, 2580, 2581, 2582, 2583, 2584, 2585, 2586, 2587, 2588, 2589, 2590, 2591, 2592, 2593, 2594, 2595, 2596, 2597, 2598, 2599, 2600, 2601, 2602, 2603, 2604, 2605, 2606, 2607, 2608, 2609, 2610, 2611, 2612, 2613, 2614, 2615, 2616, 2617, 2618, 2619, 2620, 2621, 2622, 2623, 2624, 2625, 2626, 2627, 2628, 2629, 2630, 2631, 2632, 2633, 2634, 2635, 2636, 2637, 2638, 2639, 2640, 2641, 2642, 2643, 2644, 2645, 2646]} +{"timestamp": 33000, "input_length": 26045, "output_length": 578, "hash_ids": [0, 2647, 2648, 2649, 2650, 2651, 2652, 2653, 2654, 2655, 2656, 2657, 2658, 2659, 2660, 2661, 2662, 2663, 2664, 2665, 2666, 2667, 2668, 2669, 2670, 2671, 2672, 2673, 2674, 2675, 2676, 2677, 2678, 2679, 2680, 2681, 2682, 2683, 2684, 2685, 2686, 2687, 2688, 2689, 2690, 2691, 2692, 2693, 2694, 2695, 2696]} +{"timestamp": 33000, "input_length": 120633, "output_length": 580, "hash_ids": [0, 2697, 2698, 2699, 2700, 2701, 2702, 2703, 2704, 2705, 2706, 2707, 2708, 2709, 2710, 2711, 2712, 2713, 2714, 2715, 2716, 2717, 2718, 2719, 2720, 2721, 2722, 2723, 2724, 2725, 2726, 2727, 2728, 2729, 2730, 2731, 2732, 2733, 2734, 2735, 2736, 2737, 2738, 2739, 2740, 2741, 2742, 2743, 2744, 2745, 2746, 2747, 2748, 2749, 2750, 2751, 2752, 2753, 2754, 2755, 2756, 2757, 2758, 2759, 2760, 2761, 2762, 2763, 2764, 2765, 2766, 2767, 2768, 2769, 2770, 2771, 2772, 2773, 2774, 2775, 2776, 2777, 2778, 2779, 2780, 2781, 2782, 2783, 2784, 2785, 2786, 2787, 2788, 2789, 2790, 2791, 2792, 2793, 2794, 2795, 2796, 2797, 2798, 2799, 2800, 2801, 2802, 2803, 2804, 2805, 2806, 2807, 2808, 2809, 2810, 2811, 2812, 2813, 2814, 2815, 2816, 2817, 2818, 2819, 2820, 2821, 2822, 2823, 2824, 2825, 2826, 2827, 2828, 2829, 2830, 2831, 2832, 2833, 2834, 2835, 2836, 2837, 2838, 2839, 2840, 2841, 2842, 2843, 2844, 2845, 2846, 2847, 2848, 2849, 2850, 2851, 2852, 2853, 2854, 2855, 2856, 2857, 2858, 2859, 2860, 2861, 2862, 2863, 2864, 2865, 2866, 2867, 2868, 2869, 2870, 2871, 2872, 2873, 2874, 2875, 2876, 2877, 2878, 2879, 2880, 2881, 2882, 2883, 2884, 2885, 2886, 2887, 2888, 2889, 2890, 2891, 2892, 2893, 2894, 2895, 2896, 2897, 2898, 2899, 2900, 2901, 2902, 2903, 2904, 2905, 2906, 2907, 2908, 2909, 2910, 2911, 2912, 2913, 2914, 2915, 2916, 2917, 2918, 2919, 2920, 2921, 2922, 2923, 2924, 2925, 2926, 2927, 2928, 2929, 2930, 2931]} +{"timestamp": 33000, "input_length": 1322, "output_length": 514, "hash_ids": [0, 2932, 2933]} +{"timestamp": 33000, "input_length": 916, "output_length": 265, "hash_ids": [0, 2934]} +{"timestamp": 36000, "input_length": 2885, "output_length": 613, "hash_ids": [0, 2935, 2936, 2937, 2938, 2939]} +{"timestamp": 36000, "input_length": 8619, "output_length": 593, "hash_ids": [0, 2940, 2941, 2942, 2943, 2944, 2945, 2946, 2947, 2948, 2949, 2950, 2951, 2952, 2953, 2954, 2955]} +{"timestamp": 36000, "input_length": 1729, "output_length": 31, "hash_ids": [0, 2956, 2957, 2958]} +{"timestamp": 36000, "input_length": 2428, "output_length": 508, "hash_ids": [0, 2959, 2960, 2961, 2962]} +{"timestamp": 36000, "input_length": 11527, "output_length": 373, "hash_ids": [0, 2963, 2964, 2965, 2966, 2967, 2968, 2969, 2970, 2971, 2972, 2973, 2974, 2975, 2976, 2977, 2978, 2979, 2980, 2981, 2982, 2983, 2984]} +{"timestamp": 36000, "input_length": 1087, "output_length": 505, "hash_ids": [0, 2985, 2986]} +{"timestamp": 36000, "input_length": 2392, "output_length": 621, "hash_ids": [0, 2987, 2988, 2989, 2990]} +{"timestamp": 36000, "input_length": 12000, "output_length": 529, "hash_ids": [0, 2991, 2992, 2993, 2994, 2995, 2996, 2997, 2998, 2999, 3000, 3001, 3002, 3003, 3004, 3005, 3006, 3007, 3008, 3009, 3010, 3011, 3012, 3013]} +{"timestamp": 39000, "input_length": 13546, "output_length": 17, "hash_ids": [0, 3014, 3015, 3016, 3017, 3018, 3019, 3020, 3021, 3022, 3023, 3024, 3025, 3026, 3027, 3028, 3029, 3030, 3031, 3032, 3033, 3034, 3035, 3036, 3037, 3038, 3039]} +{"timestamp": 39000, "input_length": 13689, "output_length": 358, "hash_ids": [0, 3040, 3041, 3042, 3043, 3044, 3045, 3046, 3047, 3048, 3049, 3050, 3051, 3052, 3053, 3054, 3055, 3056, 3057, 3058, 3059, 3060, 3061, 3062, 3063, 3064, 3065]} +{"timestamp": 39000, "input_length": 16464, "output_length": 182, "hash_ids": [0, 3066, 3067, 3068, 3069, 3070, 3071, 3072, 3073, 3074, 3075, 3076, 3077, 3078, 3079, 3080, 3081, 3082, 3083, 3084, 3085, 3086, 3087, 3088, 3089, 3090, 3091, 3092, 3093, 3094, 3095, 3096, 3097]} +{"timestamp": 39000, "input_length": 22808, "output_length": 463, "hash_ids": [0, 3098, 3099, 3100, 3101, 3102, 3103, 3104, 3105, 3106, 3107, 3108, 3109, 3110, 3111, 3112, 3113, 3114, 3115, 3116, 3117, 3118, 3119, 3120, 3121, 3122, 3123, 3124, 3125, 3126, 3127, 3128, 3129, 3130, 3131, 3132, 3133, 3134, 3135, 3136, 3137, 3138, 3139, 3140, 3141]} +{"timestamp": 39000, "input_length": 8487, "output_length": 306, "hash_ids": [0, 3142, 3143, 3144, 3145, 3146, 3147, 3148, 3149, 3150, 3151, 3152, 3153, 3154, 3155, 3156, 3157]} +{"timestamp": 39000, "input_length": 9227, "output_length": 160, "hash_ids": [0, 3158, 3159, 3160, 3161, 3162, 3163, 3164, 3165, 3166, 3167, 3168, 3169, 3170, 3171, 3172, 3173, 3174, 3175]} +{"timestamp": 39000, "input_length": 3535, "output_length": 228, "hash_ids": [0, 3176, 3177, 3178, 3179, 3180, 3181]} +{"timestamp": 39000, "input_length": 1206, "output_length": 223, "hash_ids": [0, 3182, 3183]} +{"timestamp": 39000, "input_length": 8520, "output_length": 232, "hash_ids": [0, 3184, 3185, 3186, 3187, 3188, 3189, 3190, 3191, 3192, 3193, 3194, 3195, 3196, 3197, 3198, 3199]} +{"timestamp": 39000, "input_length": 897, "output_length": 574, "hash_ids": [0, 3200]} +{"timestamp": 39000, "input_length": 23379, "output_length": 616, "hash_ids": [0, 3201, 3202, 3203, 3204, 3205, 3206, 3207, 3208, 3209, 3210, 3211, 3212, 3213, 3214, 3215, 3216, 3217, 3218, 3219, 3220, 3221, 3222, 3223, 3224, 3225, 3226, 3227, 3228, 3229, 3230, 3231, 3232, 3233, 3234, 3235, 3236, 3237, 3238, 3239, 3240, 3241, 3242, 3243, 3244, 3245]} +{"timestamp": 41999, "input_length": 87172, "output_length": 7, "hash_ids": [0, 3246, 3247, 3248, 3249, 3250, 3251, 3252, 3253, 3254, 3255, 3256, 3257, 3258, 3259, 3260, 3261, 3262, 3263, 3264, 3265, 3266, 3267, 3268, 3269, 3270, 3271, 3272, 3273, 3274, 3275, 3276, 3277, 3278, 3279, 3280, 3281, 3282, 3283, 3284, 3285, 3286, 3287, 3288, 3289, 3290, 3291, 3292, 3293, 3294, 3295, 3296, 3297, 3298, 3299, 3300, 3301, 3302, 3303, 3304, 3305, 3306, 3307, 3308, 3309, 3310, 3311, 3312, 3313, 3314, 3315, 3316, 3317, 3318, 3319, 3320, 3321, 3322, 3323, 3324, 3325, 3326, 3327, 3328, 3329, 3330, 3331, 3332, 3333, 3334, 3335, 3336, 3337, 3338, 3339, 3340, 3341, 3342, 3343, 3344, 3345, 3346, 3347, 3348, 3349, 3350, 3351, 3352, 3353, 3354, 3355, 3356, 3357, 3358, 3359, 3360, 3361, 3362, 3363, 3364, 3365, 3366, 3367, 3368, 3369, 3370, 3371, 3372, 3373, 3374, 3375, 3376, 3377, 3378, 3379, 3380, 3381, 3382, 3383, 3384, 3385, 3386, 3387, 3388, 3389, 3390, 3391, 3392, 3393, 3394, 3395, 3396, 3397, 3398, 3399, 3400, 3401, 3402, 3403, 3404, 3405, 3406, 3407, 3408, 3409, 3410, 3411, 3412, 3413, 3414, 3415]} +{"timestamp": 41999, "input_length": 5164, "output_length": 494, "hash_ids": [0, 3416, 3417, 3418, 3419, 3420, 3421, 3422, 3423, 3424, 3425]} +{"timestamp": 41999, "input_length": 6053, "output_length": 89, "hash_ids": [0, 3426, 3427, 3428, 3429, 3430, 3431, 3432, 3433, 3434, 3435, 3436]} +{"timestamp": 41999, "input_length": 15454, "output_length": 338, "hash_ids": [0, 3437, 3438, 3439, 3440, 3441, 3442, 3443, 3444, 3445, 3446, 3447, 3448, 3449, 3450, 3451, 3452, 3453, 3454, 3455, 3456, 3457, 3458, 3459, 3460, 3461, 3462, 3463, 3464, 3465, 3466]} +{"timestamp": 41999, "input_length": 69645, "output_length": 400, "hash_ids": [0, 3467, 3468, 3469, 3470, 3471, 3472, 3473, 3474, 3475, 3476, 3477, 3478, 3479, 3480, 3481, 3482, 3483, 3484, 3485, 3486, 3487, 3488, 3489, 3490, 3491, 3492, 3493, 3494, 3495, 3496, 3497, 3498, 3499, 3500, 3501, 3502, 3503, 3504, 3505, 3506, 3507, 3508, 3509, 3510, 3511, 3512, 3513, 3514, 3515, 3516, 3517, 3518, 3519, 3520, 3521, 3522, 3523, 3524, 3525, 3526, 3527, 3528, 3529, 3530, 3531, 3532, 3533, 3534, 3535, 3536, 3537, 3538, 3539, 3540, 3541, 3542, 3543, 3544, 3545, 3546, 3547, 3548, 3549, 3550, 3551, 3552, 3553, 3554, 3555, 3556, 3557, 3558, 3559, 3560, 3561, 3562, 3563, 3564, 3565, 3566, 3567, 3568, 3569, 3570, 3571, 3572, 3573, 3574, 3575, 3576, 3577, 3578, 3579, 3580, 3581, 3582, 3583, 3584, 3585, 3586, 3587, 3588, 3589, 3590, 3591, 3592, 3593, 3594, 3595, 3596, 3597, 3598, 3599, 3600, 3601, 3602]} +{"timestamp": 41999, "input_length": 13183, "output_length": 566, "hash_ids": [0, 3603, 3604, 3605, 3606, 3607, 3608, 3609, 3610, 3611, 3612, 3613, 3614, 3615, 3616, 3617, 3618, 3619, 3620, 3621, 3622, 3623, 3624, 3625, 3626, 3627]} +{"timestamp": 41999, "input_length": 1895, "output_length": 456, "hash_ids": [0, 3628, 3629, 3630]} +{"timestamp": 45000, "input_length": 907, "output_length": 396, "hash_ids": [0, 3631]} +{"timestamp": 45000, "input_length": 896, "output_length": 439, "hash_ids": [0, 3632]} +{"timestamp": 45000, "input_length": 27152, "output_length": 216, "hash_ids": [0, 3633, 3634, 3635, 3636, 3637, 3638, 3639, 3640, 3641, 3642, 3643, 3644, 3645, 3646, 3647, 3648, 3649, 3650, 3651, 3652, 3653, 3654, 3655, 3656, 3657, 3658, 3659, 3660, 3661, 3662, 3663, 3664, 3665, 3666, 3667, 3668, 3669, 3670, 3671, 3672, 3673, 3674, 3675, 3676, 3677, 3678, 3679, 3680, 3681, 3682, 3683, 3684, 3685]} +{"timestamp": 45000, "input_length": 6326, "output_length": 28, "hash_ids": [0, 3686, 3687, 3688, 3689, 3690, 3691, 3692, 3693, 3694, 3695, 3696, 3697]} +{"timestamp": 45000, "input_length": 2158, "output_length": 604, "hash_ids": [0, 3698, 3699, 3700, 3701]} +{"timestamp": 45000, "input_length": 11244, "output_length": 453, "hash_ids": [0, 3702, 3703, 3704, 3705, 3706, 3707, 3708, 3709, 3710, 3711, 3712, 3713, 3714, 3715, 3716, 3717, 3718, 3719, 3720, 3721, 3722]} +{"timestamp": 48000, "input_length": 15418, "output_length": 534, "hash_ids": [0, 3723, 3724, 3725, 3726, 3727, 3728, 3729, 3730, 3731, 3732, 3733, 3734, 3735, 3736, 3737, 3738, 3739, 3740, 3741, 3742, 3743, 3744, 3745, 3746, 3747, 3748, 3749, 3750, 3751, 3752]} +{"timestamp": 48000, "input_length": 3024, "output_length": 111, "hash_ids": [0, 1541, 1542, 1543, 1544, 3753]} +{"timestamp": 48000, "input_length": 49948, "output_length": 173, "hash_ids": [0, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 3754, 3755, 3756, 3757, 3758, 3759, 3760, 3761, 3762, 3763, 3764, 3765, 3766, 3767, 3768, 3769, 3770, 3771, 3772, 3773, 3774, 3775, 3776, 3777, 3778, 3779, 3780, 3781, 3782, 3783, 3784, 3785, 3786, 3787, 3788, 3789, 3790, 3791, 3792, 3793, 3794, 3795, 3796, 3797, 3798, 3799, 3800, 3801, 3802, 3803, 3804, 3805, 3806, 3807, 3808, 3809, 3810, 3811, 3812, 3813, 3814, 3815, 3816, 3817, 3818, 3819, 3820, 3821, 3822, 3823, 3824, 3825]} +{"timestamp": 48000, "input_length": 914, "output_length": 387, "hash_ids": [0, 3826]} +{"timestamp": 48000, "input_length": 21279, "output_length": 111, "hash_ids": [0, 3827, 3828, 3829, 3830, 3831, 3832, 3833, 3834, 3835, 3836, 3837, 3838, 3839, 3840, 3841, 3842, 3843, 3844, 3845, 3846, 3847, 3848, 3849, 3850, 3851, 3852, 3853, 3854, 3855, 3856, 3857, 3858, 3859, 3860, 3861, 3862, 3863, 3864, 3865, 3866, 3867]} +{"timestamp": 48000, "input_length": 7833, "output_length": 374, "hash_ids": [0, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 3868, 3869]} +{"timestamp": 48000, "input_length": 2014, "output_length": 10, "hash_ids": [0, 3870, 3871, 3872]} +{"timestamp": 48000, "input_length": 945, "output_length": 536, "hash_ids": [0, 3873]} +{"timestamp": 48000, "input_length": 2986, "output_length": 436, "hash_ids": [0, 3874, 3875, 3876, 3877, 3878]} +{"timestamp": 51000, "input_length": 1083, "output_length": 481, "hash_ids": [0, 3879, 3880]} +{"timestamp": 51000, "input_length": 13242, "output_length": 434, "hash_ids": [0, 3881, 3882, 3883, 3884, 3885, 3886, 3887, 3888, 3889, 3890, 3891, 3892, 3893, 3894, 3895, 3896, 3897, 3898, 3899, 3900, 3901, 3902, 3903, 3904, 3905]} +{"timestamp": 51000, "input_length": 41053, "output_length": 575, "hash_ids": [0, 3906, 3907, 3908, 3909, 3910, 3911, 3912, 3913, 3914, 3915, 3916, 3917, 3918, 3919, 3920, 3921, 3922, 3923, 3924, 3925, 3926, 3927, 3928, 3929, 3930, 3931, 3932, 3933, 3934, 3935, 3936, 3937, 3938, 3939, 3940, 3941, 3942, 3943, 3944, 3945, 3946, 3947, 3948, 3949, 3950, 3951, 3952, 3953, 3954, 3955, 3956, 3957, 3958, 3959, 3960, 3961, 3962, 3963, 3964, 3965, 3966, 3967, 3968, 3969, 3970, 3971, 3972, 3973, 3974, 3975, 3976, 3977, 3978, 3979, 3980, 3981, 3982, 3983, 3984, 3985]} +{"timestamp": 51000, "input_length": 24621, "output_length": 362, "hash_ids": [0, 3986, 3987, 3988, 3989, 3990, 3991, 3992, 3993, 3994, 3995, 3996, 3997, 3998, 3999, 4000, 4001, 4002, 4003, 4004, 4005, 4006, 4007, 4008, 4009, 4010, 4011, 4012, 4013, 4014, 4015, 4016, 4017, 4018, 4019, 4020, 4021, 4022, 4023, 4024, 4025, 4026, 4027, 4028, 4029, 4030, 4031, 4032, 4033]} +{"timestamp": 51000, "input_length": 898, "output_length": 906, "hash_ids": [0, 4034]} +{"timestamp": 51000, "input_length": 1085, "output_length": 75, "hash_ids": [0, 4035, 4036]} +{"timestamp": 51000, "input_length": 5654, "output_length": 442, "hash_ids": [0, 4037, 4038, 4039, 4040, 4041, 4042, 4043, 4044, 4045, 4046, 4047]} +{"timestamp": 51000, "input_length": 5939, "output_length": 15, "hash_ids": [0, 4048, 4049, 4050, 4051, 4052, 4053, 4054, 4055, 4056, 4057, 4058]} +{"timestamp": 54000, "input_length": 5100, "output_length": 413, "hash_ids": [0, 4059, 4060, 4061, 4062, 4063, 4064, 4065, 4066, 4067]} +{"timestamp": 54000, "input_length": 9421, "output_length": 17, "hash_ids": [0, 4068, 4069, 4070, 4071, 4072, 4073, 4074, 4075, 4076, 4077, 4078, 4079, 4080, 4081, 4082, 4083, 4084, 4085]} +{"timestamp": 54000, "input_length": 2187, "output_length": 186, "hash_ids": [0, 4086, 4087, 4088, 4089]} +{"timestamp": 54000, "input_length": 10704, "output_length": 346, "hash_ids": [0, 4090, 4091, 4092, 4093, 4094, 4095, 4096, 4097, 4098, 4099, 4100, 4101, 4102, 4103, 4104, 4105, 4106, 4107, 4108, 4109]} +{"timestamp": 54000, "input_length": 9359, "output_length": 339, "hash_ids": [0, 4110, 4111, 4112, 4113, 4114, 4115, 4116, 4117, 4118, 4119, 4120, 4121, 4122, 4123, 4124, 4125, 4126, 4127]} +{"timestamp": 54000, "input_length": 976, "output_length": 237, "hash_ids": [0, 4128]} +{"timestamp": 54000, "input_length": 18720, "output_length": 109, "hash_ids": [0, 4129, 4130, 4131, 4132, 4133, 4134, 4135, 4136, 4137, 4138, 4139, 4140, 4141, 4142, 4143, 4144, 4145, 4146, 4147, 4148, 4149, 4150, 4151, 4152, 4153, 4154, 4155, 4156, 4157, 4158, 4159, 4160, 4161, 4162, 4163, 4164]} +{"timestamp": 57000, "input_length": 2016, "output_length": 302, "hash_ids": [0, 4165, 4166, 4167]} +{"timestamp": 57000, "input_length": 6554, "output_length": 254, "hash_ids": [0, 4168, 4169, 4170, 4171, 4172, 4173, 4174, 4175, 4176, 4177, 4178, 4179]} +{"timestamp": 57000, "input_length": 917, "output_length": 21, "hash_ids": [0, 4180]} +{"timestamp": 57000, "input_length": 5682, "output_length": 35, "hash_ids": [0, 4181, 4182, 4183, 4184, 4185, 4186, 4187, 4188, 4189, 4190, 4191]} +{"timestamp": 57000, "input_length": 1902, "output_length": 720, "hash_ids": [0, 4192, 4193, 4194]} +{"timestamp": 57000, "input_length": 1383, "output_length": 722, "hash_ids": [0, 4195, 4196]} +{"timestamp": 60000, "input_length": 893, "output_length": 449, "hash_ids": [0, 4197]} +{"timestamp": 60000, "input_length": 11383, "output_length": 337, "hash_ids": [0, 4198, 4199, 4200, 4201, 4202, 4203, 4204, 4205, 4206, 4207, 4208, 4209, 4210, 4211, 4212, 4213, 4214, 4215, 4216, 4217, 4218, 4219]} +{"timestamp": 60000, "input_length": 2717, "output_length": 359, "hash_ids": [0, 4220, 4221, 4222, 4223, 4224]} +{"timestamp": 60000, "input_length": 12849, "output_length": 3, "hash_ids": [0, 4225, 4226, 4227, 4228, 4229, 4230, 4231, 4232, 4233, 4234, 4235, 4236, 4237, 4238, 4239, 4240, 4241, 4242, 4243, 4244, 4245, 4246, 4247, 4248, 4249]} +{"timestamp": 63000, "input_length": 19878, "output_length": 161, "hash_ids": [0, 1497, 1498, 1499, 1500, 1501, 1502, 1503, 1504, 1505, 1506, 1507, 1508, 1509, 1510, 1511, 1512, 1513, 1514, 1515, 1516, 1517, 1518, 1519, 1520, 1521, 1522, 1523, 1524, 1525, 1526, 1527, 1528, 1529, 1530, 1531, 1532, 1533, 4250]} +{"timestamp": 63000, "input_length": 895, "output_length": 133, "hash_ids": [0, 4251]} +{"timestamp": 63000, "input_length": 18306, "output_length": 575, "hash_ids": [0, 4252, 4253, 4254, 4255, 4256, 4257, 4258, 4259, 4260, 4261, 4262, 4263, 4264, 4265, 4266, 4267, 4268, 4269, 4270, 4271, 4272, 4273, 4274, 4275, 4276, 4277, 4278, 4279, 4280, 4281, 4282, 4283, 4284, 4285, 4286]} +{"timestamp": 63000, "input_length": 21641, "output_length": 467, "hash_ids": [0, 4287, 4288, 4289, 4290, 4291, 4292, 4293, 4294, 4295, 4296, 4297, 4298, 4299, 4300, 4301, 4302, 4303, 4304, 4305, 4306, 4307, 4308, 4309, 4310, 4311, 4312, 4313, 4314, 4315, 4316, 4317, 4318, 4319, 4320, 4321, 4322, 4323, 4324, 4325, 4326, 4327, 4328]} +{"timestamp": 63000, "input_length": 7103, "output_length": 48, "hash_ids": [0, 4329, 4330, 4331, 4332, 4333, 4334, 4335, 4336, 4337, 4338, 4339, 4340, 4341]} +{"timestamp": 65999, "input_length": 1529, "output_length": 513, "hash_ids": [0, 4342, 4343]} +{"timestamp": 65999, "input_length": 10938, "output_length": 247, "hash_ids": [0, 4344, 4345, 4346, 4347, 4348, 4349, 4350, 4351, 4352, 4353, 4354, 4355, 4356, 4357, 4358, 4359, 4360, 4361, 4362, 4363, 4364]} +{"timestamp": 65999, "input_length": 11954, "output_length": 463, "hash_ids": [0, 4365, 4366, 4367, 4368, 4369, 4370, 4371, 4372, 4373, 4374, 4375, 4376, 4377, 4378, 4379, 4380, 4381, 4382, 4383, 4384, 4385, 4386, 4387]} +{"timestamp": 65999, "input_length": 979, "output_length": 427, "hash_ids": [0, 4388]} +{"timestamp": 65999, "input_length": 7446, "output_length": 319, "hash_ids": [0, 4389, 4390, 4391, 4392, 4393, 4394, 4395, 4396, 4397, 4398, 4399, 4400, 4401, 4402]} +{"timestamp": 65999, "input_length": 1073, "output_length": 481, "hash_ids": [0, 4403, 4404]} +{"timestamp": 65999, "input_length": 14315, "output_length": 364, "hash_ids": [0, 1030, 1031, 1032, 1033, 1034, 1035, 1036, 1037, 1038, 1039, 1040, 1041, 1042, 1043, 1044, 1045, 1046, 4405, 4406, 4407, 4408, 4409, 4410, 4411, 4412, 4413, 4414]} +{"timestamp": 65999, "input_length": 102869, "output_length": 401, "hash_ids": [0, 4415, 4416, 4417, 4418, 4419, 4420, 4421, 4422, 4423, 4424, 4425, 4426, 4427, 4428, 4429, 4430, 4431, 4432, 4433, 4434, 4435, 4436, 4437, 4438, 4439, 4440, 4441, 4442, 4443, 4444, 4445, 4446, 4447, 4448, 4449, 4450, 4451, 4452, 4453, 4454, 4455, 4456, 4457, 4458, 4459, 4460, 4461, 4462, 4463, 4464, 4465, 4466, 4467, 4468, 4469, 4470, 4471, 4472, 4473, 4474, 4475, 4476, 4477, 4478, 4479, 4480, 4481, 4482, 4483, 4484, 4485, 4486, 4487, 4488, 4489, 4490, 4491, 4492, 4493, 4494, 4495, 4496, 4497, 4498, 4499, 4500, 4501, 4502, 4503, 4504, 4505, 4506, 4507, 4508, 4509, 4510, 4511, 4512, 4513, 4514, 4515, 4516, 4517, 4518, 4519, 4520, 4521, 4522, 4523, 4524, 4525, 4526, 4527, 4528, 4529, 4530, 4531, 4532, 4533, 4534, 4535, 4536, 4537, 4538, 4539, 4540, 4541, 4542, 4543, 4544, 4545, 4546, 4547, 4548, 4549, 4550, 4551, 4552, 4553, 4554, 4555, 4556, 4557, 4558, 4559, 4560, 4561, 4562, 4563, 4564, 4565, 4566, 4567, 4568, 4569, 4570, 4571, 4572, 4573, 4574, 4575, 4576, 4577, 4578, 4579, 4580, 4581, 4582, 4583, 4584, 4585, 4586, 4587, 4588, 4589, 4590, 4591, 4592, 4593, 4594, 4595, 4596, 4597, 4598, 4599, 4600, 4601, 4602, 4603, 4604, 4605, 4606, 4607, 4608, 4609, 4610, 4611, 4612, 4613, 4614]} +{"timestamp": 65999, "input_length": 75501, "output_length": 531, "hash_ids": [0, 4615, 4616, 4617, 4618, 4619, 4620, 4621, 4622, 4623, 4624, 4625, 4626, 4627, 4628, 4629, 4630, 4631, 4632, 4633, 4634, 4635, 4636, 4637, 4638, 4639, 4640, 4641, 4642, 4643, 4644, 4645, 4646, 4647, 4648, 4649, 4650, 4651, 4652, 4653, 4654, 4655, 4656, 4657, 4658, 4659, 4660, 4661, 4662, 4663, 4664, 4665, 4666, 4667, 4668, 4669, 4670, 4671, 4672, 4673, 4674, 4675, 4676, 4677, 4678, 4679, 4680, 4681, 4682, 4683, 4684, 4685, 4686, 4687, 4688, 4689, 4690, 4691, 4692, 4693, 4694, 4695, 4696, 4697, 4698, 4699, 4700, 4701, 4702, 4703, 4704, 4705, 4706, 4707, 4708, 4709, 4710, 4711, 4712, 4713, 4714, 4715, 4716, 4717, 4718, 4719, 4720, 4721, 4722, 4723, 4724, 4725, 4726, 4727, 4728, 4729, 4730, 4731, 4732, 4733, 4734, 4735, 4736, 4737, 4738, 4739, 4740, 4741, 4742, 4743, 4744, 4745, 4746, 4747, 4748, 4749, 4750, 4751, 4752, 4753, 4754, 4755, 4756, 4757, 4758, 4759, 4760, 4761]} +{"timestamp": 65999, "input_length": 15233, "output_length": 300, "hash_ids": [0, 978, 979, 980, 981, 982, 983, 984, 985, 986, 987, 988, 989, 990, 991, 992, 993, 994, 995, 996, 997, 998, 999, 1000, 1001, 1002, 1003, 4762, 4763, 4764]} +{"timestamp": 69000, "input_length": 10116, "output_length": 509, "hash_ids": [0, 4765, 4766, 4767, 4768, 4769, 4770, 4771, 4772, 4773, 4774, 4775, 4776, 4777, 4778, 4779, 4780, 4781, 4782, 4783]} +{"timestamp": 69000, "input_length": 4885, "output_length": 353, "hash_ids": [0, 4784, 4785, 4786, 4787, 4788, 4789, 4790, 4791, 4792]} +{"timestamp": 69000, "input_length": 4577, "output_length": 225, "hash_ids": [0, 4793, 4794, 4795, 4796, 4797, 4798, 4799, 4800]} +{"timestamp": 69000, "input_length": 893, "output_length": 334, "hash_ids": [0, 4801]} +{"timestamp": 69000, "input_length": 20509, "output_length": 5, "hash_ids": [0, 4802, 4803, 4804, 4805, 4806, 4807, 4808, 4809, 4810, 4811, 4812, 4813, 4814, 4815, 4816, 4817, 4818, 4819, 4820, 4821, 4822, 4823, 4824, 4825, 4826, 4827, 4828, 4829, 4830, 4831, 4832, 4833, 4834, 4835, 4836, 4837, 4838, 4839, 4840, 4841]} +{"timestamp": 69000, "input_length": 9921, "output_length": 356, "hash_ids": [0, 4842, 4843, 4844, 4845, 4846, 4847, 4848, 4849, 4850, 4851, 4852, 4853, 4854, 4855, 4856, 4857, 4858, 4859, 4860]} +{"timestamp": 69000, "input_length": 1176, "output_length": 120, "hash_ids": [0, 4861, 4862]} +{"timestamp": 69000, "input_length": 9271, "output_length": 306, "hash_ids": [0, 4863, 4864, 4865, 4866, 4867, 4868, 4869, 4870, 4871, 4872, 4873, 4874, 4875, 4876, 4877, 4878, 4879, 4880]} +{"timestamp": 69000, "input_length": 71938, "output_length": 433, "hash_ids": [0, 4881, 4882, 4883, 4884, 4885, 4886, 4887, 4888, 4889, 4890, 4891, 4892, 4893, 4894, 4895, 4896, 4897, 4898, 4899, 4900, 4901, 4902, 4903, 4904, 4905, 4906, 4907, 4908, 4909, 4910, 4911, 4912, 4913, 4914, 4915, 4916, 4917, 4918, 4919, 4920, 4921, 4922, 4923, 4924, 4925, 4926, 4927, 4928, 4929, 4930, 4931, 4932, 4933, 4934, 4935, 4936, 4937, 4938, 4939, 4940, 4941, 4942, 4943, 4944, 4945, 4946, 4947, 4948, 4949, 4950, 4951, 4952, 4953, 4954, 4955, 4956, 4957, 4958, 4959, 4960, 4961, 4962, 4963, 4964, 4965, 4966, 4967, 4968, 4969, 4970, 4971, 4972, 4973, 4974, 4975, 4976, 4977, 4978, 4979, 4980, 4981, 4982, 4983, 4984, 4985, 4986, 4987, 4988, 4989, 4990, 4991, 4992, 4993, 4994, 4995, 4996, 4997, 4998, 4999, 5000, 5001, 5002, 5003, 5004, 5005, 5006, 5007, 5008, 5009, 5010, 5011, 5012, 5013, 5014, 5015, 5016, 5017, 5018, 5019, 5020]} +{"timestamp": 69000, "input_length": 1341, "output_length": 128, "hash_ids": [0, 5021, 5022]} +{"timestamp": 69000, "input_length": 2103, "output_length": 508, "hash_ids": [0, 820, 5023, 5024, 5025]} +{"timestamp": 69000, "input_length": 16554, "output_length": 725, "hash_ids": [0, 5026, 5027, 5028, 5029, 5030, 5031, 5032, 5033, 5034, 5035, 5036, 5037, 5038, 5039, 5040, 5041, 5042, 5043, 5044, 5045, 5046, 5047, 5048, 5049, 5050, 5051, 5052, 5053, 5054, 5055, 5056, 5057]} +{"timestamp": 72000, "input_length": 17027, "output_length": 904, "hash_ids": [0, 5058, 5059, 5060, 5061, 5062, 5063, 5064, 5065, 5066, 5067, 5068, 5069, 5070, 5071, 5072, 5073, 5074, 5075, 5076, 5077, 5078, 5079, 5080, 5081, 5082, 5083, 5084, 5085, 5086, 5087, 5088, 5089, 5090]} +{"timestamp": 72000, "input_length": 1625, "output_length": 578, "hash_ids": [0, 5091, 5092, 5093]} +{"timestamp": 72000, "input_length": 1535, "output_length": 356, "hash_ids": [0, 5094, 5095]} +{"timestamp": 72000, "input_length": 8450, "output_length": 19, "hash_ids": [0, 5096, 5097, 5098, 5099, 5100, 5101, 5102, 5103, 5104, 5105, 5106, 5107, 5108, 5109, 5110, 5111]} +{"timestamp": 72000, "input_length": 16894, "output_length": 431, "hash_ids": [0, 5112, 5113, 5114, 5115, 5116, 5117, 5118, 5119, 5120, 5121, 5122, 5123, 5124, 5125, 5126, 5127, 5128, 5129, 5130, 5131, 5132, 5133, 5134, 5135, 5136, 5137, 5138, 5139, 5140, 5141, 5142, 5143]} +{"timestamp": 72000, "input_length": 16611, "output_length": 24, "hash_ids": [0, 5144, 5145, 5146, 5147, 5148, 5149, 5150, 5151, 5152, 5153, 5154, 5155, 5156, 5157, 5158, 5159, 5160, 5161, 5162, 5163, 5164, 5165, 5166, 5167, 5168, 5169, 5170, 5171, 5172, 5173, 5174, 5175]} +{"timestamp": 72000, "input_length": 19978, "output_length": 448, "hash_ids": [0, 5176, 5177, 5178, 5179, 5180, 5181, 5182, 5183, 5184, 5185, 5186, 5187, 5188, 5189, 5190, 5191, 5192, 5193, 5194, 5195, 5196, 5197, 5198, 5199, 5200, 5201, 5202, 5203, 5204, 5205, 5206, 5207, 5208, 5209, 5210, 5211, 5212, 5213, 5214]} +{"timestamp": 72000, "input_length": 1577, "output_length": 579, "hash_ids": [0, 5215, 5216, 5217]} +{"timestamp": 72000, "input_length": 9550, "output_length": 713, "hash_ids": [0, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, 456, 457, 458, 459, 460, 5218]} +{"timestamp": 72000, "input_length": 6517, "output_length": 565, "hash_ids": [0, 5219, 5220, 5221, 5222, 5223, 5224, 5225, 5226, 5227, 5228, 5229, 5230]} +{"timestamp": 74999, "input_length": 13538, "output_length": 480, "hash_ids": [0, 5231, 5232, 5233, 5234, 5235, 5236, 5237, 5238, 5239, 5240, 5241, 5242, 5243, 5244, 5245, 5246, 5247, 5248, 5249, 5250, 5251, 5252, 5253, 5254, 5255, 5256]} +{"timestamp": 74999, "input_length": 898, "output_length": 286, "hash_ids": [0, 5257]} +{"timestamp": 74999, "input_length": 6299, "output_length": 413, "hash_ids": [0, 5258, 5259, 5260, 5261, 5262, 5263, 5264, 5265, 5266, 5267, 5268, 5269]} +{"timestamp": 74999, "input_length": 8770, "output_length": 121, "hash_ids": [0, 5270, 5271, 5272, 5273, 5274, 5275, 5276, 5277, 5278, 5279, 5280, 5281, 5282, 5283, 5284, 5285, 5286]} +{"timestamp": 74999, "input_length": 38149, "output_length": 316, "hash_ids": [0, 5287, 5288, 5289, 5290, 5291, 5292, 5293, 5294, 5295, 5296, 5297, 5298, 5299, 5300, 5301, 5302, 5303, 5304, 5305, 5306, 5307, 5308, 5309, 5310, 5311, 5312, 5313, 5314, 5315, 5316, 5317, 5318, 5319, 5320, 5321, 5322, 5323, 5324, 5325, 5326, 5327, 5328, 5329, 5330, 5331, 5332, 5333, 5334, 5335, 5336, 5337, 5338, 5339, 5340, 5341, 5342, 5343, 5344, 5345, 5346, 5347, 5348, 5349, 5350, 5351, 5352, 5353, 5354, 5355, 5356, 5357, 5358, 5359, 5360]} +{"timestamp": 74999, "input_length": 5095, "output_length": 476, "hash_ids": [0, 5361, 5362, 5363, 5364, 5365, 5366, 5367, 5368, 5369]} +{"timestamp": 74999, "input_length": 6994, "output_length": 461, "hash_ids": [0, 5370, 5371, 5372, 5373, 5374, 5375, 5376, 5377, 5378, 5379, 5380, 5381, 5382]} +{"timestamp": 74999, "input_length": 37879, "output_length": 532, "hash_ids": [0, 5383, 5384, 5385, 5386, 5387, 5388, 5389, 5390, 5391, 5392, 5393, 5394, 5395, 5396, 5397, 5398, 5399, 5400, 5401, 5402, 5403, 5404, 5405, 5406, 5407, 5408, 5409, 5410, 5411, 5412, 5413, 5414, 5415, 5416, 5417, 5418, 5419, 5420, 5421, 5422, 5423, 5424, 5425, 5426, 5427, 5428, 5429, 5430, 5431, 5432, 5433, 5434, 5435, 5436, 5437, 5438, 5439, 5440, 5441, 5442, 5443, 5444, 5445, 5446, 5447, 5448, 5449, 5450, 5451, 5452, 5453, 5454, 5455]} +{"timestamp": 74999, "input_length": 17583, "output_length": 464, "hash_ids": [0, 5456, 5457, 5458, 5459, 5460, 5461, 5462, 5463, 5464, 5465, 5466, 5467, 5468, 5469, 5470, 5471, 5472, 5473, 5474, 5475, 5476, 5477, 5478, 5479, 5480, 5481, 5482, 5483, 5484, 5485, 5486, 5487, 5488, 5489]} +{"timestamp": 74999, "input_length": 17779, "output_length": 576, "hash_ids": [0, 5490, 5491, 5492, 5493, 5494, 5495, 5496, 5497, 5498, 5499, 5500, 5501, 5502, 5503, 5504, 5505, 5506, 5507, 5508, 5509, 5510, 5511, 5512, 5513, 5514, 5515, 5516, 5517, 5518, 5519, 5520, 5521, 5522, 5523]} +{"timestamp": 74999, "input_length": 14349, "output_length": 501, "hash_ids": [0, 5524, 5525, 5526, 5527, 5528, 5529, 5530, 5531, 5532, 5533, 5534, 5535, 5536, 5537, 5538, 5539, 5540, 5541, 5542, 5543, 5544, 5545, 5546, 5547, 5548, 5549, 5550, 5551]} +{"timestamp": 74999, "input_length": 8639, "output_length": 835, "hash_ids": [0, 5552, 5553, 5554, 5555, 5556, 5557, 5558, 5559, 5560, 5561, 5562, 5563, 5564, 5565, 5566, 5567]} +{"timestamp": 74999, "input_length": 921, "output_length": 452, "hash_ids": [0, 5568]} +{"timestamp": 74999, "input_length": 965, "output_length": 637, "hash_ids": [0, 5569]} +{"timestamp": 74999, "input_length": 13996, "output_length": 315, "hash_ids": [0, 5570, 5571, 5572, 5573, 5574, 5575, 5576, 5577, 5578, 5579, 5580, 5581, 5582, 5583, 5584, 5585, 5586, 5587, 5588, 5589, 5590, 5591, 5592, 5593, 5594, 5595, 5596]} +{"timestamp": 74999, "input_length": 6549, "output_length": 442, "hash_ids": [0, 611, 612, 613, 614, 615, 616, 617, 618, 619, 620, 5597, 5598]} +{"timestamp": 78000, "input_length": 51186, "output_length": 620, "hash_ids": [0, 5599, 5600, 5601, 5602, 5603, 5604, 5605, 5606, 5607, 5608, 5609, 5610, 5611, 5612, 5613, 5614, 5615, 5616, 5617, 5618, 5619, 5620, 5621, 5622, 5623, 5624, 5625, 5626, 5627, 5628, 5629, 5630, 5631, 5632, 5633, 5634, 5635, 5636, 5637, 5638, 5639, 5640, 5641, 5642, 5643, 5644, 5645, 5646, 5647, 5648, 5649, 5650, 5651, 5652, 5653, 5654, 5655, 5656, 5657, 5658, 5659, 5660, 5661, 5662, 5663, 5664, 5665, 5666, 5667, 5668, 5669, 5670, 5671, 5672, 5673, 5674, 5675, 5676, 5677, 5678, 5679, 5680, 5681, 5682, 5683, 5684, 5685, 5686, 5687, 5688, 5689, 5690, 5691, 5692, 5693, 5694, 5695, 5696, 5697]} +{"timestamp": 78000, "input_length": 16982, "output_length": 526, "hash_ids": [0, 655, 656, 657, 658, 659, 660, 661, 662, 663, 664, 665, 666, 667, 668, 669, 670, 671, 672, 673, 674, 675, 676, 677, 678, 679, 680, 681, 682, 5698, 5699, 5700, 5701, 5702]} +{"timestamp": 78000, "input_length": 26356, "output_length": 25, "hash_ids": [0, 5703, 5704, 5705, 5706, 5707, 5708, 5709, 5710, 5711, 5712, 5713, 5714, 5715, 5716, 5717, 5718, 5719, 5720, 5721, 5722, 5723, 5724, 5725, 5726, 5727, 5728, 5729, 5730, 5731, 5732, 5733, 5734, 5735, 5736, 5737, 5738, 5739, 5740, 5741, 5742, 5743, 5744, 5745, 5746, 5747, 5748, 5749, 5750, 5751, 5752, 5753]} +{"timestamp": 78000, "input_length": 45135, "output_length": 109, "hash_ids": [0, 5754, 5755, 5756, 5757, 5758, 5759, 5760, 5761, 5762, 5763, 5764, 5765, 5766, 5767, 5768, 5769, 5770, 5771, 5772, 5773, 5774, 5775, 5776, 5777, 5778, 5779, 5780, 5781, 5782, 5783, 5784, 5785, 5786, 5787, 5788, 5789, 5790, 5791, 5792, 5793, 5794, 5795, 5796, 5797, 5798, 5799, 5800, 5801, 5802, 5803, 5804, 5805, 5806, 5807, 5808, 5809, 5810, 5811, 5812, 5813, 5814, 5815, 5816, 5817, 5818, 5819, 5820, 5821, 5822, 5823, 5824, 5825, 5826, 5827, 5828, 5829, 5830, 5831, 5832, 5833, 5834, 5835, 5836, 5837, 5838, 5839, 5840, 5841]} +{"timestamp": 78000, "input_length": 5395, "output_length": 545, "hash_ids": [0, 5842, 5843, 5844, 5845, 5846, 5847, 5848, 5849, 5850, 5851]} +{"timestamp": 81000, "input_length": 1362, "output_length": 480, "hash_ids": [0, 5852, 5853]} +{"timestamp": 81000, "input_length": 22075, "output_length": 367, "hash_ids": [0, 5854, 5855, 5856, 5857, 5858, 5859, 5860, 5861, 5862, 5863, 5864, 5865, 5866, 5867, 5868, 5869, 5870, 5871, 5872, 5873, 5874, 5875, 5876, 5877, 5878, 5879, 5880, 5881, 5882, 5883, 5884, 5885, 5886, 5887, 5888, 5889, 5890, 5891, 5892, 5893, 5894, 5895, 5896]} +{"timestamp": 81000, "input_length": 6062, "output_length": 4, "hash_ids": [0, 5897, 5898, 5899, 5900, 5901, 5902, 5903, 5904, 5905, 5906, 5907]} +{"timestamp": 81000, "input_length": 1641, "output_length": 567, "hash_ids": [0, 5908, 5909, 5910]} +{"timestamp": 81000, "input_length": 13122, "output_length": 172, "hash_ids": [0, 3158, 3159, 3160, 3161, 5911, 5912, 5913, 5914, 5915, 5916, 5917, 5918, 5919, 5920, 5921, 5922, 5923, 5924, 5925, 5926, 5927, 5928, 5929, 5930, 5931]} +{"timestamp": 81000, "input_length": 4054, "output_length": 547, "hash_ids": [0, 5932, 5933, 5934, 5935, 5936, 5937, 5938]} +{"timestamp": 81000, "input_length": 906, "output_length": 28, "hash_ids": [0, 5939]} +{"timestamp": 81000, "input_length": 3709, "output_length": 300, "hash_ids": [0, 5940, 5941, 5942, 5943, 5944, 5945, 5946]} +{"timestamp": 81000, "input_length": 23998, "output_length": 645, "hash_ids": [0, 5947, 5948, 5949, 5950, 5951, 5952, 5953, 5954, 5955, 5956, 5957, 5958, 5959, 5960, 5961, 5962, 5963, 5964, 5965, 5966, 5967, 5968, 5969, 5970, 5971, 5972, 5973, 5974, 5975, 5976, 5977, 5978, 5979, 5980, 5981, 5982, 5983, 5984, 5985, 5986, 5987, 5988, 5989, 5990, 5991, 5992]} +{"timestamp": 81000, "input_length": 4535, "output_length": 521, "hash_ids": [0, 5993, 5994, 5995, 5996, 5997, 5998, 5999, 6000]} +{"timestamp": 81000, "input_length": 5956, "output_length": 17, "hash_ids": [0, 6001, 6002, 6003, 6004, 6005, 6006, 6007, 6008, 6009, 6010, 6011]} +{"timestamp": 81000, "input_length": 25174, "output_length": 284, "hash_ids": [0, 6012, 6013, 6014, 6015, 6016, 6017, 6018, 6019, 6020, 6021, 6022, 6023, 6024, 6025, 6026, 6027, 6028, 6029, 6030, 6031, 6032, 6033, 6034, 6035, 6036, 6037, 6038, 6039, 6040, 6041, 6042, 6043, 6044, 6045, 6046, 6047, 6048, 6049, 6050, 6051, 6052, 6053, 6054, 6055, 6056, 6057, 6058, 6059, 6060]} +{"timestamp": 84000, "input_length": 7911, "output_length": 510, "hash_ids": [0, 6061, 6062, 6063, 6064, 6065, 6066, 6067, 6068, 6069, 6070, 6071, 6072, 6073, 6074, 6075]} +{"timestamp": 84000, "input_length": 89867, "output_length": 556, "hash_ids": [0, 6076, 6077, 6078, 6079, 6080, 6081, 6082, 6083, 6084, 6085, 6086, 6087, 6088, 6089, 6090, 6091, 6092, 6093, 6094, 6095, 6096, 6097, 6098, 6099, 6100, 6101, 6102, 6103, 6104, 6105, 6106, 6107, 6108, 6109, 6110, 6111, 6112, 6113, 6114, 6115, 6116, 6117, 6118, 6119, 6120, 6121, 6122, 6123, 6124, 6125, 6126, 6127, 6128, 6129, 6130, 6131, 6132, 6133, 6134, 6135, 6136, 6137, 6138, 6139, 6140, 6141, 6142, 6143, 6144, 6145, 6146, 6147, 6148, 6149, 6150, 6151, 6152, 6153, 6154, 6155, 6156, 6157, 6158, 6159, 6160, 6161, 6162, 6163, 6164, 6165, 6166, 6167, 6168, 6169, 6170, 6171, 6172, 6173, 6174, 6175, 6176, 6177, 6178, 6179, 6180, 6181, 6182, 6183, 6184, 6185, 6186, 6187, 6188, 6189, 6190, 6191, 6192, 6193, 6194, 6195, 6196, 6197, 6198, 6199, 6200, 6201, 6202, 6203, 6204, 6205, 6206, 6207, 6208, 6209, 6210, 6211, 6212, 6213, 6214, 6215, 6216, 6217, 6218, 6219, 6220, 6221, 6222, 6223, 6224, 6225, 6226, 6227, 6228, 6229, 6230, 6231, 6232, 6233, 6234, 6235, 6236, 6237, 6238, 6239, 6240, 6241, 6242, 6243, 6244, 6245, 6246, 6247, 6248, 6249, 6250]} +{"timestamp": 84000, "input_length": 6299, "output_length": 165, "hash_ids": [0, 6251, 6252, 6253, 6254, 6255, 6256, 6257, 6258, 6259, 6260, 6261, 6262]} +{"timestamp": 84000, "input_length": 6809, "output_length": 196, "hash_ids": [0, 6263, 6264, 6265, 6266, 6267, 6268, 6269, 6270, 6271, 6272, 6273, 6274, 6275]} +{"timestamp": 84000, "input_length": 7190, "output_length": 26, "hash_ids": [0, 4329, 4330, 4331, 4332, 4333, 4334, 4335, 4336, 4337, 4338, 4339, 4340, 6276, 6277]} +{"timestamp": 84000, "input_length": 20529, "output_length": 362, "hash_ids": [0, 6278, 6279, 6280, 6281, 6282, 6283, 6284, 6285, 6286, 6287, 6288, 6289, 6290, 6291, 6292, 6293, 6294, 6295, 6296, 6297, 6298, 6299, 6300, 6301, 6302, 6303, 6304, 6305, 6306, 6307, 6308, 6309, 6310, 6311, 6312, 6313, 6314, 6315, 6316, 6317]} +{"timestamp": 84000, "input_length": 1372, "output_length": 427, "hash_ids": [0, 6318, 6319]} +{"timestamp": 84000, "input_length": 10089, "output_length": 421, "hash_ids": [0, 6320, 6321, 6322, 6323, 6324, 6325, 6326, 6327, 6328, 6329, 6330, 6331, 6332, 6333, 6334, 6335, 6336, 6337, 6338]} +{"timestamp": 84000, "input_length": 18081, "output_length": 428, "hash_ids": [0, 6339, 6340, 6341, 6342, 6343, 6344, 6345, 6346, 6347, 6348, 6349, 6350, 6351, 6352, 6353, 6354, 6355, 6356, 6357, 6358, 6359, 6360, 6361, 6362, 6363, 6364, 6365, 6366, 6367, 6368, 6369, 6370, 6371, 6372, 6373]} +{"timestamp": 84000, "input_length": 75162, "output_length": 456, "hash_ids": [0, 6374, 6375, 6376, 6377, 6378, 6379, 6380, 6381, 6382, 6383, 6384, 6385, 6386, 6387, 6388, 6389, 6390, 6391, 6392, 6393, 6394, 6395, 6396, 6397, 6398, 6399, 6400, 6401, 6402, 6403, 6404, 6405, 6406, 6407, 6408, 6409, 6410, 6411, 6412, 6413, 6414, 6415, 6416, 6417, 6418, 6419, 6420, 6421, 6422, 6423, 6424, 6425, 6426, 6427, 6428, 6429, 6430, 6431, 6432, 6433, 6434, 6435, 6436, 6437, 6438, 6439, 6440, 6441, 6442, 6443, 6444, 6445, 6446, 6447, 6448, 6449, 6450, 6451, 6452, 6453, 6454, 6455, 6456, 6457, 6458, 6459, 6460, 6461, 6462, 6463, 6464, 6465, 6466, 6467, 6468, 6469, 6470, 6471, 6472, 6473, 6474, 6475, 6476, 6477, 6478, 6479, 6480, 6481, 6482, 6483, 6484, 6485, 6486, 6487, 6488, 6489, 6490, 6491, 6492, 6493, 6494, 6495, 6496, 6497, 6498, 6499, 6500, 6501, 6502, 6503, 6504, 6505, 6506, 6507, 6508, 6509, 6510, 6511, 6512, 6513, 6514, 6515, 6516, 6517, 6518, 6519]} +{"timestamp": 86999, "input_length": 892, "output_length": 78, "hash_ids": [0, 6520]} +{"timestamp": 86999, "input_length": 11404, "output_length": 445, "hash_ids": [0, 2322, 6521, 6522, 6523, 6524, 6525, 6526, 6527, 6528, 6529, 6530, 6531, 6532, 6533, 6534, 6535, 6536, 6537, 6538, 6539, 6540, 6541]} +{"timestamp": 86999, "input_length": 7764, "output_length": 328, "hash_ids": [0, 6542, 6543, 6544, 6545, 6546, 6547, 6548, 6549, 6550, 6551, 6552, 6553, 6554, 6555, 6556]} +{"timestamp": 86999, "input_length": 14794, "output_length": 409, "hash_ids": [0, 6557, 6558, 6559, 6560, 6561, 6562, 6563, 6564, 6565, 6566, 6567, 6568, 6569, 6570, 6571, 6572, 6573, 6574, 6575, 6576, 6577, 6578, 6579, 6580, 6581, 6582, 6583, 6584]} +{"timestamp": 86999, "input_length": 1035, "output_length": 183, "hash_ids": [0, 6585, 6586]} +{"timestamp": 86999, "input_length": 19716, "output_length": 445, "hash_ids": [0, 6587, 6588, 6589, 6590, 6591, 6592, 6593, 6594, 6595, 6596, 6597, 6598, 6599, 6600, 6601, 6602, 6603, 6604, 6605, 6606, 6607, 6608, 6609, 6610, 6611, 6612, 6613, 6614, 6615, 6616, 6617, 6618, 6619, 6620, 6621, 6622, 6623, 6624]} +{"timestamp": 86999, "input_length": 1309, "output_length": 111, "hash_ids": [0, 6625, 6626]} +{"timestamp": 86999, "input_length": 2520, "output_length": 549, "hash_ids": [0, 6627, 6628, 6629, 6630]} +{"timestamp": 86999, "input_length": 16706, "output_length": 445, "hash_ids": [0, 6631, 6632, 6633, 6634, 6635, 6636, 6637, 6638, 6639, 6640, 6641, 6642, 6643, 6644, 6645, 6646, 6647, 6648, 6649, 6650, 6651, 6652, 6653, 6654, 6655, 6656, 6657, 6658, 6659, 6660, 6661, 6662]} +{"timestamp": 86999, "input_length": 1757, "output_length": 431, "hash_ids": [0, 6663, 6664, 6665]} +{"timestamp": 86999, "input_length": 11341, "output_length": 16, "hash_ids": [0, 6666, 6667, 6668, 6669, 6670, 6671, 6672, 6673, 6674, 6675, 6676, 6677, 6678, 6679, 6680, 6681, 6682, 6683, 6684, 6685, 6686, 6687]} +{"timestamp": 86999, "input_length": 906, "output_length": 28, "hash_ids": [0, 6688]} +{"timestamp": 86999, "input_length": 1887, "output_length": 312, "hash_ids": [0, 6689, 6690, 6691]} +{"timestamp": 86999, "input_length": 17869, "output_length": 881, "hash_ids": [0, 6692, 6693, 6694, 6695, 6696, 6697, 6698, 6699, 6700, 6701, 6702, 6703, 6704, 6705, 6706, 6707, 6708, 6709, 6710, 6711, 6712, 6713, 6714, 6715, 6716, 6717, 6718, 6719, 6720, 6721, 6722, 6723, 6724, 6725]} +{"timestamp": 90000, "input_length": 3954, "output_length": 297, "hash_ids": [0, 3874, 3875, 3876, 3877, 6726, 6727, 6728]} +{"timestamp": 90000, "input_length": 1902, "output_length": 654, "hash_ids": [0, 975, 976, 977]} +{"timestamp": 90000, "input_length": 27294, "output_length": 268, "hash_ids": [0, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1023, 1024, 1025, 1026, 6729, 6730, 6731, 6732, 6733, 6734, 6735, 6736, 6737, 6738, 6739, 6740, 6741, 6742, 6743, 6744, 6745, 6746, 6747, 6748, 6749, 6750, 6751, 6752, 6753, 6754, 6755, 6756, 6757, 6758, 6759]} +{"timestamp": 90000, "input_length": 9412, "output_length": 555, "hash_ids": [0, 6760, 6761, 6762, 6763, 6764, 6765, 6766, 6767, 6768, 6769, 6770, 6771, 6772, 6773, 6774, 6775, 6776, 6777]} +{"timestamp": 90000, "input_length": 12436, "output_length": 1644, "hash_ids": [0, 6778, 6779, 6780, 6781, 6782, 6783, 6784, 6785, 6786, 6787, 6788, 6789, 6790, 6791, 6792, 6793, 6794, 6795, 6796, 6797, 6798, 6799, 6800, 6801]} +{"timestamp": 90000, "input_length": 5240, "output_length": 534, "hash_ids": [0, 6802, 6803, 6804, 6805, 6806, 6807, 6808, 6809, 6810, 6811]} +{"timestamp": 90000, "input_length": 55009, "output_length": 644, "hash_ids": [0, 6812, 6813, 6814, 6815, 6816, 6817, 6818, 6819, 6820, 6821, 6822, 6823, 6824, 6825, 6826, 6827, 6828, 6829, 6830, 6831, 6832, 6833, 6834, 6835, 6836, 6837, 6838, 6839, 6840, 6841, 6842, 6843, 6844, 6845, 6846, 6847, 6848, 6849, 6850, 6851, 6852, 6853, 6854, 6855, 6856, 6857, 6858, 6859, 6860, 6861, 6862, 6863, 6864, 6865, 6866, 6867, 6868, 6869, 6870, 6871, 6872, 6873, 6874, 6875, 6876, 6877, 6878, 6879, 6880, 6881, 6882, 6883, 6884, 6885, 6886, 6887, 6888, 6889, 6890, 6891, 6892, 6893, 6894, 6895, 6896, 6897, 6898, 6899, 6900, 6901, 6902, 6903, 6904, 6905, 6906, 6907, 6908, 6909, 6910, 6911, 6912, 6913, 6914, 6915, 6916, 6917, 6918]} +{"timestamp": 90000, "input_length": 15174, "output_length": 13, "hash_ids": [0, 6919, 6920, 6921, 6922, 6923, 6924, 6925, 6926, 6927, 6928, 6929, 6930, 6931, 6932, 6933, 6934, 6935, 6936, 6937, 6938, 6939, 6940, 6941, 6942, 6943, 6944, 6945, 6946, 6947]} +{"timestamp": 90000, "input_length": 39419, "output_length": 243, "hash_ids": [0, 6948, 6949, 6950, 6951, 6952, 6953, 6954, 6955, 6956, 6957, 6958, 6959, 6960, 6961, 6962, 6963, 6964, 6965, 6966, 6967, 6968, 6969, 6970, 6971, 6972, 6973, 6974, 6975, 6976, 6977, 6978, 6979, 6980, 6981, 6982, 6983, 6984, 6985, 6986, 6987, 6988, 6989, 6990, 6991, 6992, 6993, 6994, 6995, 6996, 6997, 6998, 6999, 7000, 7001, 7002, 7003, 7004, 7005, 7006, 7007, 7008, 7009, 7010, 7011, 7012, 7013, 7014, 7015, 7016, 7017, 7018, 7019, 7020, 7021, 7022, 7023]} +{"timestamp": 93000, "input_length": 903, "output_length": 171, "hash_ids": [0, 7024]} +{"timestamp": 93000, "input_length": 2787, "output_length": 415, "hash_ids": [0, 7025, 7026, 7027, 7028, 7029]} +{"timestamp": 93000, "input_length": 9177, "output_length": 531, "hash_ids": [0, 7030, 7031, 7032, 7033, 7034, 7035, 7036, 7037, 7038, 7039, 7040, 7041, 7042, 7043, 7044, 7045, 7046]} +{"timestamp": 93000, "input_length": 36053, "output_length": 632, "hash_ids": [0, 7047, 7048, 7049, 7050, 7051, 7052, 7053, 7054, 7055, 7056, 7057, 7058, 7059, 7060, 7061, 7062, 7063, 7064, 7065, 7066, 7067, 7068, 7069, 7070, 7071, 7072, 7073, 7074, 7075, 7076, 7077, 7078, 7079, 7080, 7081, 7082, 7083, 7084, 7085, 7086, 7087, 7088, 7089, 7090, 7091, 7092, 7093, 7094, 7095, 7096, 7097, 7098, 7099, 7100, 7101, 7102, 7103, 7104, 7105, 7106, 7107, 7108, 7109, 7110, 7111, 7112, 7113, 7114, 7115, 7116]} +{"timestamp": 95999, "input_length": 13883, "output_length": 775, "hash_ids": [0, 7117, 7118, 7119, 7120, 7121, 7122, 7123, 7124, 7125, 7126, 7127, 7128, 7129, 7130, 7131, 7132, 7133, 7134, 7135, 7136, 7137, 7138, 7139, 7140, 7141, 7142, 7143]} +{"timestamp": 95999, "input_length": 8767, "output_length": 449, "hash_ids": [0, 7144, 7145, 7146, 7147, 7148, 7149, 7150, 7151, 7152, 7153, 7154, 7155, 7156, 7157, 7158, 7159, 7160]} +{"timestamp": 95999, "input_length": 7736, "output_length": 347, "hash_ids": [0, 7161, 7162, 7163, 7164, 7165, 7166, 7167, 7168, 7169, 7170, 7171, 7172, 7173, 7174, 7175]} +{"timestamp": 95999, "input_length": 13742, "output_length": 304, "hash_ids": [0, 7176, 7177, 7178, 7179, 7180, 7181, 7182, 7183, 7184, 7185, 7186, 7187, 7188, 7189, 7190, 7191, 7192, 7193, 7194, 7195, 7196, 7197, 7198, 7199, 7200, 7201]} +{"timestamp": 95999, "input_length": 897, "output_length": 94, "hash_ids": [0, 7202]} +{"timestamp": 95999, "input_length": 45924, "output_length": 24, "hash_ids": [0, 7203, 7204, 7205, 7206, 7207, 7208, 7209, 7210, 7211, 7212, 7213, 7214, 7215, 7216, 7217, 7218, 7219, 7220, 7221, 7222, 7223, 7224, 7225, 7226, 7227, 7228, 7229, 7230, 7231, 7232, 7233, 7234, 7235, 7236, 7237, 7238, 7239, 7240, 7241, 7242, 7243, 7244, 7245, 7246, 7247, 7248, 7249, 7250, 7251, 7252, 7253, 7254, 7255, 7256, 7257, 7258, 7259, 7260, 7261, 7262, 7263, 7264, 7265, 7266, 7267, 7268, 7269, 7270, 7271, 7272, 7273, 7274, 7275, 7276, 7277, 7278, 7279, 7280, 7281, 7282, 7283, 7284, 7285, 7286, 7287, 7288, 7289, 7290, 7291]} +{"timestamp": 95999, "input_length": 7617, "output_length": 435, "hash_ids": [0, 7292, 7293, 7294, 7295, 7296, 7297, 7298, 7299, 7300, 7301, 7302, 7303, 7304, 7305]} +{"timestamp": 95999, "input_length": 3142, "output_length": 398, "hash_ids": [0, 1541, 1542, 1543, 1544, 7306, 7307]} +{"timestamp": 95999, "input_length": 24132, "output_length": 2000, "hash_ids": [0, 7308, 7309, 7310, 7311, 7312, 7313, 7314, 7315, 7316, 7317, 7318, 7319, 7320, 7321, 7322, 7323, 7324, 7325, 7326, 7327, 7328, 7329, 7330, 7331, 7332, 7333, 7334, 7335, 7336, 7337, 7338, 7339, 7340, 7341, 7342, 7343, 7344, 7345, 7346, 7347, 7348, 7349, 7350, 7351, 7352, 7353, 7354]} +{"timestamp": 99000, "input_length": 4701, "output_length": 446, "hash_ids": [0, 7355, 7356, 7357, 7358, 7359, 7360, 7361, 7362, 7363]} +{"timestamp": 99000, "input_length": 19085, "output_length": 247, "hash_ids": [0, 7364, 7365, 7366, 7367, 7368, 7369, 7370, 7371, 7372, 7373, 7374, 7375, 7376, 7377, 7378, 7379, 7380, 7381, 7382, 7383, 7384, 7385, 7386, 7387, 7388, 7389, 7390, 7391, 7392, 7393, 7394, 7395, 7396, 7397, 7398, 7399, 7400]} +{"timestamp": 99000, "input_length": 899, "output_length": 418, "hash_ids": [0, 7401]} +{"timestamp": 99000, "input_length": 6603, "output_length": 20, "hash_ids": [0, 7402, 7403, 7404, 7405, 7406, 7407, 7408, 7409, 7410, 7411, 7412, 7413]} +{"timestamp": 99000, "input_length": 50465, "output_length": 621, "hash_ids": [0, 2225, 2226, 2227, 2228, 2229, 2230, 2231, 2232, 2233, 2234, 2235, 2236, 2237, 2238, 2239, 2240, 2241, 2242, 2243, 2244, 2245, 2246, 2247, 2248, 2249, 2250, 2251, 2252, 2253, 2254, 2255, 2256, 2257, 2258, 2259, 2260, 2261, 2262, 2263, 2264, 2265, 2266, 2267, 2268, 2269, 2270, 2271, 2272, 2273, 2274, 2275, 2276, 2277, 2278, 2279, 2280, 2281, 2282, 2283, 2284, 2285, 2286, 2287, 2288, 2289, 2290, 2291, 2292, 2293, 2294, 2295, 2296, 2297, 2298, 2299, 2300, 2301, 2302, 2303, 2304, 2305, 2306, 2307, 2308, 2309, 2310, 2311, 2312, 2313, 2314, 2315, 2316, 2317, 2318, 2319, 2320, 7414, 7415]} +{"timestamp": 99000, "input_length": 3025, "output_length": 673, "hash_ids": [0, 7416, 7417, 7418, 7419, 7420]} +{"timestamp": 99000, "input_length": 914, "output_length": 275, "hash_ids": [0, 7421]} +{"timestamp": 99000, "input_length": 14434, "output_length": 385, "hash_ids": [0, 1469, 1470, 1471, 1472, 1473, 1474, 1475, 1476, 1477, 1478, 1479, 1480, 1481, 1482, 1483, 1484, 1485, 1486, 1487, 1488, 1489, 1490, 1491, 1492, 1493, 1494, 7422, 7423]} +{"timestamp": 99000, "input_length": 17188, "output_length": 595, "hash_ids": [0, 7424, 7425, 7426, 7427, 7428, 7429, 7430, 7431, 7432, 7433, 7434, 7435, 7436, 7437, 7438, 7439, 7440, 7441, 7442, 7443, 7444, 7445, 7446, 7447, 7448, 7449, 7450, 7451, 7452, 7453, 7454, 7455, 7456]} +{"timestamp": 102000, "input_length": 67220, "output_length": 358, "hash_ids": [0, 7457, 7458, 7459, 7460, 7461, 7462, 7463, 7464, 7465, 7466, 7467, 7468, 7469, 7470, 7471, 7472, 7473, 7474, 7475, 7476, 7477, 7478, 7479, 7480, 7481, 7482, 7483, 7484, 7485, 7486, 7487, 7488, 7489, 7490, 7491, 7492, 7493, 7494, 7495, 7496, 7497, 7498, 7499, 7500, 7501, 7502, 7503, 7504, 7505, 7506, 7507, 7508, 7509, 7510, 7511, 7512, 7513, 7514, 7515, 7516, 7517, 7518, 7519, 7520, 7521, 7522, 7523, 7524, 7525, 7526, 7527, 7528, 7529, 7530, 7531, 7532, 7533, 7534, 7535, 7536, 7537, 7538, 7539, 7540, 7541, 7542, 7543, 7544, 7545, 7546, 7547, 7548, 7549, 7550, 7551, 7552, 7553, 7554, 7555, 7556, 7557, 7558, 7559, 7560, 7561, 7562, 7563, 7564, 7565, 7566, 7567, 7568, 7569, 7570, 7571, 7572, 7573, 7574, 7575, 7576, 7577, 7578, 7579, 7580, 7581, 7582, 7583, 7584, 7585, 7586, 7587]} +{"timestamp": 102000, "input_length": 1056, "output_length": 3, "hash_ids": [0, 7588, 7589]} +{"timestamp": 102000, "input_length": 10275, "output_length": 269, "hash_ids": [0, 7590, 7591, 7592, 7593, 7594, 7595, 7596, 7597, 7598, 7599, 7600, 7601, 7602, 7603, 7604, 7605, 7606, 7607, 7608, 7609]} +{"timestamp": 102000, "input_length": 13630, "output_length": 510, "hash_ids": [0, 1634, 1635, 1636, 1637, 1638, 1639, 1640, 1641, 1642, 1643, 1644, 1645, 1646, 1647, 1648, 1649, 1650, 1651, 1652, 1653, 1654, 1655, 7610, 7611, 7612, 7613]} +{"timestamp": 102000, "input_length": 8659, "output_length": 831, "hash_ids": [0, 7614, 7615, 7616, 7617, 7618, 7619, 7620, 7621, 7622, 7623, 7624, 7625, 7626, 7627, 7628, 7629]} +{"timestamp": 102000, "input_length": 24495, "output_length": 306, "hash_ids": [0, 7630, 7631, 7632, 7633, 7634, 7635, 7636, 7637, 7638, 7639, 7640, 7641, 7642, 7643, 7644, 7645, 7646, 7647, 7648, 7649, 7650, 7651, 7652, 7653, 7654, 7655, 7656, 7657, 7658, 7659, 7660, 7661, 7662, 7663, 7664, 7665, 7666, 7667, 7668, 7669, 7670, 7671, 7672, 7673, 7674, 7675, 7676]} +{"timestamp": 102000, "input_length": 69282, "output_length": 430, "hash_ids": [0, 7677, 7678, 7679, 7680, 7681, 7682, 7683, 7684, 7685, 7686, 7687, 7688, 7689, 7690, 7691, 7692, 7693, 7694, 7695, 7696, 7697, 7698, 7699, 7700, 7701, 7702, 7703, 7704, 7705, 7706, 7707, 7708, 7709, 7710, 7711, 7712, 7713, 7714, 7715, 7716, 7717, 7718, 7719, 7720, 7721, 7722, 7723, 7724, 7725, 7726, 7727, 7728, 7729, 7730, 7731, 7732, 7733, 7734, 7735, 7736, 7737, 7738, 7739, 7740, 7741, 7742, 7743, 7744, 7745, 7746, 7747, 7748, 7749, 7750, 7751, 7752, 7753, 7754, 7755, 7756, 7757, 7758, 7759, 7760, 7761, 7762, 7763, 7764, 7765, 7766, 7767, 7768, 7769, 7770, 7771, 7772, 7773, 7774, 7775, 7776, 7777, 7778, 7779, 7780, 7781, 7782, 7783, 7784, 7785, 7786, 7787, 7788, 7789, 7790, 7791, 7792, 7793, 7794, 7795, 7796, 7797, 7798, 7799, 7800, 7801, 7802, 7803, 7804, 7805, 7806, 7807, 7808, 7809, 7810, 7811]} +{"timestamp": 102000, "input_length": 3378, "output_length": 617, "hash_ids": [0, 1535, 1536, 1537, 1538, 1539, 7812]} +{"timestamp": 102000, "input_length": 979, "output_length": 140, "hash_ids": [0, 7813]} +{"timestamp": 102000, "input_length": 4743, "output_length": 541, "hash_ids": [0, 7814, 7815, 7816, 7817, 7818, 7819, 7820, 7821, 7822]} +{"timestamp": 102000, "input_length": 5714, "output_length": 5, "hash_ids": [0, 7823, 7824, 7825, 7826, 7827, 7828, 7829, 7830, 7831, 7832, 7833]} +{"timestamp": 105000, "input_length": 28831, "output_length": 634, "hash_ids": [0, 1657, 1658, 1659, 1660, 1661, 1662, 1663, 1664, 1665, 1666, 1667, 1668, 1669, 1670, 1671, 1672, 1673, 1674, 1675, 1676, 1677, 1678, 1679, 1680, 1681, 1682, 1683, 1684, 1685, 1686, 1687, 1688, 1689, 1690, 1691, 1692, 1693, 1694, 1695, 1696, 1697, 1698, 1699, 1700, 1701, 1702, 1703, 1704, 1705, 1706, 1707, 1708, 1709, 1710, 7834, 7835]} +{"timestamp": 105000, "input_length": 11850, "output_length": 382, "hash_ids": [0, 4110, 4111, 4112, 4113, 4114, 4115, 4116, 4117, 4118, 4119, 4120, 4121, 4122, 4123, 4124, 4125, 4126, 7836, 7837, 7838, 7839, 7840, 7841]} +{"timestamp": 105000, "input_length": 10357, "output_length": 35, "hash_ids": [0, 7842, 7843, 7844, 7845, 7846, 7847, 7848, 7849, 7850, 7851, 7852, 7853, 7854, 7855, 7856, 7857, 7858, 7859, 7860, 7861]} +{"timestamp": 105000, "input_length": 21398, "output_length": 384, "hash_ids": [0, 3827, 3828, 3829, 3830, 3831, 3832, 3833, 3834, 3835, 3836, 3837, 3838, 3839, 3840, 3841, 3842, 3843, 3844, 3845, 3846, 3847, 3848, 3849, 3850, 3851, 3852, 3853, 3854, 3855, 3856, 3857, 3858, 3859, 3860, 3861, 3862, 3863, 3864, 3865, 3866, 7862]} +{"timestamp": 105000, "input_length": 3944, "output_length": 515, "hash_ids": [0, 7863, 7864, 7865, 7866, 7867, 7868, 7869]} +{"timestamp": 108000, "input_length": 8532, "output_length": 38, "hash_ids": [0, 5096, 5097, 5098, 5099, 5100, 5101, 5102, 5103, 5104, 5105, 5106, 5107, 5108, 5109, 5110, 7870]} +{"timestamp": 108000, "input_length": 26728, "output_length": 370, "hash_ids": [0, 560, 561, 562, 563, 564, 565, 566, 567, 568, 569, 570, 571, 572, 573, 574, 575, 576, 577, 578, 579, 580, 581, 582, 583, 584, 585, 586, 587, 588, 589, 590, 591, 592, 593, 594, 595, 596, 597, 598, 599, 600, 601, 602, 603, 604, 605, 606, 607, 608, 609, 7871, 7872]} +{"timestamp": 108000, "input_length": 12588, "output_length": 387, "hash_ids": [0, 7873, 7874, 7875, 7876, 7877, 7878, 7879, 7880, 7881, 7882, 7883, 7884, 7885, 7886, 7887, 7888, 7889, 7890, 7891, 7892, 7893, 7894, 7895, 7896]} +{"timestamp": 108000, "input_length": 1661, "output_length": 93, "hash_ids": [0, 5021, 7897, 7898]} +{"timestamp": 108000, "input_length": 3510, "output_length": 760, "hash_ids": [0, 2935, 2936, 2937, 2938, 7899, 7900]} +{"timestamp": 108000, "input_length": 20187, "output_length": 124, "hash_ids": [0, 7901, 7902, 7903, 7904, 7905, 7906, 7907, 7908, 7909, 7910, 7911, 7912, 7913, 7914, 7915, 7916, 7917, 7918, 7919, 7920, 7921, 7922, 7923, 7924, 7925, 7926, 7927, 7928, 7929, 7930, 7931, 7932, 7933, 7934, 7935, 7936, 7937, 7938, 7939]} +{"timestamp": 108000, "input_length": 14980, "output_length": 32, "hash_ids": [0, 7940, 7941, 7942, 7943, 7944, 7945, 7946, 7947, 7948, 7949, 7950, 7951, 7952, 7953, 7954, 7955, 7956, 7957, 7958, 7959, 7960, 7961, 7962, 7963, 7964, 7965, 7966, 7967, 7968]} +{"timestamp": 108000, "input_length": 13695, "output_length": 527, "hash_ids": [0, 7969, 7970, 7971, 7972, 7973, 7974, 7975, 7976, 7977, 7978, 7979, 7980, 7981, 7982, 7983, 7984, 7985, 7986, 7987, 7988, 7989, 7990, 7991, 7992, 7993, 7994]} +{"timestamp": 108000, "input_length": 8058, "output_length": 457, "hash_ids": [0, 2052, 2053, 2054, 2055, 2056, 2057, 2058, 2059, 2060, 2061, 2062, 2063, 2064, 2065, 7995]} +{"timestamp": 111000, "input_length": 9325, "output_length": 644, "hash_ids": [0, 7996, 7997, 7998, 7999, 8000, 8001, 8002, 8003, 8004, 8005, 8006, 8007, 8008, 8009, 8010, 8011, 8012, 8013]} +{"timestamp": 111000, "input_length": 1479, "output_length": 41, "hash_ids": [0, 8014, 8015]} +{"timestamp": 111000, "input_length": 14314, "output_length": 57, "hash_ids": [0, 8016, 8017, 8018, 8019, 8020, 8021, 8022, 8023, 8024, 8025, 8026, 8027, 8028, 8029, 8030, 8031, 8032, 8033, 8034, 8035, 8036, 8037, 8038, 8039, 8040, 8041, 8042]} +{"timestamp": 111000, "input_length": 1060, "output_length": 60, "hash_ids": [0, 8043, 8044]} +{"timestamp": 111000, "input_length": 9535, "output_length": 529, "hash_ids": [0, 8045, 8046, 8047, 8048, 8049, 8050, 8051, 8052, 8053, 8054, 8055, 8056, 8057, 8058, 8059, 8060, 8061, 8062]} +{"timestamp": 111000, "input_length": 33990, "output_length": 521, "hash_ids": [0, 3098, 3099, 3100, 3101, 3102, 3103, 3104, 3105, 3106, 3107, 3108, 3109, 3110, 3111, 3112, 3113, 3114, 3115, 3116, 3117, 3118, 3119, 3120, 3121, 3122, 3123, 3124, 3125, 3126, 3127, 3128, 3129, 3130, 3131, 3132, 3133, 3134, 3135, 3136, 3137, 3138, 3139, 3140, 8063, 8064, 8065, 8066, 8067, 8068, 8069, 8070, 8071, 8072, 8073, 8074, 8075, 8076, 8077, 8078, 8079, 8080, 8081, 8082, 8083, 8084, 8085]} +{"timestamp": 111000, "input_length": 5240, "output_length": 111, "hash_ids": [0, 6802, 6803, 6804, 6805, 6806, 6807, 6808, 6809, 6810, 6811]} +{"timestamp": 111000, "input_length": 23983, "output_length": 553, "hash_ids": [0, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 8086, 8087, 8088, 8089, 8090, 8091, 8092, 8093, 8094, 8095, 8096, 8097, 8098, 8099, 8100, 8101, 8102]} +{"timestamp": 111000, "input_length": 6886, "output_length": 366, "hash_ids": [0, 8103, 8104, 8105, 8106, 8107, 8108, 8109, 8110, 8111, 8112, 8113, 8114, 8115]} +{"timestamp": 113999, "input_length": 10602, "output_length": 123, "hash_ids": [0, 8116, 8117, 8118, 8119, 8120, 8121, 8122, 8123, 8124, 8125, 8126, 8127, 8128, 8129, 8130, 8131, 8132, 8133, 8134, 8135]} +{"timestamp": 113999, "input_length": 115635, "output_length": 271, "hash_ids": [0, 8136, 8137, 8138, 8139, 8140, 8141, 8142, 8143, 8144, 8145, 8146, 8147, 8148, 8149, 8150, 8151, 8152, 8153, 8154, 8155, 8156, 8157, 8158, 8159, 8160, 8161, 8162, 8163, 8164, 8165, 8166, 8167, 8168, 8169, 8170, 8171, 8172, 8173, 8174, 8175, 8176, 8177, 8178, 8179, 8180, 8181, 8182, 8183, 8184, 8185, 8186, 8187, 8188, 8189, 8190, 8191, 8192, 8193, 8194, 8195, 8196, 8197, 8198, 8199, 8200, 8201, 8202, 8203, 8204, 8205, 8206, 8207, 8208, 8209, 8210, 8211, 8212, 8213, 8214, 8215, 8216, 8217, 8218, 8219, 8220, 8221, 8222, 8223, 8224, 8225, 8226, 8227, 8228, 8229, 8230, 8231, 8232, 8233, 8234, 8235, 8236, 8237, 8238, 8239, 8240, 8241, 8242, 8243, 8244, 8245, 8246, 8247, 8248, 8249, 8250, 8251, 8252, 8253, 8254, 8255, 8256, 8257, 8258, 8259, 8260, 8261, 8262, 8263, 8264, 8265, 8266, 8267, 8268, 8269, 8270, 8271, 8272, 8273, 8274, 8275, 8276, 8277, 8278, 8279, 8280, 8281, 8282, 8283, 8284, 8285, 8286, 8287, 8288, 8289, 8290, 8291, 8292, 8293, 8294, 8295, 8296, 8297, 8298, 8299, 8300, 8301, 8302, 8303, 8304, 8305, 8306, 8307, 8308, 8309, 8310, 8311, 8312, 8313, 8314, 8315, 8316, 8317, 8318, 8319, 8320, 8321, 8322, 8323, 8324, 8325, 8326, 8327, 8328, 8329, 8330, 8331, 8332, 8333, 8334, 8335, 8336, 8337, 8338, 8339, 8340, 8341, 8342, 8343, 8344, 8345, 8346, 8347, 8348, 8349, 8350, 8351, 8352, 8353, 8354, 8355, 8356, 8357, 8358, 8359, 8360]} +{"timestamp": 113999, "input_length": 24829, "output_length": 614, "hash_ids": [0, 1261, 1262, 1263, 1264, 1265, 1266, 1267, 1268, 1269, 1270, 1271, 1272, 1273, 1274, 1275, 1276, 1277, 1278, 1279, 1280, 1281, 1282, 1283, 1284, 1285, 1286, 1287, 1288, 1289, 1290, 1291, 1292, 1293, 1294, 1295, 1296, 1297, 1298, 1299, 1300, 1301, 1302, 1303, 1304, 8361, 8362, 8363, 8364]} +{"timestamp": 113999, "input_length": 40536, "output_length": 356, "hash_ids": [0, 8365, 8366, 8367, 8368, 8369, 8370, 8371, 8372, 8373, 8374, 8375, 8376, 8377, 8378, 8379, 8380, 8381, 8382, 8383, 8384, 8385, 8386, 8387, 8388, 8389, 8390, 8391, 8392, 8393, 8394, 8395, 8396, 8397, 8398, 8399, 8400, 8401, 8402, 8403, 8404, 8405, 8406, 8407, 8408, 8409, 8410, 8411, 8412, 8413, 8414, 8415, 8416, 8417, 8418, 8419, 8420, 8421, 8422, 8423, 8424, 8425, 8426, 8427, 8428, 8429, 8430, 8431, 8432, 8433, 8434, 8435, 8436, 8437, 8438, 8439, 8440, 8441, 8442, 8443]} +{"timestamp": 113999, "input_length": 13712, "output_length": 394, "hash_ids": [0, 8444, 8445, 8446, 8447, 8448, 8449, 8450, 8451, 8452, 8453, 8454, 8455, 8456, 8457, 8458, 8459, 8460, 8461, 8462, 8463, 8464, 8465, 8466, 8467, 8468, 8469]} +{"timestamp": 113999, "input_length": 20151, "output_length": 662, "hash_ids": [0, 8470, 8471, 8472, 8473, 8474, 8475, 8476, 8477, 8478, 8479, 8480, 8481, 8482, 8483, 8484, 8485, 8486, 8487, 8488, 8489, 8490, 8491, 8492, 8493, 8494, 8495, 8496, 8497, 8498, 8499, 8500, 8501, 8502, 8503, 8504, 8505, 8506, 8507, 8508]} +{"timestamp": 113999, "input_length": 6303, "output_length": 104, "hash_ids": [0, 8509, 8510, 8511, 8512, 8513, 8514, 8515, 8516, 8517, 8518, 8519, 8520]} +{"timestamp": 113999, "input_length": 15368, "output_length": 23, "hash_ids": [0, 8521, 8522, 8523, 8524, 8525, 8526, 8527, 8528, 8529, 8530, 8531, 8532, 8533, 8534, 8535, 8536, 8537, 8538, 8539, 8540, 8541, 8542, 8543, 8544, 8545, 8546, 8547, 8548, 8549, 8550]} +{"timestamp": 113999, "input_length": 6077, "output_length": 128, "hash_ids": [0, 4048, 4049, 4050, 4051, 4052, 4053, 4054, 4055, 4056, 4057, 8551]} +{"timestamp": 117000, "input_length": 900, "output_length": 200, "hash_ids": [0, 8552]} +{"timestamp": 117000, "input_length": 1216, "output_length": 384, "hash_ids": [0, 8553, 8554]} +{"timestamp": 117000, "input_length": 28631, "output_length": 372, "hash_ids": [0, 8555, 8556, 8557, 8558, 8559, 8560, 8561, 8562, 8563, 8564, 8565, 8566, 8567, 8568, 8569, 8570, 8571, 8572, 8573, 8574, 8575, 8576, 8577, 8578, 8579, 8580, 8581, 8582, 8583, 8584, 8585, 8586, 8587, 8588, 8589, 8590, 8591, 8592, 8593, 8594, 8595, 8596, 8597, 8598, 8599, 8600, 8601, 8602, 8603, 8604, 8605, 8606, 8607, 8608, 8609]} +{"timestamp": 117000, "input_length": 1888, "output_length": 407, "hash_ids": [0, 8610, 8611, 8612]} +{"timestamp": 117000, "input_length": 1434, "output_length": 90, "hash_ids": [0, 6625, 8613]} +{"timestamp": 120000, "input_length": 3808, "output_length": 16, "hash_ids": [0, 8614, 8615, 8616, 8617, 8618, 8619, 8620]} +{"timestamp": 120000, "input_length": 894, "output_length": 349, "hash_ids": [0, 8621]} +{"timestamp": 120000, "input_length": 35126, "output_length": 618, "hash_ids": [0, 1876, 1877, 1878, 1879, 1880, 1881, 1882, 1883, 1884, 1885, 1886, 1887, 1888, 1889, 1890, 1891, 1892, 1893, 1894, 1895, 1896, 1897, 1898, 1899, 1900, 1901, 1902, 1903, 1904, 1905, 1906, 1907, 1908, 1909, 1910, 1911, 1912, 1913, 1914, 1915, 1916, 1917, 1918, 1919, 1920, 1921, 1922, 1923, 1924, 1925, 1926, 1927, 1928, 1929, 1930, 1931, 1932, 1933, 1934, 1935, 1936, 1937, 1938, 1939, 1940, 1941, 1942, 1943]} +{"timestamp": 120000, "input_length": 18907, "output_length": 327, "hash_ids": [0, 4129, 4130, 4131, 4132, 4133, 4134, 4135, 4136, 4137, 4138, 4139, 4140, 4141, 4142, 4143, 4144, 4145, 4146, 4147, 4148, 4149, 4150, 4151, 4152, 4153, 4154, 4155, 4156, 4157, 4158, 4159, 4160, 4161, 4162, 4163, 8622]} +{"timestamp": 120000, "input_length": 6929, "output_length": 303, "hash_ids": [0, 8623, 8624, 8625, 8626, 8627, 8628, 8629, 8630, 8631, 8632, 8633, 8634, 8635]} +{"timestamp": 120000, "input_length": 896, "output_length": 354, "hash_ids": [0, 8636]} +{"timestamp": 120000, "input_length": 895, "output_length": 301, "hash_ids": [0, 8637]} +{"timestamp": 123000, "input_length": 10242, "output_length": 504, "hash_ids": [0, 8638, 8639, 8640, 8641, 8642, 8643, 8644, 8645, 8646, 8647, 8648, 8649, 8650, 8651, 8652, 8653, 8654, 8655, 8656, 8657]} +{"timestamp": 123000, "input_length": 1176, "output_length": 401, "hash_ids": [0, 8658, 8659]} +{"timestamp": 123000, "input_length": 1412, "output_length": 369, "hash_ids": [0, 8660, 8661]} +{"timestamp": 123000, "input_length": 19453, "output_length": 510, "hash_ids": [0, 8662, 8663, 8664, 8665, 8666, 8667, 8668, 8669, 8670, 8671, 8672, 8673, 8674, 8675, 8676, 8677, 8678, 8679, 8680, 8681, 8682, 8683, 8684, 8685, 8686, 8687, 8688, 8689, 8690, 8691, 8692, 8693, 8694, 8695, 8696, 8697, 8698]} +{"timestamp": 125999, "input_length": 10048, "output_length": 172, "hash_ids": [0, 8699, 8700, 8701, 8702, 8703, 8704, 8705, 8706, 8707, 8708, 8709, 8710, 8711, 8712, 8713, 8714, 8715, 8716, 8717]} +{"timestamp": 125999, "input_length": 8848, "output_length": 225, "hash_ids": [0, 8718, 8719, 8720, 8721, 8722, 8723, 8724, 8725, 8726, 8727, 8728, 8729, 8730, 8731, 8732, 8733, 8734]} +{"timestamp": 125999, "input_length": 24663, "output_length": 110, "hash_ids": [0, 8735, 8736, 8737, 8738, 8739, 8740, 8741, 8742, 8743, 8744, 8745, 8746, 8747, 8748, 8749, 8750, 8751, 8752, 8753, 8754, 8755, 8756, 8757, 8758, 8759, 8760, 8761, 8762, 8763, 8764, 8765, 8766, 8767, 8768, 8769, 8770, 8771, 8772, 8773, 8774, 8775, 8776, 8777, 8778, 8779, 8780, 8781, 8782]} +{"timestamp": 125999, "input_length": 6630, "output_length": 465, "hash_ids": [0, 8783, 8784, 8785, 8786, 8787, 8788, 8789, 8790, 8791, 8792, 8793, 8794]} +{"timestamp": 125999, "input_length": 12745, "output_length": 347, "hash_ids": [0, 8795, 8796, 8797, 8798, 8799, 8800, 8801, 8802, 8803, 8804, 8805, 8806, 8807, 8808, 8809, 8810, 8811, 8812, 8813, 8814, 8815, 8816, 8817, 8818]} +{"timestamp": 125999, "input_length": 67013, "output_length": 360, "hash_ids": [0, 8819, 8820, 8821, 8822, 8823, 8824, 8825, 8826, 8827, 8828, 8829, 8830, 8831, 8832, 8833, 8834, 8835, 8836, 8837, 8838, 8839, 8840, 8841, 8842, 8843, 8844, 8845, 8846, 8847, 8848, 8849, 8850, 8851, 8852, 8853, 8854, 8855, 8856, 8857, 8858, 8859, 8860, 8861, 8862, 8863, 8864, 8865, 8866, 8867, 8868, 8869, 8870, 8871, 8872, 8873, 8874, 8875, 8876, 8877, 8878, 8879, 8880, 8881, 8882, 8883, 8884, 8885, 8886, 8887, 8888, 8889, 8890, 8891, 8892, 8893, 8894, 8895, 8896, 8897, 8898, 8899, 8900, 8901, 8902, 8903, 8904, 8905, 8906, 8907, 8908, 8909, 8910, 8911, 8912, 8913, 8914, 8915, 8916, 8917, 8918, 8919, 8920, 8921, 8922, 8923, 8924, 8925, 8926, 8927, 8928, 8929, 8930, 8931, 8932, 8933, 8934, 8935, 8936, 8937, 8938, 8939, 8940, 8941, 8942, 8943, 8944, 8945, 8946, 8947, 8948]} +{"timestamp": 125999, "input_length": 4316, "output_length": 221, "hash_ids": [0, 8949, 8950, 8951, 8952, 8953, 8954, 8955, 8956]} +{"timestamp": 125999, "input_length": 8774, "output_length": 8, "hash_ids": [0, 8957, 8958, 8959, 8960, 8961, 8962, 8963, 8964, 8965, 8966, 8967, 8968, 8969, 8970, 8971, 8972, 8973]} +{"timestamp": 125999, "input_length": 1008, "output_length": 39, "hash_ids": [0, 8974]} +{"timestamp": 125999, "input_length": 14131, "output_length": 267, "hash_ids": [0, 8975, 8976, 8977, 8978, 8979, 8980, 8981, 8982, 8983, 8984, 8985, 8986, 8987, 8988, 8989, 8990, 8991, 8992, 8993, 8994, 8995, 8996, 8997, 8998, 8999, 9000, 9001]} +{"timestamp": 125999, "input_length": 23911, "output_length": 383, "hash_ids": [0, 3986, 3987, 3988, 3989, 3990, 3991, 3992, 3993, 3994, 3995, 3996, 3997, 3998, 3999, 4000, 4001, 4002, 4003, 4004, 4005, 4006, 4007, 4008, 4009, 4010, 4011, 4012, 4013, 4014, 4015, 4016, 4017, 4018, 4019, 4020, 4021, 9002, 9003, 9004, 9005, 9006, 9007, 9008, 9009, 9010, 9011]} +{"timestamp": 129000, "input_length": 9582, "output_length": 300, "hash_ids": [0, 9012, 9013, 9014, 9015, 9016, 9017, 9018, 9019, 9020, 9021, 9022, 9023, 9024, 9025, 9026, 9027, 9028, 9029]} +{"timestamp": 129000, "input_length": 14376, "output_length": 241, "hash_ids": [0, 9030, 9031, 9032, 9033, 9034, 9035, 9036, 9037, 9038, 9039, 9040, 9041, 9042, 9043, 9044, 9045, 9046, 9047, 9048, 9049, 9050, 9051, 9052, 9053, 9054, 9055, 9056, 9057]} +{"timestamp": 129000, "input_length": 16577, "output_length": 2000, "hash_ids": [0, 9058, 9059, 9060, 9061, 9062, 9063, 9064, 9065, 9066, 9067, 9068, 9069, 9070, 9071, 9072, 9073, 9074, 9075, 9076, 9077, 9078, 9079, 9080, 9081, 9082, 9083, 9084, 9085, 9086, 9087, 9088, 9089]} +{"timestamp": 129000, "input_length": 4999, "output_length": 230, "hash_ids": [0, 5993, 5994, 5995, 5996, 5997, 5998, 5999, 9090, 9091]} +{"timestamp": 129000, "input_length": 26158, "output_length": 3, "hash_ids": [0, 9092, 9093, 9094, 9095, 9096, 9097, 9098, 9099, 9100, 9101, 9102, 9103, 9104, 9105, 9106, 9107, 9108, 9109, 9110, 9111, 9112, 9113, 9114, 9115, 9116, 9117, 9118, 9119, 9120, 9121, 9122, 9123, 9124, 9125, 9126, 9127, 9128, 9129, 9130, 9131, 9132, 9133, 9134, 9135, 9136, 9137, 9138, 9139, 9140, 9141, 9142]} +{"timestamp": 131999, "input_length": 1318, "output_length": 314, "hash_ids": [0, 9143, 9144]} +{"timestamp": 131999, "input_length": 11113, "output_length": 467, "hash_ids": [0, 9145, 9146, 9147, 9148, 9149, 9150, 9151, 9152, 9153, 9154, 9155, 9156, 9157, 9158, 9159, 9160, 9161, 9162, 9163, 9164, 9165]} +{"timestamp": 131999, "input_length": 9586, "output_length": 270, "hash_ids": [0, 4863, 4864, 4865, 4866, 4867, 4868, 4869, 4870, 4871, 4872, 4873, 4874, 4875, 4876, 4877, 4878, 4879, 9166]} +{"timestamp": 131999, "input_length": 8341, "output_length": 455, "hash_ids": [0, 9167, 9168, 9169, 9170, 9171, 9172, 9173, 9174, 9175, 9176, 9177, 9178, 9179, 9180, 9181, 9182]} +{"timestamp": 131999, "input_length": 11667, "output_length": 324, "hash_ids": [0, 9183, 9184, 9185, 9186, 9187, 9188, 9189, 9190, 9191, 9192, 9193, 9194, 9195, 9196, 9197, 9198, 9199, 9200, 9201, 9202, 9203, 9204]} +{"timestamp": 131999, "input_length": 1000, "output_length": 125, "hash_ids": [0, 9205]} +{"timestamp": 131999, "input_length": 5780, "output_length": 405, "hash_ids": [0, 9206, 9207, 9208, 9209, 9210, 9211, 9212, 9213, 9214, 9215, 9216]} +{"timestamp": 131999, "input_length": 2533, "output_length": 241, "hash_ids": [0, 9217, 9218, 9219, 9220]} +{"timestamp": 131999, "input_length": 16637, "output_length": 3, "hash_ids": [0, 9221, 9222, 9223, 9224, 9225, 9226, 9227, 9228, 9229, 9230, 9231, 9232, 9233, 9234, 9235, 9236, 9237, 9238, 9239, 9240, 9241, 9242, 9243, 9244, 9245, 9246, 9247, 9248, 9249, 9250, 9251, 9252]} +{"timestamp": 131999, "input_length": 20422, "output_length": 459, "hash_ids": [0, 4287, 4288, 4289, 4290, 4291, 4292, 4293, 4294, 4295, 4296, 4297, 4298, 4299, 9253, 9254, 9255, 9256, 9257, 9258, 9259, 9260, 9261, 9262, 9263, 9264, 9265, 9266, 9267, 9268, 9269, 9270, 9271, 9272, 9273, 9274, 9275, 9276, 9277, 9278]} +{"timestamp": 131999, "input_length": 915, "output_length": 506, "hash_ids": [0, 9279]} +{"timestamp": 135000, "input_length": 11945, "output_length": 413, "hash_ids": [0, 9280, 9281, 9282, 9283, 9284, 9285, 9286, 9287, 9288, 9289, 9290, 9291, 9292, 9293, 9294, 9295, 9296, 9297, 9298, 9299, 9300, 9301, 9302]} +{"timestamp": 135000, "input_length": 9553, "output_length": 61, "hash_ids": [0, 9303, 9304, 9305, 9306, 9307, 9308, 9309, 9310, 9311, 9312, 9313, 9314, 9315, 9316, 9317, 9318, 9319, 9320]} +{"timestamp": 135000, "input_length": 10299, "output_length": 395, "hash_ids": [0, 4037, 4038, 4039, 4040, 4041, 4042, 4043, 4044, 4045, 4046, 9321, 9322, 9323, 9324, 9325, 9326, 9327, 9328, 9329, 9330]} +{"timestamp": 135000, "input_length": 26570, "output_length": 655, "hash_ids": [0, 9331, 9332, 9333, 9334, 9335, 9336, 9337, 9338, 9339, 9340, 9341, 9342, 9343, 9344, 9345, 9346, 9347, 9348, 9349, 9350, 9351, 9352, 9353, 9354, 9355, 9356, 9357, 9358, 9359, 9360, 9361, 9362, 9363, 9364, 9365, 9366, 9367, 9368, 9369, 9370, 9371, 9372, 9373, 9374, 9375, 9376, 9377, 9378, 9379, 9380, 9381]} +{"timestamp": 135000, "input_length": 81445, "output_length": 365, "hash_ids": [0, 9382, 9383, 9384, 9385, 9386, 9387, 9388, 9389, 9390, 9391, 9392, 9393, 9394, 9395, 9396, 9397, 9398, 9399, 9400, 9401, 9402, 9403, 9404, 9405, 9406, 9407, 9408, 9409, 9410, 9411, 9412, 9413, 9414, 9415, 9416, 9417, 9418, 9419, 9420, 9421, 9422, 9423, 9424, 9425, 9426, 9427, 9428, 9429, 9430, 9431, 9432, 9433, 9434, 9435, 9436, 9437, 9438, 9439, 9440, 9441, 9442, 9443, 9444, 9445, 9446, 9447, 9448, 9449, 9450, 9451, 9452, 9453, 9454, 9455, 9456, 9457, 9458, 9459, 9460, 9461, 9462, 9463, 9464, 9465, 9466, 9467, 9468, 9469, 9470, 9471, 9472, 9473, 9474, 9475, 9476, 9477, 9478, 9479, 9480, 9481, 9482, 9483, 9484, 9485, 9486, 9487, 9488, 9489, 9490, 9491, 9492, 9493, 9494, 9495, 9496, 9497, 9498, 9499, 9500, 9501, 9502, 9503, 9504, 9505, 9506, 9507, 9508, 9509, 9510, 9511, 9512, 9513, 9514, 9515, 9516, 9517, 9518, 9519, 9520, 9521, 9522, 9523, 9524, 9525, 9526, 9527, 9528, 9529, 9530, 9531, 9532, 9533, 9534, 9535, 9536, 9537, 9538, 9539, 9540]} +{"timestamp": 135000, "input_length": 5472, "output_length": 472, "hash_ids": [0, 9541, 9542, 9543, 9544, 9545, 9546, 9547, 9548, 9549, 9550]} +{"timestamp": 135000, "input_length": 24080, "output_length": 422, "hash_ids": [0, 9551, 9552, 9553, 9554, 9555, 9556, 9557, 9558, 9559, 9560, 9561, 9562, 9563, 9564, 9565, 9566, 9567, 9568, 9569, 9570, 9571, 9572, 9573, 9574, 9575, 9576, 9577, 9578, 9579, 9580, 9581, 9582, 9583, 9584, 9585, 9586, 9587, 9588, 9589, 9590, 9591, 9592, 9593, 9594, 9595, 9596, 9597]} +{"timestamp": 135000, "input_length": 4961, "output_length": 294, "hash_ids": [0, 9598, 9599, 9600, 9601, 9602, 9603, 9604, 9605, 9606]} +{"timestamp": 135000, "input_length": 2991, "output_length": 296, "hash_ids": [0, 9607, 9608, 9609, 9610, 9611]} +{"timestamp": 138000, "input_length": 1112, "output_length": 25, "hash_ids": [0, 9612, 9613]} +{"timestamp": 138000, "input_length": 3313, "output_length": 52, "hash_ids": [0, 9614, 9615, 9616, 9617, 9618, 9619]} +{"timestamp": 138000, "input_length": 8904, "output_length": 499, "hash_ids": [0, 9620, 9621, 9622, 9623, 9624, 9625, 9626, 9627, 9628, 9629, 9630, 9631, 9632, 9633, 9634, 9635, 9636]} +{"timestamp": 138000, "input_length": 9870, "output_length": 321, "hash_ids": [0, 9637, 9638, 9639, 9640, 9641, 9642, 9643, 9644, 9645, 9646, 9647, 9648, 9649, 9650, 9651, 9652, 9653, 9654, 9655]} +{"timestamp": 138000, "input_length": 6097, "output_length": 364, "hash_ids": [0, 9656, 9657, 9658, 9659, 9660, 9661, 9662, 9663, 9664, 9665, 9666]} +{"timestamp": 138000, "input_length": 11983, "output_length": 450, "hash_ids": [0, 9667, 9668, 9669, 9670, 9671, 9672, 9673, 9674, 9675, 9676, 9677, 9678, 9679, 9680, 9681, 9682, 9683, 9684, 9685, 9686, 9687, 9688, 9689]} +{"timestamp": 138000, "input_length": 7138, "output_length": 126, "hash_ids": [0, 9690, 9691, 9692, 9693, 9694, 9695, 9696, 9697, 9698, 9699, 9700, 9701, 9702]} +{"timestamp": 138000, "input_length": 908, "output_length": 341, "hash_ids": [0, 9703]} +{"timestamp": 138000, "input_length": 121298, "output_length": 614, "hash_ids": [0, 2697, 2698, 2699, 2700, 2701, 2702, 2703, 2704, 2705, 2706, 2707, 2708, 2709, 2710, 2711, 2712, 2713, 2714, 2715, 2716, 2717, 2718, 2719, 2720, 2721, 2722, 2723, 2724, 2725, 2726, 2727, 2728, 2729, 2730, 2731, 2732, 2733, 2734, 2735, 2736, 2737, 2738, 2739, 2740, 2741, 2742, 2743, 2744, 2745, 2746, 2747, 2748, 2749, 2750, 2751, 2752, 2753, 2754, 2755, 2756, 2757, 2758, 2759, 2760, 2761, 2762, 2763, 2764, 2765, 2766, 2767, 2768, 2769, 2770, 2771, 2772, 2773, 2774, 2775, 2776, 2777, 2778, 2779, 2780, 2781, 2782, 2783, 2784, 2785, 2786, 2787, 2788, 2789, 2790, 2791, 2792, 2793, 2794, 2795, 2796, 2797, 2798, 2799, 2800, 2801, 2802, 2803, 2804, 2805, 2806, 2807, 2808, 2809, 2810, 2811, 2812, 2813, 2814, 2815, 2816, 2817, 2818, 2819, 2820, 2821, 2822, 2823, 2824, 2825, 2826, 2827, 2828, 2829, 2830, 2831, 2832, 2833, 2834, 2835, 2836, 2837, 2838, 2839, 2840, 2841, 2842, 2843, 2844, 2845, 2846, 2847, 2848, 2849, 2850, 2851, 2852, 2853, 2854, 2855, 2856, 2857, 2858, 2859, 2860, 2861, 2862, 2863, 2864, 2865, 2866, 2867, 2868, 2869, 2870, 2871, 2872, 2873, 2874, 2875, 2876, 2877, 2878, 2879, 2880, 2881, 2882, 2883, 2884, 2885, 2886, 2887, 2888, 2889, 2890, 2891, 2892, 2893, 2894, 2895, 2896, 2897, 2898, 2899, 2900, 2901, 2902, 2903, 2904, 2905, 2906, 2907, 2908, 2909, 2910, 2911, 2912, 2913, 2914, 2915, 2916, 2917, 2918, 2919, 2920, 2921, 2922, 2923, 2924, 2925, 2926, 2927, 2928, 2929, 2930, 9704, 9705]} +{"timestamp": 138000, "input_length": 11296, "output_length": 48, "hash_ids": [0, 9706, 9707, 9708, 9709, 9710, 9711, 9712, 9713, 9714, 9715, 9716, 9717, 9718, 9719, 9720, 9721, 9722, 9723, 9724, 9725, 9726, 9727]} +{"timestamp": 141000, "input_length": 1809, "output_length": 634, "hash_ids": [0, 9728, 9729, 9730]} +{"timestamp": 141000, "input_length": 24982, "output_length": 480, "hash_ids": [0, 9731, 9732, 9733, 9734, 9735, 9736, 9737, 9738, 9739, 9740, 9741, 9742, 9743, 9744, 9745, 9746, 9747, 9748, 9749, 9750, 9751, 9752, 9753, 9754, 9755, 9756, 9757, 9758, 9759, 9760, 9761, 9762, 9763, 9764, 9765, 9766, 9767, 9768, 9769, 9770, 9771, 9772, 9773, 9774, 9775, 9776, 9777, 9778]} +{"timestamp": 141000, "input_length": 3631, "output_length": 22, "hash_ids": [0, 9779, 9780, 9781, 9782, 9783, 9784, 9785]} +{"timestamp": 141000, "input_length": 18203, "output_length": 359, "hash_ids": [0, 9786, 9787, 9788, 9789, 9790, 9791, 9792, 9793, 9794, 9795, 9796, 9797, 9798, 9799, 9800, 9801, 9802, 9803, 9804, 9805, 9806, 9807, 9808, 9809, 9810, 9811, 9812, 9813, 9814, 9815, 9816, 9817, 9818, 9819, 9820]} diff --git a/benchmarks/traffic_replay/manifest.yaml b/benchmarks/traffic_replay/manifest.yaml new file mode 100644 index 0000000..eb64087 --- /dev/null +++ b/benchmarks/traffic_replay/manifest.yaml @@ -0,0 +1,28 @@ +schema: gitm.bench.manifest/v1 +benchmark: traffic_replay +dataset_root: traffic_replay/fixtures +generated_by_gitm: 0.1.13 +file_count: 7 +total_bytes: 150941 +files: +- path: benchserve_result.json + sha256: a780cd29916523c38c12ca833e92896bc75c93b6f94d92826ad10f440e2e35a4 + bytes: 1045 +- path: burstgpt3_dirty.csv + sha256: 6b78a95497587a539f0f10fd0db4eb2a56aadf1af69c3f3eb058ba1c4cee73fd + bytes: 474 +- path: burstgpt3_slice.csv + sha256: 5465fc4c38b0815b8f75d126a120c229ae93632d62e3adf1792f5e8401a73559 + bytes: 34278 +- path: burstgpt_dirty.csv + sha256: 8002d35e1b3ab5bd1169426036d3e7af93c08acd453bb879391eac752e210bc5 + bytes: 372 +- path: burstgpt_slice.csv + sha256: 7865fda5dac50938a84dfccb06c405e2f5202969ecfbd671c8399de00f4335f0 + bytes: 16840 +- path: mooncake_dirty.jsonl + sha256: 483a2e36924902f4bf7fc52ca29d73897918e04e9c30abd521b13852432621a0 + bytes: 639 +- path: mooncake_slice.jsonl + sha256: 8090d6a38401d280fd88112ba22007ff4354be59c986f4f6bcc250edd45a52dc + bytes: 97293 diff --git a/benchmarks/traffic_replay/spec.md b/benchmarks/traffic_replay/spec.md new file mode 100644 index 0000000..b661387 --- /dev/null +++ b/benchmarks/traffic_replay/spec.md @@ -0,0 +1,237 @@ +# Traffic replay v1 — what is frozen, and why + +Deliverable 1 of the validation infrastructure. The load generator for every +experiment: real production traces normalized into one format, fired through the +harness, and tagged with the workload regime every result row is keyed on. + +**Library:** `gitm/traffic/`. **CPU-only** — nothing here needs a GPU, and only +the final firing needs a server. + +```bash +python -m gitm.traffic --selftest # the check +python -m gitm.traffic --describe burstgpt # meta + regime +python -m gitm.traffic --replay mooncake --out replay.jsonl \ + --model Qwen/Qwen3.6-35B-A3B-FP8 # + validation +python -m gitm.traffic --sweep burstgpt # the grid +``` + +## 1. Sources + +| source | format | timestamps | session | prefix identity | fixture | +|---|---|---|---|---|---| +| **BurstGPT_1/_2** | CSV, 6 columns: `Timestamp,Model,Request tokens,Response tokens,Total tokens,Log Type` | **seconds** | none | none | `fixtures/burstgpt_slice.csv` | +| **BurstGPT_3** | CSV, 8 columns: `Timestamp,`**`Session ID,Elapsed time,`**`Model,Request tokens,Response tokens,Total tokens,Log Type` | **seconds** | `Session ID` (UUID) | none | `fixtures/burstgpt3_slice.csv` | +| **Mooncake** | JSONL, `{timestamp, input_length, output_length, hash_ids}` | **milliseconds** | none | `hash_ids`, **512-token** blocks | `fixtures/mooncake_slice.jsonl` | + +All three formats were read off the real published files, not off a paper. +Sources: `HPMLL/BurstGPT` — `data/BurstGPT_1.csv` on `main`, and +`BurstGPT_3.csv` from **release v2.0** (the `_3` files exist only there; `main` +carries `_1` alone) — and `kvcache-ai/Mooncake` +(`FAST25-release/traces/conversation_trace.jsonl`). + +**BurstGPT_3 inserts its two columns at positions 1 and 2 — it does not append +them.** A positional reader does not merely miss them; it reads `Session ID` as +the model and `Elapsed time` as the request length. `read_burstgpt` therefore +goes by column **name**: the six core columns are required, `Session ID` and +`Elapsed time` are used when present, and an unrecognized extra column is +recorded in `TraceMeta.notes` rather than rejected — so a future `BurstGPT_4` +loads instead of raising. `BurstGPT_without_fails_3.csv` carries the same eight. + +Real fixtures are the **first 400 rows of each real file**, unmodified, plus +three hand-authored dirty files that exercise every drop reason. All six are +pinned by sha256 in `manifest.yaml` (`gitm.bench.manifest/v1`, the same contract +the other benchmarks use). + +## 2. The canonical schema + +`gitm.traffic.schema`. Per request: `arrival_s` (**seconds offset from trace +start**, never an epoch), `input_tokens`, `output_tokens`, `session_id`, +`prefix_blocks`. Absent-from-source behaviour is documented per field in the +docstring and is *never* a silent default — a row that cannot supply a required +field is dropped under a named reason. + +`prefix_blocks` is a **chain**, not one hash: two requests share a prefix exactly +as far as their leading block ids agree, and a single digest of the whole chain +would only match identical prompts — the case that does not need measuring. + +`TraceMeta` carries the raw file's sha256, `rows_read`, `rows_emitted` and the +per-reason drop counts. **`Trace` refuses to exist unless those reconcile** +(`rows_read == rows_emitted + dropped`), so "we dropped some bad rows" can never +be a hand-wave. + +## 3. What the real data actually contains + +Pinned in `gitm/traffic/_selftest.py`, measured not assumed: + +| | BurstGPT_1 slice | BurstGPT_3 slice | Mooncake slice | +|---|---|---|---| +| rows read / emitted | 400 / **383** | 400 / **399** | 400 / 400 | +| drops | `zero_input_tokens=17` | `zero_input_tokens=1` | none | +| span | 37,269 s | 41,453 s | 141.0 s | +| session rows / sessions | — | **393 / 134** | — | +| regime label | `prod/io1/in256/out128/burst-poisson/copen` | `prod/io2/in256/out64/burst-poisson/copen` | `prod/io32/in8k/out256/burst-hi/copen` | +| input p50 / p95 | 353 / 1,638 | — | 9,075 / 49,904 | +| output p50 / p95 | 238 / 841 | — | 370 / 662 | +| burstiness (D @ 1 s) | 1.01 | — | 6.74 | + +### Session identity is partial by design, so it is counted, not flagged + +In the full `BurstGPT_3` file, `Session ID` is populated on **exactly** the +`Conversation log` rows and empty on **exactly** the `API log` rows — 528 with, +5,115 without, in the first 5,643. So **an empty `Session ID` is normal data, not +a defect**: the row is emitted with `session_id=None`. Dropping them would throw +away 90 % of a real v3 trace. + +That is also why `TraceMeta` carries `session_rows` and `sessions` and not just +`has_session_identity`. The boolean says "yes" on a trace that is 90 % single-shot; +a multi-turn experiment needs the counts to decide whether the trace can carry it. +In the committed slice: 393 of 399 emitted rows across **134 conversations**, +longest 24 turns. + +`Elapsed time` becomes `CanonicalRequest.source_e2e_latency_s` — deliberately +long-named. It is the **source system's end-to-end submission-to-final-response +time on OpenAI's hardware**: not TTFT, not ITL, and not ours. It must never be +compared against a measured latency or used to promote a playbook row. It is +carried rather than discarded because losing real data at the adapter boundary is +unrecoverable; its one legitimate use is bounding think-time between turns of a +session. An unparseable value is read as *absent* (the row survives, the count +lands in `TraceMeta.notes`) — an optional annotation being junk is no reason to +throw away a valid request. + +**7.9 % of the full BurstGPT file carries zero input *and* zero output tokens** +(4.3 % of this slice). A loader that kept them would put empty prefills into +every regime fit. This is the "handle real data" requirement firing on the first +source, not a hypothetical. + +## 4. Regime axes + +`gitm.traffic.regime`. Prefill/decode token ratio, input and output length +distributions, arrival burstiness, concurrency — plus `source_kind`. + +Burstiness is the **index of dispersion** (variance/mean of arrival counts per +1 s bin), not the CV of interarrival times. Both are 1 for a Poisson process, but +two traces with the same mean rate and different bunching collapse to one CV, and +bunching is the axis the customer's traffic actually varies on. + +`source_kind ∈ {production, synthetic, scoreboard}` is a schema field, not a +naming convention. **Artificial Analysis's fixed-length workload is a +`scoreboard` regime** and its label starts `board/`, so it can never be read as +production traffic after being copied into a spreadsheet. + +`Regime.label()` is bucketed on purpose — two runs of the same workload must +produce the same label, and raw quantiles never repeat. Out-of-envelope sampled +points are suffixed `/xenv`. + +## 5. Replay mode — no custom load generator + +vLLM's bench-serve has a native `timed_trace` dataset that consumes exactly +`{timestamp, input_length, output_length, hash_ids}` JSONL and, under +`--self-timed`, schedules **each request at its own timestamp**. That is faithful +replay, already written and maintained, so `gitm.traffic.replay` writes that file +and builds that command line. Nothing here fires traffic itself. + +Two silent failures this module exists to make loud: + +1. **Block coverage.** `timed_trace` expands each `hash_ids` entry to + `--timed-trace-chunk-hash-size` tokens and *stops when the ids run out*. Pass + vLLM's default of 16 against Mooncake's 512-token blocks and every prompt is + 32× short while every count still looks right. `write_timed_trace` checks + `len(blocks) * block_tokens >= input_tokens` per request and **refuses the + whole file** rather than truncating. +2. **Sources with no prefix identity.** BurstGPT has no `hash_ids`; an empty list + produces a *zero-length* prompt. Blocks are therefore **synthesized** — a + fresh, globally unique run per request, so lengths are honoured and **no + prefix sharing is invented that the source never had**. `ReplayPlan` + records `prefix_synthesized=True`, and a prefix-cache experiment must reject + such a plan. + +## 6. Parameterized mode + +`gitm.traffic.parameterize`. Fits each trace along the regime axes, then samples +the grid — including points beyond any single trace, because the customer's next +hour is never the trace's next hour. + +- Length distributions are fitted **empirically** (101-point quantile grid, + inverse-CDF sampling). No parametric family: production length distributions + are multi-modal — short chat turns and long document prompts in one trace — and + a lognormal fit would smear the modes and quietly move the prefill/decode ratio + the whole exercise turns on. +- Arrival burstiness is generated by drawing **per-bin counts from a negative + binomial**, whose index of dispersion is `1 + m/r` and so can be set directly to + the target. Poisson is the `D = 1` case. `D < 1` clamps to Poisson and says so. +- Every point outside the fitted envelope is `in_envelope=False` and labelled + `/xenv`. An extrapolation that cannot be told from a measurement is worse than + no extrapolation. +- A synthetic trace's identity is the **sha256 of its generating parameters** — + same digest, same trace, reproducible without storing the file. + +## 7. Validation — the deliverable, shown + +`gitm.traffic.validate` compares the replayed stream against the source on +arrival timing, both length distributions, mean rate and burstiness, and renders +the check table beside both arrival-rate profiles. Any mismatch is **explained in +prose**, not just printed. + +What is compared is the **file bench-serve will actually read**: the emitter +writes it, `read_timed_trace` reads it back, and `compare` puts it beside the +adapter's output. So "the pipeline preserves the trace" is a measurement of the +artifact, not an argument about the code. + +Two standards, named rather than implied: + +| | `REPLAY_THRESHOLDS` | `SAMPLED_THRESHOLDS` | +|---|---|---| +| KS (arrival, lengths) | 0.001 | 0.15 | +| mean rate | 1e-6 | 0.25 | +| burstiness | 1e-6 | 0.60 | +| request count | exact | 0.35 | +| arrival timeline compared | **yes** | **no** | + +A replay must *reproduce* the trace — the emitter is a format change, so every +statistic comes back identical. A parameterized sample is a *draw* from the +envelope: it reproduces the rate and the dispersion by construction and the +timeline by nothing, so comparing timelines there would only measure that a +sample is not a copy. + +Arrival times are compared at **microsecond resolution** +(`ARRIVAL_RESOLUTION_S`). Sub-microsecond replay fidelity is meaningless next to +millisecond network jitter, and without the quantization the KS reports the +6e-16 s residue of `5999 / 1000` as a real distribution gap — which it did, on +the Mooncake fixture, before this was added. + +Measured, on the committed fixtures: **every replay check returns exactly 0.0** +for both adapters. + +## 8. The check + +```bash +python -m gitm.traffic --selftest # 10 checks, 2 real traces, 7 drop reasons +python -m pytest tests/test_traffic.py -q # the same assertions, as pytest cases +``` + +The assertions live in `gitm/traffic/_selftest.py` and both entry points call +them, so the runnable check named here and the one CI runs cannot drift apart. + +## 9. Not in v1, and why + +- **Adapters beyond these two.** Azure LLM inference, the Prism provider traces, + TraceLab, ShareGPT/LMSYS lengths — "later" in the brief. Two adapters are what + proves the canonical schema is real; the third would only prove it again. +- **Session-aware *firing*.** The adapter is session-aware; the replay path + cannot be. vLLM's `timed_trace` format has **no session field**, so conversation + identity stops at the emitter — pinned by + `check_session_trace_replay_understates_reuse`, not left as folklore. Sessions + are available for analysis and regime characterization today. Making them flow + would mean deriving prefix blocks from session membership, i.e. asserting how + much each turn re-sends: an *invented* cache hit, which is the one thing this + module will not do. Consequence, stated on every such plan: prefix reuse on a + BurstGPT_3 replay is **understated, never overstated** — a floor, not an + estimate. +- **A streaming `Trace`.** The whole trace is materialized; fine to a few million + rows (BurstGPT_1 is ~1.4 M). Named ceiling, not an oversight. +- **Firing at a real endpoint.** vLLM is not installed on the authoring box. The + emitted file and the argv are built against vLLM's current `timed_trace` + contract, read from source; the first run against a live server is the + outstanding confirmation. +- **Concurrency as a fitted axis.** It is an offered-load *setting*, not a + property of a trace: it is carried on `Regime` and set by the caller. diff --git a/gitm/_banner.py b/gitm/_banner.py new file mode 100644 index 0000000..74a74d9 --- /dev/null +++ b/gitm/_banner.py @@ -0,0 +1,56 @@ +"""The ``GitM`` banner, and the three rules that stop it breaking a pipe. + +Decoration on a CLI that also emits JSON and parseable tables is a hazard, not a +feature, so this module is mostly the guard rather than the art: + +1. **stderr, never stdout.** Every byte on stdout belongs to whoever is reading + it — `jq`, a CI step, a `> results.json`. A banner there corrupts the payload, + and the failure shows up far from here as a JSON parse error. +2. **Only when stdout is a TTY.** Redirected stdout means a machine is reading, + and a machine reading stdout usually means stderr is being captured into the + same log. Gate on stdout even though we write to stderr: the question is "is a + human watching this run", and stdout is where the answer is. +3. **Two escapes.** ``--no-banner`` for one invocation, ``GITM_NO_BANNER`` for a + shell, a Makefile or a container that should never show it. + +Cosmetic by intent. The point of writing the guard out is that the cosmetic part +is the part that is safe to change. +""" + +from __future__ import annotations + +import os +import sys + +#: Set to anything non-empty to suppress the banner everywhere. +ENV_VAR = "GITM_NO_BANNER" + +BANNER = r""" + ___ _ _ __ __ + / __(_) |_| \/ | git machines + | (_ | | _| |\/| | inference runtime + \___|_|\__|_| |_| +""" + + +def show_banner(*, suppressed: bool = False, stream=None) -> bool: + """Write the banner to stderr. Returns whether it was actually written. + + ``suppressed`` is the parsed ``--no-banner`` flag. ``stream`` exists for the + selftest; production callers never pass it, and it does not change the TTY + gate — the gate asks about **stdout** regardless of where the banner goes, + because the question is whether a human is watching, not where the bytes land. + """ + if suppressed or os.environ.get(ENV_VAR): + return False + if not (hasattr(sys.stdout, "isatty") and sys.stdout.isatty()): + return False + print(BANNER.strip("\n"), file=stream if stream is not None else sys.stderr) + return True + + +def add_banner_argument(parser) -> None: + """Add ``--no-banner`` to an ``argparse`` parser. One line per entry point.""" + parser.add_argument( + "--no-banner", action="store_true", help="suppress the startup banner" + ) diff --git a/gitm/cli.py b/gitm/cli.py index 7bd63fc..f5e56da 100644 --- a/gitm/cli.py +++ b/gitm/cli.py @@ -7,6 +7,7 @@ import sys from pathlib import Path +from gitm._banner import add_banner_argument, show_banner from gitm.optimizer.deviation import add_deviate_arguments from gitm.planner.registry import add_plan_arguments @@ -79,6 +80,7 @@ def _parser() -> argparse.ArgumentParser: description="Behavioral compiler and intervention runtime.", ) p.add_argument("--version", action="store_true", help="Print version and exit.") + add_banner_argument(p) sub = p.add_subparsers(dest="cmd") run = sub.add_parser("run", help="Run the autonomous optimization loop.") @@ -318,6 +320,10 @@ def main(argv: list[str] | None = None) -> int: args = _parser().parse_args(argv) + # Before any output, and never on stdout: --version and the JSON-emitting + # subcommands write machine-readable payloads there. + show_banner(suppressed=args.no_banner) + if args.version: from gitm import __version__ diff --git a/gitm/traffic/__init__.py b/gitm/traffic/__init__.py new file mode 100644 index 0000000..628e544 --- /dev/null +++ b/gitm/traffic/__init__.py @@ -0,0 +1,73 @@ +"""Traffic replay library — production traces in, harness workloads out. + +Deliverable 1 of the validation infrastructure. Real production traces normalized +into one canonical format, fired through the harness by a tool that already +exists (vLLM's ``bench serve``), and tagged with the workload regime every result +row is keyed on. + + from gitm.traffic import read_burstgpt, Regime, SourceKind, write_timed_trace + + trace = read_burstgpt("BurstGPT_1.csv") + print(trace.meta.summary()) # provenance and drops + print(Regime.from_trace(trace).label()) # the result-row key + plan = write_timed_trace(trace, "replay.jsonl") # what bench serve consumes + print(" ".join(plan.bench_serve_argv(model="..."))) + +CPU-only: nothing here needs a GPU, and only the final firing needs a server. +Run ``python -m gitm.traffic --selftest`` for the check that fails if any of it +regresses. +""" + +from gitm.traffic.adapters import ADAPTERS, read_burstgpt, read_mooncake +from gitm.traffic.parameterize import RegimeFit, fit, grid, sample_trace +from gitm.traffic.regime import Regime, SourceKind, index_of_dispersion +from gitm.traffic.replay import ( + VLLM_MIN_VERSION, + ReplayPlan, + read_timed_trace, + write_timed_trace, +) +from gitm.traffic.results import BenchRun, join_result, unjoined_keys +from gitm.traffic.runner import RunResult, VllmUnavailable, check_vllm, run_replay +from gitm.traffic.schema import SCHEMA, CanonicalRequest, DropReason, Trace, TraceMeta +from gitm.traffic.validate import ( + REPLAY_THRESHOLDS, + SAMPLED_THRESHOLDS, + ValidationReport, + compare, + ks_statistic, +) + +__all__ = [ + "ADAPTERS", + "REPLAY_THRESHOLDS", + "SAMPLED_THRESHOLDS", + "SCHEMA", + "BenchRun", + "CanonicalRequest", + "DropReason", + "Regime", + "RegimeFit", + "ReplayPlan", + "RunResult", + "VLLM_MIN_VERSION", + "VllmUnavailable", + "SourceKind", + "Trace", + "TraceMeta", + "ValidationReport", + "compare", + "fit", + "grid", + "index_of_dispersion", + "join_result", + "check_vllm", + "ks_statistic", + "read_burstgpt", + "read_mooncake", + "read_timed_trace", + "run_replay", + "sample_trace", + "unjoined_keys", + "write_timed_trace", +] diff --git a/gitm/traffic/__main__.py b/gitm/traffic/__main__.py new file mode 100644 index 0000000..5187e86 --- /dev/null +++ b/gitm/traffic/__main__.py @@ -0,0 +1,147 @@ +"""CLI for the traffic library: describe a trace, emit a replay, validate, selftest. + + python -m gitm.traffic --selftest + python -m gitm.traffic --describe burstgpt BurstGPT_1.csv + python -m gitm.traffic --replay mooncake trace.jsonl --out replay.jsonl --model Qwen/... + python -m gitm.traffic --sweep burstgpt BurstGPT_1.csv + python -m gitm.traffic --replay mooncake t.jsonl --fire --model Qwen/... --result-dir runs/ + python -m gitm.traffic --gui + +Everything except ``--fire`` is CPU-only and needs no vLLM. ``--replay`` writes +the file, validates the round trip and prints the ``vllm bench serve`` command; +``--fire`` then runs it, which needs vLLM >= 0.23.0 and a server. Without +``--fire`` the command is printed and not run, because this box may have neither. +""" + +from __future__ import annotations + +import argparse +import sys + +from gitm._banner import add_banner_argument, show_banner +from gitm.traffic._selftest import run_all +from gitm.traffic.adapters import ADAPTERS +from gitm.traffic.parameterize import fit, grid +from gitm.traffic.regime import Regime, SourceKind +from gitm.traffic.replay import VLLM_MIN_VERSION, read_timed_trace, write_timed_trace +from gitm.traffic.runner import VllmUnavailable, run_replay +from gitm.traffic.validate import REPLAY_THRESHOLDS, compare + + +def main(argv: list[str] | None = None) -> int: + # The validation render uses block characters; a cp1252 console would raise + # on them mid-report rather than at the start. + if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding="utf-8", errors="replace") + + p = argparse.ArgumentParser(prog="python -m gitm.traffic") + add_banner_argument(p) + p.add_argument("--selftest", action="store_true", help="run every check and exit") + p.add_argument("--describe", nargs=2, metavar=("ADAPTER", "PATH")) + p.add_argument("--replay", nargs=2, metavar=("ADAPTER", "PATH")) + p.add_argument("--sweep", nargs=2, metavar=("ADAPTER", "PATH")) + p.add_argument("--gui", action="store_true", + help="serve the localhost viewer (read-only; 127.0.0.1 only)") + p.add_argument("--gui-port", type=int, default=8765) + p.add_argument("--gui-root", default=None, + help="directory traces may be read from (default: the committed fixtures)") + p.add_argument("--no-open", action="store_true", help="do not open a browser") + p.add_argument("--fire", action="store_true", + help="after --replay, actually run the command (needs vLLM and a server)") + p.add_argument("--result-dir", default=None, + help="where --fire saves bench serve's result JSON") + p.add_argument("--dry-run", action="store_true", + help="with --fire: build and check everything, launch nothing") + p.add_argument("--out", default="replay.jsonl") + p.add_argument("--model", default="MODEL") + p.add_argument("--tokenizer", default=None, + help="tokenizer id; needed when the served model name is not " + "resolvable on HuggingFace (a stub, or --served-model-name)") + p.add_argument("--base-url", default="http://127.0.0.1:8000") + p.add_argument("--max-rows", type=int, default=None) + p.add_argument("--kind", default="production", choices=[k.value for k in SourceKind]) + a = p.parse_args(argv) + show_banner(suppressed=a.no_banner) + + if a.selftest: + return run_all() + + if a.gui: + from pathlib import Path + + from gitm.traffic.gui import serve + + return serve(port=a.gui_port, + root=Path(a.gui_root) if a.gui_root else None, + open_browser=not a.no_open) + + spec = a.describe or a.replay or a.sweep + if spec is None: + p.print_help() + return 2 + adapter, path = spec + if adapter not in ADAPTERS: + p.error(f"unknown adapter {adapter!r}; known: {', '.join(sorted(ADAPTERS))}") + trace = ADAPTERS[adapter](path, max_rows=a.max_rows) + regime = Regime.from_trace(trace, source_kind=SourceKind(a.kind)) + + print(trace.meta.summary()) + print(f"regime: {regime.summary()}") + for note in trace.meta.notes: + print(f" note: {note}") + + if a.replay: + plan = write_timed_trace(trace, a.out) + print(f"\nwrote {plan.path} ({plan.requests} requests, " + f"{plan.chunk_hash_size}-token blocks)") + for note in plan.notes: + print(f" note: {note}") + report = compare(trace, read_timed_trace(a.out), thresholds=REPLAY_THRESHOLDS) + print() + print(report.render()) + # not `argv` — that is main()'s own parameter, and shadowing it here reads + # like a bug even though parse_args has already run. + cmd = plan.bench_serve_argv(model=a.model, base_url=a.base_url, + tokenizer=a.tokenizer) + print("\n" + " ".join(cmd)) + if not a.fire: + print(f"\n(not run — pass --fire to launch it; needs vLLM >= " + f"{VLLM_MIN_VERSION} and a server at {a.base_url})") + return 0 if report.passed else 1 + + # The version guard runs inside run_replay, before anything launches, so + # a too-old vLLM is a sentence here rather than an argparse error there. + try: + res = run_replay(plan, model=a.model, base_url=a.base_url, + result_dir=a.result_dir, regime=regime, + tokenizer=a.tokenizer, dry_run=a.dry_run) + except VllmUnavailable as exc: + print(f"\nnot fired: {exc}") + return 2 + print(f"\n{res.summary()}") + for note in res.notes: + print(f" note: {note}") + # Seam 3: the result joined to the workload that produced it. + if res.joined is not None: + print() + print(res.joined.render()) + if not res.ok: + print(f"\nstderr tail:\n{res.stderr_tail}") + # A result that does not reconcile with its trace is not evidence, so it + # fails the command even when bench serve itself exited 0. + reconciled = res.joined is None or res.joined.reconciled + return 0 if (report.passed and res.ok and reconciled) else 1 + + if a.sweep: + f = fit(trace) + print(f"\nfitted envelope: {f.rate_rps:.4f} rps, D={f.burstiness:.2f}, " + f"span {f.span_s:.0f}s") + print(f"{'regime label':<48} {'req':>6} {'rps':>8} {'D':>6}") + for sampled, reg in grid(f): + print(f"{reg.label():<48} {len(sampled):>6} {reg.rate_rps:>8.3f} " + f"{reg.burstiness:>6.2f}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/gitm/traffic/_selftest.py b/gitm/traffic/_selftest.py new file mode 100644 index 0000000..6725d24 --- /dev/null +++ b/gitm/traffic/_selftest.py @@ -0,0 +1,682 @@ +"""The check that fails if the traffic library breaks. + +One runnable thing, ``python -m gitm.traffic --selftest``, and the same functions +are the pytest cases in ``tests/test_traffic.py`` — no assertions written twice. + +Every expected number below was measured on the committed fixtures and pinned +here. That is the point: a pinned count is a regression guard, an unpinned one is +a comment. The fixtures are **real published bytes** (the first 400 rows of each +source), so these numbers describe real production traffic, not a mock. +""" + +from __future__ import annotations + +import json +import os +import sys +import tempfile +from pathlib import Path + +from gitm.traffic.adapters import ADAPTERS, read_burstgpt, read_mooncake +from gitm.traffic.parameterize import fit, sample_trace +from gitm.traffic.regime import Regime, SourceKind +from gitm.traffic.replay import read_timed_trace, write_timed_trace +from gitm.traffic.schema import CanonicalRequest, DropReason, Trace, TraceMeta +from gitm.traffic.validate import REPLAY_THRESHOLDS, SAMPLED_THRESHOLDS, compare + +#: Fixtures live beside the benchmark spec, not inside the package: they are +#: data, and the wheel ships ``gitm`` only. ``$GITM_TRAFFIC_FIXTURES`` overrides +#: for an installed checkout. +FIXTURES = Path( + os.environ.get( + "GITM_TRAFFIC_FIXTURES", + Path(__file__).resolve().parents[2] / "benchmarks" / "traffic_replay" / "fixtures", + ) +) + +# --- pinned on the committed fixtures --------------------------------------- +BURSTGPT_ROWS = 400 +BURSTGPT_EMITTED = 383 +BURSTGPT_DROPS = {"zero_input_tokens": 17} # real: 4.3% of the slice is 0-in/0-out +BURSTGPT_LABEL = "prod/io1/in256/out128/burst-poisson/copen" + +#: BurstGPT_3 (release v2.0) — the eight-column layout with Session ID and +#: Elapsed time inserted at positions 1 and 2. +BURSTGPT3_ROWS = 400 +BURSTGPT3_EMITTED = 399 +BURSTGPT3_DROPS = {"zero_input_tokens": 1} +BURSTGPT3_SESSION_ROWS = 393 # the rest are API-log rows with no conversation +BURSTGPT3_SESSIONS = 134 +BURSTGPT3_LABEL = "prod/io2/in256/out64/burst-poisson/copen" + +#: The v3 dirty fixture: 6 rows, 3 emitted, 3 defects, 1 junk Elapsed time. +BURSTGPT3_DIRTY_EMITTED = 3 +BURSTGPT3_DIRTY_DROPS = {"malformed_row": 1, "non_monotonic_arrival": 1, "zero_input_tokens": 1} + +MOONCAKE_ROWS = 400 +MOONCAKE_EMITTED = 400 +MOONCAKE_SPAN_S = 141.0 +MOONCAKE_LABEL = "prod/io32/in8k/out256/burst-hi/copen" + +#: GPT-4 rows in the BurstGPT slice, and the split that must reconcile. +BURSTGPT_GPT4_EMITTED = 77 +BURSTGPT_GPT4_FILTERED = 320 + +#: Every defect reason, each firing exactly once across the two dirty fixtures. +DIRTY_ROWS = 9 +DIRTY_EMITTED = 2 + + +def _fixture(name: str) -> Path: + p = FIXTURES / name + if not p.exists(): + raise FileNotFoundError( + f"fixture {p} not found. Fixtures live in benchmarks/traffic_replay/fixtures/; " + "set $GITM_TRAFFIC_FIXTURES if this is an installed checkout." + ) + return p + + +def check_burstgpt_fixture() -> None: + """The BurstGPT adapter on real bytes, counts and regime pinned.""" + t = read_burstgpt(_fixture("burstgpt_slice.csv")) + assert t.meta.rows_read == BURSTGPT_ROWS, t.meta.rows_read + assert t.meta.rows_emitted == BURSTGPT_EMITTED, t.meta.rows_emitted + assert t.meta.drops == BURSTGPT_DROPS, t.meta.drops + assert t.meta.raw_time_unit == "s" + assert not t.meta.has_prefix_identity # BurstGPT has no hash_ids + assert t.requests[0].arrival_s == 0.0 # clock anchored on the first row read + assert Regime.from_trace(t).label() == BURSTGPT_LABEL, Regime.from_trace(t).label() + + +def check_burstgpt3_layout() -> None: + """The eight-column BurstGPT_3 layout, read by column name rather than index. + + The two extra columns are *inserted* at positions 1 and 2, so a positional + reader does not merely miss them — it reads Session ID as the model and + Elapsed time as the request length. This is the check that the reader is + name-based. + """ + t = read_burstgpt(_fixture("burstgpt3_slice.csv")) + assert t.meta.rows_read == BURSTGPT3_ROWS, t.meta.rows_read + assert t.meta.rows_emitted == BURSTGPT3_EMITTED, t.meta.rows_emitted + assert t.meta.drops == BURSTGPT3_DROPS, t.meta.drops + + # Session identity: present, and quantified. The flag alone would say "yes" + # on a trace that is 90% single-shot API traffic, which the full v3 file is. + assert t.meta.has_session_identity + assert t.meta.session_rows == BURSTGPT3_SESSION_ROWS, t.meta.session_rows + assert t.meta.sessions == BURSTGPT3_SESSIONS, t.meta.sessions + assert 0 < t.meta.session_rows <= t.meta.rows_emitted + + # An empty Session ID is by design (API-log rows), never a drop. + blank = [r for r in t.requests if r.session_id is None] + assert blank, "no API-log rows survived; an empty session id must not drop a row" + assert len(blank) == BURSTGPT3_EMITTED - BURSTGPT3_SESSION_ROWS + + # A real multi-turn conversation is visible, which is the point of the column. + from collections import Counter + + turns = Counter(r.session_id for r in t.requests if r.session_id) + assert max(turns.values()) > 1, "no multi-turn session in the fixture" + + # Elapsed time is carried, in seconds, and is not confused with a token count. + latencies = [r.source_e2e_latency_s for r in t.requests if r.source_e2e_latency_s] + assert latencies and all(0 <= v <= 3600 for v in latencies), latencies[:5] + assert t.requests[0].source_e2e_latency_s == 43.0 # first row of the real file + + assert Regime.from_trace(t).label() == BURSTGPT3_LABEL, Regime.from_trace(t).label() + + +def check_burstgpt3_defects() -> None: + """v3-specific handling: junk in an optional column does not lose the row.""" + t = read_burstgpt(_fixture("burstgpt3_dirty.csv")) + assert t.meta.rows_emitted == BURSTGPT3_DIRTY_EMITTED, t.meta.rows_emitted + assert t.meta.drops == BURSTGPT3_DIRTY_DROPS, t.meta.drops + # An unparseable Elapsed time is read as absent and said out loud... + assert any("unparseable" in n for n in t.meta.notes), t.meta.notes + # ...and the request itself survives with its lengths intact. + survived = [r for r in t.requests if r.source_e2e_latency_s is None and r.session_id] + assert survived and survived[0].input_tokens == 120, survived + + +def check_burstgpt_layouts_are_read_by_name() -> None: + """A six-column file and an eight-column file both load; junk does not.""" + v1 = read_burstgpt(_fixture("burstgpt_slice.csv")) + v3 = read_burstgpt(_fixture("burstgpt3_slice.csv")) + assert not v1.meta.has_session_identity and v3.meta.has_session_identity + assert v1.meta.session_rows == 0 and v1.meta.sessions == 0 + # v1 is unchanged by v3 support — the regression this whole check exists for. + assert v1.meta.rows_emitted == BURSTGPT_EMITTED and v1.meta.drops == BURSTGPT_DROPS + # Every layout records the columns it actually saw. + assert any(n.startswith("columns: ") for n in v3.meta.notes) + # A file missing a core column is still rejected, and says which. + try: + read_burstgpt(_fixture("mooncake_slice.jsonl")) + except ValueError as e: + assert "missing column" in str(e) or "not a BurstGPT" in str(e), e + else: + raise AssertionError("a non-BurstGPT file was accepted as one") + + +def check_session_trace_replay_understates_reuse() -> None: + """A session-aware source with no prefix hashes must say so on the plan. + + BurstGPT_3 knows which requests are turns of one conversation but not what + they share. Synthesized unique blocks therefore understate real prefix reuse + — the safe direction, but only if it is stated. + """ + t = read_burstgpt(_fixture("burstgpt3_slice.csv")) + with tempfile.TemporaryDirectory() as td: + out = Path(td) / "bg3.jsonl" + plan = write_timed_trace(t, out) + replayed = read_timed_trace(out) + # The load shape crosses intact... + assert compare(t, replayed, thresholds=REPLAY_THRESHOLDS).passed + # ...but session identity does not, because timed_trace has no field for + # it. Pinned so the limitation is a fact in the suite, not folklore. + assert not any(r.session_id for r in replayed.requests) + assert not any(r.source_e2e_latency_s for r in replayed.requests) + assert plan.prefix_synthesized + assert any("UNDERSTATED" in n for n in plan.notes), plan.notes + assert any(f"{t.meta.sessions} sessions" in n for n in plan.notes), plan.notes + assert any("NOT carried into the replay file" in n for n in plan.notes), plan.notes + + +def check_mooncake_fixture() -> None: + """The Mooncake adapter, including the 512-token block reading.""" + t = read_mooncake(_fixture("mooncake_slice.jsonl")) + assert t.meta.rows_read == MOONCAKE_ROWS, t.meta.rows_read + assert t.meta.rows_emitted == MOONCAKE_EMITTED, t.meta.rows_emitted + assert t.meta.drops == {}, t.meta.drops + assert t.meta.raw_time_unit == "ms" + assert abs(t.meta.span_s - MOONCAKE_SPAN_S) < 1e-9, t.meta.span_s + assert t.meta.has_prefix_identity and t.meta.prefix_block_tokens == 512 + # The blocks tile the prompt exactly — the property that makes 512 the right + # value and 16 (vLLM's default) a silent 32x truncation. + for r in t.requests: + assert len(r.prefix_blocks) * 512 >= r.input_tokens + assert (len(r.prefix_blocks) - 1) * 512 < r.input_tokens + assert Regime.from_trace(t).label() == MOONCAKE_LABEL, Regime.from_trace(t).label() + + +def check_every_drop_reason_fires() -> None: + """Both dirty fixtures exercise all seven defect reasons, one row each.""" + bg = read_burstgpt(_fixture("burstgpt_dirty.csv")) + mc = read_mooncake(_fixture("mooncake_dirty.jsonl")) + defects = {r.value for r in DropReason} - {DropReason.FILTERED_OUT.value} + for t in (bg, mc): + assert t.meta.rows_read == DIRTY_ROWS, t.meta.rows_read + assert t.meta.rows_emitted == DIRTY_EMITTED, t.meta.rows_emitted + assert set(t.meta.drops) == defects, set(t.meta.drops) + assert all(v == 1 for v in t.meta.drops.values()), t.meta.drops + + +def check_filtering_is_not_a_defect() -> None: + """A caller-supplied filter counts separately from bad data, and reconciles.""" + t = read_burstgpt(_fixture("burstgpt_slice.csv"), model="GPT-4") + assert t.meta.rows_emitted == BURSTGPT_GPT4_EMITTED, t.meta.rows_emitted + assert t.meta.drops["filtered_out"] == BURSTGPT_GPT4_FILTERED, t.meta.drops + assert t.meta.defects == t.meta.dropped - BURSTGPT_GPT4_FILTERED + assert t.meta.rows_read == BURSTGPT_ROWS # the whole file was still read + + +def check_provenance_must_reconcile() -> None: + """A Trace whose counts do not add up cannot be constructed at all.""" + meta = TraceMeta(source="x", path="x", sha256="0" * 64, rows_read=5, rows_emitted=2) + try: + Trace(meta=meta, requests=[CanonicalRequest(0.0, 10, 10)] * 2) + except ValueError as e: + assert "unattributed" in str(e), e + else: + raise AssertionError("a trace with 3 unaccounted rows was accepted") + + +def check_replay_roundtrip() -> None: + """The emitted timed_trace file reproduces the source, on both adapters. + + This is the brief's validation deliverable: the comparison is against the + file bench-serve will actually read, so it is evidence about the artifact. + """ + with tempfile.TemporaryDirectory() as td: + for name, trace in ( + ("burstgpt", read_burstgpt(_fixture("burstgpt_slice.csv"))), + ("mooncake", read_mooncake(_fixture("mooncake_slice.jsonl"))), + ): + out = Path(td) / f"{name}.jsonl" + plan = write_timed_trace(trace, out) + report = compare(trace, read_timed_trace(out), thresholds=REPLAY_THRESHOLDS) + assert report.passed, f"{name}:\n{report.render()}" + assert plan.self_timed and plan.sec_multiplier == 1.0 + assert plan.requests == len(trace) + assert plan.source.sha256 == trace.meta.sha256 # provenance survives + + +def check_replay_refuses_to_truncate() -> None: + """The two silent failures the replay path exists to make loud.""" + mc = read_mooncake(_fixture("mooncake_slice.jsonl")) + with tempfile.TemporaryDirectory() as td: + # vLLM's default chunk size against 512-token Mooncake blocks would + # silently emit prompts 32x short. It must refuse instead. + try: + write_timed_trace(mc, Path(td) / "bad.jsonl", block_tokens=16) + except ValueError as e: + assert "truncated" in str(e), e + else: + raise AssertionError("a 32x prompt truncation was accepted") + + # A trace with no output lengths cannot be replayed as-is. + meta = TraceMeta(source="x", path="x", sha256="0" * 64, rows_read=1, rows_emitted=1) + no_out = Trace(meta=meta, requests=[CanonicalRequest(0.0, 128, None)]) + try: + write_timed_trace(no_out, Path(td) / "noout.jsonl") + except ValueError as e: + assert "output length" in str(e), e + else: + raise AssertionError("a trace with no output lengths was replayed as-is") + + # BurstGPT has no prefix identity, so blocks are synthesized and said so. + bg = read_burstgpt(_fixture("burstgpt_slice.csv")) + plan = write_timed_trace(bg, Path(td) / "bg.jsonl") + assert plan.prefix_synthesized + assert any("SYNTHESIZED" in n for n in plan.notes) + # ...and synthesized ids never collide, so no prefix sharing is invented. + seen: set[int] = set() + for req in read_timed_trace(Path(td) / "bg.jsonl").requests: + assert not seen & set(req.prefix_blocks) + seen |= set(req.prefix_blocks) + + +def check_regime_axes_separate_the_traces() -> None: + """The axes have to tell the two real traces apart, or they are decoration.""" + bg = Regime.from_trace(read_burstgpt(_fixture("burstgpt_slice.csv"))) + mc = Regime.from_trace(read_mooncake(_fixture("mooncake_slice.jsonl"))) + assert mc.burstiness > 5.0 > bg.burstiness, (mc.burstiness, bg.burstiness) + assert mc.input_p50 > 10 * bg.input_p50 # 9075 vs 353 tokens + assert bg.label() != mc.label() + # A scoreboard workload can never be read as production traffic. + board = Regime.from_trace( + read_burstgpt(_fixture("burstgpt_slice.csv")), source_kind=SourceKind.SCOREBOARD + ) + assert board.label().startswith("board/") and bg.label().startswith("prod/") + + +def check_parameterized_envelope() -> None: + """A sample matches the envelope it was fitted on; margin is labelled margin.""" + src = read_burstgpt(_fixture("burstgpt_slice.csv")) + f = fit(src) + inside, reg = sample_trace(f, seed=1) + report = compare(src, inside, thresholds=SAMPLED_THRESHOLDS) + assert report.passed, report.render() + assert reg.in_envelope and "xenv" not in reg.label() + + outside, reg_out = sample_trace(f, rate_mult=4.0, burstiness=16.0, seed=2) + assert not reg_out.in_envelope + assert reg_out.label().endswith("/xenv") + assert len(outside) > 3 * len(inside) # 4x the rate really is 4x the load + # The dispersion target is honoured, not merely requested. + assert reg_out.burstiness > 4.0, reg_out.burstiness + # A sample is reproducible from its parameters alone. + again, _ = sample_trace(f, rate_mult=4.0, burstiness=16.0, seed=2) + assert again.meta.sha256 == outside.meta.sha256 + + +def check_bench_serve_argv() -> None: + """The command line carries the flags that decide whether a replay is real.""" + mc = read_mooncake(_fixture("mooncake_slice.jsonl")) + with tempfile.TemporaryDirectory() as td: + plan = write_timed_trace(mc, Path(td) / "mc.jsonl") + argv = plan.bench_serve_argv(model="Qwen/Qwen3.6-35B-A3B-FP8", max_concurrency=64) + assert argv[:3] == ["vllm", "bench", "serve"] + assert "--self-timed" in argv # without this, timestamps are ignored + assert argv[argv.index("--dataset-name") + 1] == "timed_trace" + assert argv[argv.index("--timed-trace-chunk-hash-size") + 1] == "512" + assert argv[argv.index("--timed-trace-sec-multiplier") + 1] == "1" + assert argv[argv.index("--max-concurrency") + 1] == "64" + assert argv[argv.index("--num-prompts") + 1] == str(MOONCAKE_EMITTED) + + +def check_banner_can_never_corrupt_stdout() -> None: + """The banner's three guards. Cosmetic feature, real failure mode. + + A banner on stdout breaks `jq`, a `> results.json`, and every CI step that + parses output — and it breaks them far from here, as a JSON parse error. + """ + import io + + from gitm import _banner + + sink = io.StringIO() + + class _Tty(io.StringIO): + def isatty(self): + return True + + real_stdout, real_env = sys.stdout, os.environ.get(_banner.ENV_VAR) + try: + # not a TTY -> silent, which is the redirected/CI case + sys.stdout = io.StringIO() + os.environ.pop(_banner.ENV_VAR, None) + assert _banner.show_banner(stream=sink) is False + assert sink.getvalue() == "" + + # a TTY -> shown, and on the stream we were given, never on stdout + sys.stdout = _Tty() + assert _banner.show_banner(stream=sink) is True + assert "GitM" in sink.getvalue() or "git machines" in sink.getvalue() + assert sys.stdout.getvalue() == "", "banner reached stdout" + + # both escapes work even on a TTY + sink.truncate(0), sink.seek(0) + assert _banner.show_banner(suppressed=True, stream=sink) is False + os.environ[_banner.ENV_VAR] = "1" + assert _banner.show_banner(stream=sink) is False + assert sink.getvalue() == "" + finally: + sys.stdout = real_stdout + os.environ.pop(_banner.ENV_VAR, None) + if real_env is not None: + os.environ[_banner.ENV_VAR] = real_env + + # and the flag exists on every entry point, so one of them cannot drift + import argparse + + p = argparse.ArgumentParser() + _banner.add_banner_argument(p) + assert p.parse_args(["--no-banner"]).no_banner is True + assert p.parse_args([]).no_banner is False + + +def check_gui_refuses_paths_it_was_not_configured_for() -> None: + """The viewer is browser-reachable, so its trace root is a trust boundary.""" + from gitm.traffic import gui + + root = FIXTURES.resolve() + + # a name from the server's own list resolves + assert gui._resolve(root, "burstgpt_slice.csv") == root / "burstgpt_slice.csv" + + # anything that is not a bare name inside the root does not + for bad in ( + "../../../etc/passwd", + "..\\..\\windows\\win.ini", + "/etc/passwd", + "subdir/trace.csv", + ".hidden", + "", + "does_not_exist.csv", + ): + try: + gui._resolve(root, bad) + except gui._Rejected: + continue + raise AssertionError(f"path traversal not refused: {bad!r}") + + # loopback only, and a module constant rather than a default someone can pass + assert gui.HOST == "127.0.0.1" + + # the listing only offers files it knows an adapter for + names = {t["name"] for t in gui._list_traces(root)} + assert "burstgpt_slice.csv" in names and "mooncake_slice.jsonl" in names + assert all(t["adapter"] in ADAPTERS for t in gui._list_traces(root)) + + # a Host header that is not loopback is refused, so DNS rebinding cannot + # reach the filesystem reader through a browser on another origin + h = gui._Handler.__new__(gui._Handler) + for host, ok in [("127.0.0.1:8765", True), ("localhost", True), + ("evil.example.com", False), ("", False)]: + h.headers = {"Host": host} + assert h._host_is_loopback() is ok, host + + +def check_version_guard_fires_before_launching() -> None: + """Seam 2's whole point: a too-old vLLM must fail as a sentence, not argparse. + + Below the floor there is no ``timed_trace`` dataset and vLLM complains about + an unknown dataset name — which reads like a typo in *our* command. The guard + turns that into a message naming the version and the flag, before anything is + launched. + """ + from gitm.traffic import runner + from gitm.traffic.replay import VLLM_MIN_VERSION + + # the release comparison, including the two ways a version string lies + assert runner._release("0.23.0") == (0, 23, 0) + assert runner._release("0.23.0+cu128") == (0, 23, 0) # a build tag is not older + assert runner._release("0.23.0rc1") == (0, 23, 0) # nor an rc newer + assert runner._release("0.22.1") < runner._release(VLLM_MIN_VERSION) + assert runner._release("0.6") < runner._release(VLLM_MIN_VERSION) # the old floor + assert runner._release("1.0.0") > runner._release(VLLM_MIN_VERSION) + + real = runner.installed_vllm_version + try: + runner.installed_vllm_version = lambda: None + try: + runner.check_vllm() + except runner.VllmUnavailable as e: + assert "not installed" in str(e) and VLLM_MIN_VERSION in str(e) + else: + raise AssertionError("a missing vllm was not refused") + + runner.installed_vllm_version = lambda: "0.22.1" + try: + runner.check_vllm() + except runner.VllmUnavailable as e: + # the message has to name the flag, or it is the same puzzle as the + # argparse error it exists to replace + assert "timed_trace" in str(e) and "0.22.1" in str(e), e + else: + raise AssertionError("a too-old vllm was not refused") + + runner.installed_vllm_version = lambda: "0.23.0" + assert runner.check_vllm() == "0.23.0" + finally: + runner.installed_vllm_version = real + + +def check_runner_builds_the_pinned_argv_and_keeps_provenance() -> None: + """A dry run exercises everything except the subprocess — argv and shape.""" + from gitm.traffic import runner + + t = read_mooncake(_fixture("mooncake_slice.jsonl")) + with tempfile.TemporaryDirectory() as d: + plan = write_timed_trace(t, Path(d) / "replay.jsonl") + res = runner.run_replay(plan, model="Qwen/Qwen3.6-35B-A3B-FP8", + result_dir=d, dry_run=True) + + assert res.ok and res.returncode == 0 + assert res.argv[:3] == ["vllm", "bench", "serve"] + assert res.argv[res.argv.index("--dataset-name") + 1] == "timed_trace" + assert "--self-timed" in res.argv + # the 512-vs-16 finding survives into the command that actually runs + assert res.argv[res.argv.index("--timed-trace-chunk-hash-size") + 1] == "512" + assert "--save-result" in res.argv + + # provenance is carried, and it is the source trace's own, byte-identical + assert res.source == plan.source == t.meta + assert res.source.sha256 == t.meta.sha256 and res.source.raw_time_unit == "ms" + # nothing was joined: seam 3 is not this module + assert res.result is None + assert any("dry run" in n for n in res.notes) + + +#: The REAL bench serve result JSON, from the 0.28.0 run in session 8. Committed +#: rather than mocked: the two fields the joiner drops are only wrong in a way a +#: mock would have gotten right by accident, and ``request_rate`` arrives as the +#: STRING "inf" because json.dumps cannot write a bare Infinity. +BENCHSERVE_RESULT = "benchserve_result.json" +REAL_RUN_REQUESTS = 40 +REAL_RUN_SPAN_S = 12.0 +REAL_RUN_INPUT_TOKENS = 506_280 +REAL_RUN_DURATION_S = 12.00791824299995 +#: The regime of the 40 rows that actually ran — NOT MOONCAKE_LABEL, which is +#: pinned for the whole 400-row fixture. A joined record must carry the regime of +#: the workload that ran, and a 40-row head of a trace is a different workload: +#: input p50 7,323 (in4k) against the full slice's 9,075 (in8k). +REAL_RUN_LABEL = "prod/io32/in4k/out256/burst-hi/copen" + + +def _real_run(): + """The committed result JSON, plus the plan and regime it came from.""" + result = json.loads(_fixture(BENCHSERVE_RESULT).read_text(encoding="utf-8")) + t = read_mooncake(_fixture("mooncake_slice.jsonl"), max_rows=REAL_RUN_REQUESTS) + with tempfile.TemporaryDirectory() as d: + plan = write_timed_trace(t, Path(d) / "r.jsonl") + return result, plan, Regime.from_trace(t), t + + +def check_join_drops_the_two_fields_that_are_wrong() -> None: + """Seam 3's core claim, against the real result JSON. + + Under ``--self-timed`` the trace decides the schedule, so vLLM never consults + ``request_rate`` or ``burstiness`` — but records them anyway, untouched, next + to real metrics and on exactly the two axes the playbook keys on. + """ + from gitm.traffic.results import MISLEADING_UNDER_SELF_TIMED, is_infinite, join_result + + result, plan, reg, _ = _real_run() + # what the file actually says, before anything touches it + assert result["request_rate"] == "inf" # a STRING, not a float + assert result["burstiness"] == 1.0 + assert is_infinite(result["request_rate"]) + + run = join_result(result, plan, reg) + + # dropped, and visibly so + assert set(run.dropped) == set(MISLEADING_UNDER_SELF_TIMED) + assert "request_rate" not in run.metrics and "burstiness" not in run.metrics + assert run.dropped_values["request_rate"] == "inf" + assert run.dropped_values["burstiness"] == 1.0 + + # and the truth is on the record instead, differing by a lot + assert reg.burstiness > 5.0 and run.dropped_values["burstiness"] == 1.0 + assert reg.rate_rps > 1.0 and is_infinite(run.dropped_values["request_rate"]) + + # nothing was lost: the raw JSON survives the join + assert run.raw == result + assert "request_rate" in run.raw + + +def check_join_attaches_the_identity_the_result_lacks() -> None: + """D1's stated purpose: the regime label reaches a measured number.""" + from gitm.traffic.results import join_result + + result, plan, reg, t = _real_run() + for k in ("regime", "regime_label", "trace", "sha256", "source"): + assert k not in result, f"result JSON unexpectedly carries {k}" + + run = join_result(result, plan, reg) + assert run.regime_label == reg.label() == REAL_RUN_LABEL, run.regime_label + assert run.source.sha256 == t.meta.sha256 and len(run.source.sha256) == 64 + assert run.source.raw_time_unit == "ms" + assert run.chunk_hash_size == 512 + assert run.config_capture == "pending-adit" and run.knobs == {} # R1 + assert run.metrics["p99_ttft_ms"] == result["p99_ttft_ms"] + + +def check_join_reconciles_the_real_run() -> None: + """The real run must pass every check; a mismatch means it is not evidence.""" + from gitm.traffic.results import join_result + + result, plan, reg, _ = _real_run() + # the plan carries the totals a result is reconciled against + assert plan.input_tokens_total == REAL_RUN_INPUT_TOKENS, plan.input_tokens_total + assert abs(plan.span_s - REAL_RUN_SPAN_S) < 1e-9 + + run = join_result(result, plan, reg) + assert run.reconciled, run.render() + assert run.promotable + names = [c.name for c in run.checks] + assert "input_tokens_match_trace" in names and "paced_to_trace_span" in names + assert result["total_input_tokens"] == plan.input_tokens_total == REAL_RUN_INPUT_TOKENS + assert abs(result["duration"] - REAL_RUN_DURATION_S) < 1e-9 + + +def check_join_catches_the_32x_truncation_after_the_fact() -> None: + """The emitter refuses to WRITE a truncating file; this catches one that ran. + + At vLLM's default 16-token blocks every prompt is 32x short while completed, + duration, throughput and every percentile still read perfectly. Input tokens + are the only number that moves. + """ + from gitm.traffic.results import join_result + + result, plan, reg, _ = _real_run() + truncated = dict(result, total_input_tokens=result["total_input_tokens"] // 32) + run = join_result(truncated, plan, reg) + assert not run.reconciled and not run.promotable + bad = [c for c in run.failures() if c.name == "input_tokens_match_trace"] + assert bad and "chunk-hash-size" in bad[0].detail, run.render() + + # every OTHER check still passes, which is exactly why this one is needed + assert [c.name for c in run.failures()] == ["input_tokens_match_trace"] + + +def check_join_reads_both_directions_of_pacing_failure() -> None: + """Too fast and too slow are different failures and must not be conflated.""" + from gitm.traffic.results import join_result + + result, plan, reg, _ = _real_run() + + def pacing(dur): + run = join_result(dict(result, duration=dur), plan, reg) + return next(c for c in run.checks if c.name == "paced_to_trace_span") + + fast = pacing(0.4) # a client that ignored the timestamps + assert not fast.ok and "FASTER" in fast.detail + + slow = pacing(40.0) # a server that saturated + assert not slow.ok and "drifted" in slow.detail + + assert pacing(REAL_RUN_DURATION_S).ok # the real run + + # a run that was NOT self-timed has no schedule to hold, so no such check + unpaced = plan.model_copy(update={"self_timed": False}) + run = join_result(result, unpaced, reg) + assert "paced_to_trace_span" not in [c.name for c in run.checks] + assert "NOT self-timed" in run.dropped["request_rate"] + + +def check_join_accounts_for_every_result_key() -> None: + """A new vLLM field must be a decision, not a silent omission.""" + from gitm.traffic.results import unjoined_keys + + result = json.loads(_fixture(BENCHSERVE_RESULT).read_text(encoding="utf-8")) + assert len(result) == 34, len(result) + assert unjoined_keys(result) == [], unjoined_keys(result) + + +CHECKS = ( + check_burstgpt_fixture, + check_burstgpt3_layout, + check_burstgpt3_defects, + check_burstgpt_layouts_are_read_by_name, + check_session_trace_replay_understates_reuse, + check_mooncake_fixture, + check_every_drop_reason_fires, + check_filtering_is_not_a_defect, + check_provenance_must_reconcile, + check_replay_roundtrip, + check_replay_refuses_to_truncate, + check_regime_axes_separate_the_traces, + check_parameterized_envelope, + check_bench_serve_argv, + check_banner_can_never_corrupt_stdout, + check_gui_refuses_paths_it_was_not_configured_for, + check_version_guard_fires_before_launching, + check_runner_builds_the_pinned_argv_and_keeps_provenance, + check_join_drops_the_two_fields_that_are_wrong, + check_join_attaches_the_identity_the_result_lacks, + check_join_reconciles_the_real_run, + check_join_catches_the_32x_truncation_after_the_fact, + check_join_reads_both_directions_of_pacing_failure, + check_join_accounts_for_every_result_key, +) + + +def run_all() -> int: + for fn in CHECKS: + fn() + print(f" ok {fn.__name__}") + print(f"selftest ok -- {len(CHECKS)} checks, 3 real traces, 7 drop reasons") + return 0 diff --git a/gitm/traffic/adapters.py b/gitm/traffic/adapters.py new file mode 100644 index 0000000..f0a9689 --- /dev/null +++ b/gitm/traffic/adapters.py @@ -0,0 +1,407 @@ +"""Source adapters: raw trace file in, :class:`~gitm.traffic.schema.Trace` out. + +Two sources for v1. Both formats were read off the real published files, not off +a paper: + +* **BurstGPT** — real Azure OpenAI traffic, the burstiness reference. + ``Timestamp,Model,Request tokens,Response tokens,Total tokens,Log Type``, + timestamps in **seconds**. No prefix or session identity. + https://github.com/HPMLL/BurstGPT +* **Mooncake** — Kimi production serving traces. JSONL, + ``{"timestamp", "input_length", "output_length", "hash_ids"}``, timestamps in + **milliseconds**, ``hash_ids`` are **512-token** cache blocks (vLLM's own + ``--timed-trace-chunk-hash-size`` help names 512 for the Moonshot traces; on + the published slice ``len(hash_ids) * 512`` covers ``input_length`` exactly, + never over-covering by a whole block). + https://github.com/kvcache-ai/Mooncake + +Both adapters share the same discipline: **nothing is defaulted, everything +rejected is counted**. Real data is the work here — 7.9 % of the BurstGPT rows +carry zero input *and* zero output tokens, and a loader that quietly kept them +would put empty prefills into every regime fit. +""" + +from __future__ import annotations + +import csv +import json +from collections import Counter +from pathlib import Path + +from gitm.bench.manifest import sha256_file +from gitm.traffic.schema import CanonicalRequest, DropReason, Trace, TraceMeta + +#: Tokens per ``hash_ids`` entry in the published Mooncake traces. +MOONCAKE_BLOCK_TOKENS = 512 + +#: The six columns every BurstGPT release has carried. Required. +_BURSTGPT_COLUMNS = ( + "Timestamp", + "Model", + "Request tokens", + "Response tokens", + "Total tokens", + "Log Type", +) + +#: Columns BurstGPT_3 (release v2.0) adds, at positions 1 and 2 — inserted, not +#: appended, which is why the reader goes by column *name* and never by index. +#: ``BurstGPT_without_fails_3.csv`` carries the same eight. +_BURSTGPT_SESSION = "Session ID" +_BURSTGPT_ELAPSED = "Elapsed time" +_BURSTGPT_OPTIONAL = (_BURSTGPT_SESSION, _BURSTGPT_ELAPSED) + + +class _Collector: + """Accumulates survivors and drop counts so an adapter never loses a row. + + ``last_ts`` tracks the newest *parsed* raw timestamp, which is what the + monotonicity check compares against: a row that was dropped for some other + defect still tells us where the file's clock had reached. + """ + + def __init__(self) -> None: + self.requests: list[CanonicalRequest] = [] + self.drops: Counter[str] = Counter() + self.rows_read = 0 + self.last_ts: float | None = None + + def drop(self, reason: DropReason) -> None: + self.drops[reason.value] += 1 + + def check_lengths(self, inp: int, out: int | None) -> DropReason | None: + """Shared length validation — every adapter routes through here. + + One place, so a new source cannot invent a different definition of + "unusable row" and quietly widen the envelope. + """ + if inp < 0 or (out is not None and out < 0): + return DropReason.NEGATIVE_VALUE + if inp == 0: + return DropReason.ZERO_INPUT_TOKENS + if out == 0: + return DropReason.ZERO_OUTPUT_TOKENS + return None + + def check_monotonic(self, ts: float) -> DropReason | None: + if self.last_ts is not None and ts < self.last_ts: + return DropReason.NON_MONOTONIC_ARRIVAL + return None + + +def _finish( + coll: _Collector, + *, + source: str, + path: Path, + source_url: str | None, + raw_time_unit: str, + prefix_block_tokens: int | None, + has_prefix_identity: bool, + has_session_identity: bool, + notes: list[str], + session_rows: int = 0, + sessions: int = 0, +) -> Trace: + arrivals = [r.arrival_s for r in coll.requests] + span = (arrivals[-1] - arrivals[0]) if len(arrivals) > 1 else 0.0 + digest, nbytes = sha256_file(path) + if any(r.output_tokens is None for r in coll.requests): + notes = [*notes, "some requests carry no output length; replay-as-is is not valid"] + meta = TraceMeta( + source=source, + path=str(path), + sha256=digest, + source_url=source_url, + rows_read=coll.rows_read, + rows_emitted=len(coll.requests), + drops=dict(coll.drops), + span_s=span, + raw_time_unit=raw_time_unit, + prefix_block_tokens=prefix_block_tokens, + has_prefix_identity=has_prefix_identity, + has_session_identity=has_session_identity, + session_rows=session_rows, + sessions=sessions, + notes=[*notes, f"raw file {nbytes} bytes"], + ) + return Trace(meta=meta, requests=coll.requests) + + +def read_burstgpt( + path: str | Path, + *, + model: str | None = None, + log_type: str | None = None, + max_rows: int | None = None, + source_url: str | None = None, +) -> Trace: + """Read a BurstGPT CSV into canonical form. Handles every published layout. + + Two layouts exist. ``BurstGPT_1`` / ``_2`` carry six columns; + **``BurstGPT_3`` (release v2.0) carries eight**, inserting ``Session ID`` and + ``Elapsed time`` at positions 1 and 2 — *inserted*, not appended. So the + reader goes by column **name**: the six core columns are required, the two + extras are used when present, and an unrecognized extra column is recorded in + ``TraceMeta.notes`` rather than rejected. A future ``BurstGPT_4`` that adds a + column will load rather than raise. + + ``Session ID`` is populated **only for ``Conversation log`` rows** — in the + published v3 file every ``API log`` row has it empty, and those are 90 % of + the trace. An empty session id is therefore *by design and not a defect*: + the row is emitted with ``session_id=None``. Dropping them would discard most + of a real v3 trace. ``TraceMeta.session_rows`` / ``.sessions`` report how much + conversation identity actually survived, which is what a multi-turn + experiment must check — the boolean flag alone would say "yes" on a trace + that is 90 % single-shot. + + ``Elapsed time`` becomes ``CanonicalRequest.source_e2e_latency_s``. Read that + field's docstring before using it: it is end-to-end latency on the *source* + system, not TTFT, and not ours. An unparseable value is treated as absent + (the row survives; the count lands in ``TraceMeta.notes``), because an + optional annotation being junk is no reason to throw away a valid request. + + ``model`` (``"ChatGPT"`` / ``"GPT-4"``) and ``log_type`` (``"Conversation + log"`` / ``"API log"``) select a subset; excluded rows count as + ``FILTERED_OUT``, never as defects. + + The arrival clock is anchored on the **first row read**, before any filtering, + so narrowing the selection shifts which requests appear but never shifts when + they appear. + """ + path = Path(path) + coll = _Collector() + t0: float | None = None + sessions: set[str] = set() + session_rows = 0 + bad_elapsed = 0 + + with path.open(newline="", encoding="utf-8") as fh: + reader = csv.reader(fh) + header = next(reader, None) + if header is None: + raise ValueError(f"{path}: empty file, not a BurstGPT CSV") + columns = [h.strip() for h in header] + missing = [c for c in _BURSTGPT_COLUMNS if c not in columns] + if missing: + raise ValueError( + f"{path}: not a BurstGPT CSV — missing column(s) {missing}; " + f"header was {columns!r}" + ) + idx = {name: i for i, name in enumerate(columns)} + has_session = _BURSTGPT_SESSION in idx + has_elapsed = _BURSTGPT_ELAPSED in idx + unknown = [c for c in columns if c not in _BURSTGPT_COLUMNS + _BURSTGPT_OPTIONAL] + + for row in reader: + if max_rows is not None and coll.rows_read >= max_rows: + break + coll.rows_read += 1 + + if len(row) != len(columns): + coll.drop(DropReason.MALFORMED_ROW) + continue + cells = [c.strip() for c in row] + ts_raw = cells[idx["Timestamp"]] + row_model = cells[idx["Model"]] + inp_raw = cells[idx["Request tokens"]] + out_raw = cells[idx["Response tokens"]] + row_log = cells[idx["Log Type"]] + + # The timestamp is parsed first and alone. A row whose *lengths* are + # junk still says where the file's clock had reached, and folding both + # parses into one try lets a bad length hide a backwards jump from the + # monotonicity check entirely. + if not ts_raw: + coll.drop(DropReason.MISSING_FIELD) + continue + try: + ts = float(ts_raw) + except ValueError: + coll.drop(DropReason.NON_NUMERIC) + continue + if t0 is None: + t0 = ts + reason = coll.check_monotonic(ts) + coll.last_ts = ts if coll.last_ts is None else max(coll.last_ts, ts) + if reason is not None: + coll.drop(reason) + continue + + if not inp_raw or not out_raw: + coll.drop(DropReason.MISSING_FIELD) + continue + try: + inp = int(inp_raw) + out = int(out_raw) + except ValueError: + coll.drop(DropReason.NON_NUMERIC) + continue + if (model is not None and row_model != model) or ( + log_type is not None and row_log != log_type + ): + coll.drop(DropReason.FILTERED_OUT) + continue + reason = coll.check_lengths(inp, out) + if reason is not None: + coll.drop(reason) + continue + + # Optional columns. Absent or blank is normal, never a drop: v3 + # leaves Session ID empty on every API-log row by design. + session_id = cells[idx[_BURSTGPT_SESSION]] if has_session else "" + if session_id: + sessions.add(session_id) + session_rows += 1 + elapsed: float | None = None + if has_elapsed and cells[idx[_BURSTGPT_ELAPSED]]: + try: + elapsed = float(cells[idx[_BURSTGPT_ELAPSED]]) + except ValueError: + bad_elapsed += 1 + + coll.requests.append( + CanonicalRequest( + arrival_s=ts - t0, + input_tokens=inp, + output_tokens=out, + session_id=session_id or None, + source_e2e_latency_s=elapsed, + ) + ) + + notes = [f"columns: {', '.join(columns)}"] + if session_rows: + notes.append( + f"session identity on {session_rows}/{len(coll.requests)} emitted rows " + f"({len(sessions)} sessions) — the rest are single-shot API traffic" + ) + else: + notes.append("no session identity in this layout (BurstGPT_1/_2)") + notes.append("BurstGPT carries no prefix identity in any layout") + if bad_elapsed: + notes.append(f"{bad_elapsed} unparseable '{_BURSTGPT_ELAPSED}' values read as absent") + if unknown: + notes.append(f"unrecognized columns ignored: {', '.join(unknown)}") + if model or log_type: + notes.append(f"filtered: model={model!r} log_type={log_type!r}") + return _finish( + coll, + source="burstgpt", + path=path, + source_url=source_url, + raw_time_unit="s", + prefix_block_tokens=None, + has_prefix_identity=False, + has_session_identity=bool(session_rows), + session_rows=session_rows, + sessions=len(sessions), + notes=notes, + ) + + +def read_mooncake( + path: str | Path, + *, + block_tokens: int = MOONCAKE_BLOCK_TOKENS, + max_rows: int | None = None, + source_url: str | None = None, + time_scale: float = 0.001, + source: str = "mooncake", +) -> Trace: + """Read a Mooncake JSONL trace into canonical form. + + ``block_tokens`` is how many tokens one ``hash_ids`` entry stands for. It is + **512 for the published Moonshot traces** and getting it wrong is silent: the + replay path expands each block to that many tokens, so a wrong value produces + prompts that are a clean multiple too short while every count still looks + right. :func:`gitm.traffic.replay.write_timed_trace` checks the coverage and + refuses rather than truncating. + + ``time_scale`` converts the source's timestamps to seconds (0.001 for + Mooncake's milliseconds). :func:`gitm.traffic.replay.read_timed_trace` reuses + this reader at ``time_scale=1.0`` — the emitted replay file is the same shape, + so re-parsing it needs one parameter, not a second parser that can drift. + """ + path = Path(path) + coll = _Collector() + t0: float | None = None + + with path.open(encoding="utf-8") as fh: + for line in fh: + if not line.strip(): + continue + if max_rows is not None and coll.rows_read >= max_rows: + break + coll.rows_read += 1 + + try: + rec = json.loads(line) + except json.JSONDecodeError: + coll.drop(DropReason.MALFORMED_ROW) + continue + if not isinstance(rec, dict): + coll.drop(DropReason.MALFORMED_ROW) + continue + # Timestamp first and alone — see the note in read_burstgpt. + if "timestamp" not in rec: + coll.drop(DropReason.MISSING_FIELD) + continue + try: + ts_raw = float(rec["timestamp"]) + except (TypeError, ValueError): + coll.drop(DropReason.NON_NUMERIC) + continue + if t0 is None: + t0 = ts_raw + reason = coll.check_monotonic(ts_raw) + coll.last_ts = ts_raw if coll.last_ts is None else max(coll.last_ts, ts_raw) + if reason is not None: + coll.drop(reason) + continue + + if not {"input_length", "output_length"} <= rec.keys(): + coll.drop(DropReason.MISSING_FIELD) + continue + try: + inp = int(rec["input_length"]) + out = int(rec["output_length"]) + except (TypeError, ValueError): + coll.drop(DropReason.NON_NUMERIC) + continue + reason = coll.check_lengths(inp, out) + if reason is not None: + coll.drop(reason) + continue + + blocks = rec.get("hash_ids") or [] + if not isinstance(blocks, list) or any(not isinstance(b, int) for b in blocks): + coll.drop(DropReason.MALFORMED_ROW) + continue + + coll.requests.append( + CanonicalRequest( + arrival_s=(ts_raw - t0) * time_scale, + input_tokens=inp, + output_tokens=out, + prefix_blocks=tuple(blocks), + ) + ) + + has_prefix = any(r.prefix_blocks for r in coll.requests) + return _finish( + coll, + source=source, + path=path, + source_url=source_url, + raw_time_unit="ms" if time_scale == 0.001 else "s", + prefix_block_tokens=block_tokens if has_prefix else None, + has_prefix_identity=has_prefix, + has_session_identity=False, + notes=[f"hash_ids read as {block_tokens}-token cache blocks"], + ) + + +#: Adapter registry — name to reader. Keeps the CLI and the selftest from +#: growing an if-chain per source. +ADAPTERS = {"burstgpt": read_burstgpt, "mooncake": read_mooncake} diff --git a/gitm/traffic/gui.py b/gitm/traffic/gui.py new file mode 100644 index 0000000..0379876 --- /dev/null +++ b/gitm/traffic/gui.py @@ -0,0 +1,391 @@ +"""A localhost viewer for the traffic library. stdlib only, one page, read-only. + +``python -m gitm.traffic --gui`` and open the printed URL. Describe a trace, +replay it and see the validation table with the arrival profiles drawn properly, +or sweep the parameterized grid. + +**It is a viewer, not a control panel.** Everything here is CPU-only. It does not +fire traffic — that is :mod:`gitm.traffic.runner` (``--fire`` on the CLI), which +needs vLLM and a live server. The viewer deliberately does not reach for it: +firing from a browser page means a long-running subprocess behind a synchronous +handler and a result nobody is waiting for. Saying so up front is the point — a +page with a "replay" button that only writes a file is a page someone will assume +hit a server. + +No Flask, no React, no build step: the library's functions already return +pydantic models that serialize straight to JSON, so the server is a thin shell +around ``model_dump`` and the page is one string. + +Three things this gets right because it is browser-reachable, even on loopback: + +* **Binds 127.0.0.1 only.** Never ``0.0.0.0`` — that would expose a filesystem + reader to the network the moment someone runs it on a shared box. +* **Never accepts a path from the form.** The client sends a *name*, chosen from + a list the server produced; the server joins it under one configured root and + re-checks containment after resolving. Path traversal is a real vector here, + and "it's only localhost" has never been a defence. +* **Checks the Host header.** A page on any other origin can still POST to + ``127.0.0.1`` via DNS rebinding. Requests whose Host is not loopback are + refused. +""" + +from __future__ import annotations + +import json +import os +import tempfile +import webbrowser +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +from gitm.traffic.adapters import ADAPTERS +from gitm.traffic.parameterize import fit, grid +from gitm.traffic.regime import DEFAULT_BIN_S, Regime, SourceKind +from gitm.traffic.replay import read_timed_trace, write_timed_trace +from gitm.traffic.validate import REPLAY_THRESHOLDS, compare + +#: Loopback only. Not a default to be overridden — a constant, so binding +#: anywhere else is an edit somebody has to justify in a diff. +HOST = "127.0.0.1" + +#: Upper bound on rows read per request. The page is a viewer; a 1.4 M-row file +#: behind a synchronous handler is a hung browser tab, not a feature. +MAX_ROWS_CAP = 20_000 + +#: Where traces may be read from. One directory, resolved once at startup. +DEFAULT_ROOT = Path(__file__).resolve().parents[2] / "benchmarks" / "traffic_replay" / "fixtures" + +_SUFFIX_ADAPTER = {".csv": "burstgpt", ".jsonl": "mooncake"} + + +class _Rejected(Exception): + """A request that will not be served, with the reason shown to the user.""" + + +def _resolve(root: Path, name: str) -> Path: + """Resolve a client-supplied trace *name* under ``root``, or refuse. + + Two independent guards, because either alone has a bypass: rejecting + separators stops the obvious ``../..``, and the containment check after + ``resolve()`` catches what symlinks and Windows path quirks let through. + """ + if not name or "/" in name or "\\" in name or name.startswith("."): + raise _Rejected(f"not a trace name: {name!r}") + path = (root / name).resolve() + if path.parent != root or not path.is_file(): + raise _Rejected(f"{name!r} is not a file under the configured trace root") + return path + + +def _list_traces(root: Path) -> list[dict]: + return sorted( + ( + { + "name": p.name, + "bytes": p.stat().st_size, + "adapter": _SUFFIX_ADAPTER.get(p.suffix, ""), + } + for p in root.iterdir() + if p.is_file() and p.suffix in _SUFFIX_ADAPTER + ), + key=lambda d: d["name"], + ) + + +def _load(root: Path, body: dict): + """Adapter + trace, from a validated request body.""" + adapter = body.get("adapter", "") + if adapter not in ADAPTERS: + raise _Rejected(f"unknown adapter {adapter!r}") + path = _resolve(root, body.get("trace", "")) + max_rows = body.get("max_rows") or None + if max_rows is not None: + max_rows = max(1, min(int(max_rows), MAX_ROWS_CAP)) + return ADAPTERS[adapter](path, max_rows=max_rows) + + +def _regime_of(trace, body: dict) -> Regime: + concurrency = body.get("concurrency") or None + return Regime.from_trace( + trace, + source_kind=SourceKind(body.get("source_kind", "production")), + bin_s=float(body.get("bin_s") or DEFAULT_BIN_S), + concurrency=int(concurrency) if concurrency else None, + ) + + +def _describe(root: Path, body: dict) -> dict: + trace = _load(root, body) + regime = _regime_of(trace, body) + return { + "meta": trace.meta.model_dump(mode="json"), + "summary": trace.meta.summary(), + "regime": regime.model_dump(mode="json"), + "label": regime.label(), + "regime_summary": regime.summary(), + } + + +def _replay(root: Path, body: dict) -> dict: + trace = _load(root, body) + # The output path is ours, never the client's: writing where a form says to + # is the same vulnerability as reading where it says to. + with tempfile.TemporaryDirectory(prefix="gitm-gui-") as tmp: + out = Path(tmp) / "replay.jsonl" + plan = write_timed_trace(trace, out) + report = compare(trace, read_timed_trace(out), thresholds=REPLAY_THRESHOLDS) + return { + "plan": plan.model_dump(mode="json"), + "argv": plan.bench_serve_argv(model=body.get("model") or "MODEL"), + "report": report.model_dump(mode="json"), + "passed": report.passed, + "explain": report.explain(), + } + + +def _sweep(root: Path, body: dict) -> dict: + trace = _load(root, body) + fitted = fit(trace, bin_s=float(body.get("bin_s") or DEFAULT_BIN_S)) + rows = [ + { + "label": reg.label(), + "requests": len(sampled), + "rate_rps": reg.rate_rps, + "burstiness": reg.burstiness, + "input_p50": reg.input_p50, + "output_p50": reg.output_p50, + "in_envelope": reg.in_envelope, + } + for sampled, reg in grid(fitted) + ] + return {"fit": fitted.model_dump(mode="json"), "rows": rows} + + +ROUTES = {"/api/describe": _describe, "/api/replay": _replay, "/api/sweep": _sweep} + + +class _Handler(BaseHTTPRequestHandler): + server_version = "gitm-traffic-viewer" + root: Path = DEFAULT_ROOT + + def log_message(self, fmt, *args): # noqa: A003 - stdlib hook name + pass # the terminal belongs to whatever else is running + + def _host_is_loopback(self) -> bool: + host = (self.headers.get("Host") or "").rsplit(":", 1)[0].strip("[]") + return host in {"127.0.0.1", "localhost", "::1"} + + def _send(self, code: int, body: bytes, ctype: str) -> None: + self.send_response(code) + self.send_header("Content-Type", ctype) + self.send_header("Content-Length", str(len(body))) + # The page is self-contained; nothing should be able to load anything. + self.send_header("Content-Security-Policy", "default-src 'none'; style-src 'unsafe-inline'; script-src 'unsafe-inline'") + self.send_header("X-Content-Type-Options", "nosniff") + self.end_headers() + self.wfile.write(body) + + def _json(self, code: int, payload: dict) -> None: + self._send(code, json.dumps(payload).encode("utf-8"), "application/json") + + def do_GET(self) -> None: # noqa: N802 - stdlib hook name + if not self._host_is_loopback(): + return self._json(403, {"error": "non-loopback Host header refused"}) + if self.path == "/": + return self._send(200, PAGE.encode("utf-8"), "text/html; charset=utf-8") + if self.path == "/api/traces": + return self._json(200, {"root": str(self.root), "traces": _list_traces(self.root)}) + self._json(404, {"error": "no such path"}) + + def do_POST(self) -> None: # noqa: N802 - stdlib hook name + if not self._host_is_loopback(): + return self._json(403, {"error": "non-loopback Host header refused"}) + handler = ROUTES.get(self.path) + if handler is None: + return self._json(404, {"error": "no such path"}) + try: + n = min(int(self.headers.get("Content-Length") or 0), 64 * 1024) + body = json.loads(self.rfile.read(n) or b"{}") + return self._json(200, handler(self.root, body)) + except _Rejected as exc: + return self._json(400, {"error": str(exc)}) + except Exception as exc: # a bad adapter/threshold error belongs on the page + return self._json(400, {"error": f"{type(exc).__name__}: {exc}"}) + + +def serve(port: int = 8765, root: Path | None = None, *, open_browser: bool = True) -> int: + """Run the viewer until interrupted.""" + _Handler.root = (root or Path(os.environ.get("GITM_TRAFFIC_FIXTURES", DEFAULT_ROOT))).resolve() + if not _Handler.root.is_dir(): + raise SystemExit(f"trace root does not exist: {_Handler.root}") + httpd = ThreadingHTTPServer((HOST, port), _Handler) + url = f"http://{HOST}:{httpd.server_address[1]}/" + print(f"traffic viewer on {url}\n trace root: {_Handler.root}\n" + f" read-only: it does not fire traffic (that is --fire, which needs vLLM)\n" + f" ctrl-c to stop") + if open_browser: + webbrowser.open(url) + try: + httpd.serve_forever() + except KeyboardInterrupt: + print("\nstopped") + finally: + httpd.server_close() + return 0 + + +PAGE = r""" +gitm traffic viewer + +
+

gitm — traffic viewer

+
read-only. describe, replay-and-validate, sweep. it does not fire traffic — that is --fire on the CLI, which needs vLLM and a live server.
+
+
+
+ + + + + + + + + +
+
+
+ +""" diff --git a/gitm/traffic/parameterize.py b/gitm/traffic/parameterize.py new file mode 100644 index 0000000..4d8372e --- /dev/null +++ b/gitm/traffic/parameterize.py @@ -0,0 +1,216 @@ +"""Parameterized mode — fit a trace's envelope, then sample the grid around it. + +Replay answers "what happens under this exact traffic". Parameterized mode +answers "what happens across the region this traffic lives in, **and beyond it**" +— which is the question a playbook row has to survive, because the customer's +next hour is never the trace's next hour. + +The traces define the realistic envelope; the sampler covers the envelope plus +margin, and every point outside it is labelled ``in_envelope=False`` and carries +``/xenv`` in its regime label. An extrapolation that cannot be told apart from a +measurement is worse than no extrapolation. + +Distributions are fitted **empirically** — a 101-point quantile grid, sampled by +inverse-CDF interpolation. No parametric family is assumed: production length +distributions are multi-modal (short chat turns and long document prompts in one +trace) and a lognormal fit would smear the modes together and quietly change the +prefill/decode ratio the whole exercise turns on. + +Arrival burstiness is generated by drawing **per-bin counts from a negative +binomial**, whose index of dispersion is ``1 + m/r`` and so can be set directly +to the target. A Poisson process is the ``D = 1`` special case and is used as +such. ``D < 1`` (more regular than Poisson — a paced or rate-limited client) is +clamped to Poisson and noted; ponytail: a binomial thinning would cover it, add +it when a real trace shows underdispersion. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Iterator + +import numpy as np +from pydantic import BaseModel, ConfigDict, Field + +from gitm.traffic.regime import DEFAULT_BIN_S, Regime, SourceKind, index_of_dispersion +from gitm.traffic.schema import CanonicalRequest, Trace, TraceMeta + +#: Quantile grid used for both length distributions. 101 points keeps the tails +#: (p99, p100) as real samples rather than as an extrapolated fit. +_QUANTILES = np.linspace(0.0, 100.0, 101) + + +class RegimeFit(BaseModel): + """One trace's envelope, as the axes the sampler moves along.""" + + model_config = ConfigDict(extra="forbid") + + trace: str # source identity the fit came from + source_sha256: str # the bytes it was fitted on + requests: int + span_s: float + rate_rps: float + burstiness: float + bin_s: float = DEFAULT_BIN_S + input_quantiles: list[float] = Field(default_factory=list) + output_quantiles: list[float] = Field(default_factory=list) + + def sample_lengths(self, rng: np.random.Generator, n: int, *, output: bool) -> np.ndarray: + """Inverse-CDF draw from the fitted empirical distribution.""" + q = np.asarray(self.output_quantiles if output else self.input_quantiles) + u = rng.uniform(0.0, 100.0, size=n) + return np.interp(u, _QUANTILES, q) + + +def fit(trace: Trace, *, bin_s: float = DEFAULT_BIN_S) -> RegimeFit: + """Fit the regime axes of a real trace.""" + if len(trace) < 2: + raise ValueError("need at least two requests to fit an envelope") + outs = trace.output_tokens + if not outs: + raise ValueError("cannot fit output lengths: the trace carries none") + return RegimeFit( + trace=trace.meta.source, + source_sha256=trace.meta.sha256, + requests=len(trace), + span_s=trace.meta.span_s, + rate_rps=trace.rate_rps(), + burstiness=index_of_dispersion( + trace.arrivals, bin_s=bin_s, span_s=trace.meta.span_s + ), + bin_s=bin_s, + input_quantiles=[float(x) for x in np.percentile(trace.input_tokens, _QUANTILES)], + output_quantiles=[float(x) for x in np.percentile(outs, _QUANTILES)], + ) + + +def _arrivals( + rng: np.random.Generator, *, rate_rps: float, burstiness: float, duration_s: float, bin_s: float +) -> np.ndarray: + """Arrival times with a controlled index of dispersion.""" + nbins = max(int(np.ceil(duration_s / bin_s)), 1) + m = rate_rps * bin_s + if m <= 0: + return np.zeros(0) + if burstiness <= 1.0: + counts = rng.poisson(m, size=nbins) + else: + var = burstiness * m + p = m / var # NB with mean m, variance var + r = m * p / (1.0 - p) + counts = rng.negative_binomial(r, p, size=nbins) + starts = np.repeat(np.arange(nbins) * bin_s, counts) + return np.sort(starts + rng.uniform(0.0, bin_s, size=starts.size)) + + +def sample_trace( + fit_: RegimeFit, + *, + rate_mult: float = 1.0, + burstiness: float | None = None, + input_scale: float = 1.0, + output_scale: float = 1.0, + duration_s: float | None = None, + seed: int = 0, +) -> tuple[Trace, Regime]: + """Draw one synthetic trace from the fitted envelope, plus its regime tag. + + Returns the trace and its :class:`~gitm.traffic.regime.Regime`. The regime's + ``in_envelope`` is False whenever any axis is pushed past what the source + trace actually showed — the margin the brief asks for, labelled as margin. + """ + burstiness = fit_.burstiness if burstiness is None else burstiness + duration_s = fit_.span_s if duration_s is None else duration_s + rate = fit_.rate_rps * rate_mult + rng = np.random.default_rng(seed) + + arrivals = _arrivals( + rng, rate_rps=rate, burstiness=burstiness, duration_s=duration_s, bin_s=fit_.bin_s + ) + n = arrivals.size + if n == 0: + raise ValueError("sampled zero requests — raise rate_mult or duration_s") + inp = np.maximum(np.rint(fit_.sample_lengths(rng, n, output=False) * input_scale), 1) + out = np.maximum(np.rint(fit_.sample_lengths(rng, n, output=True) * output_scale), 1) + + params = { + "fit": fit_.trace, + "source_sha256": fit_.source_sha256, + "rate_mult": rate_mult, + "burstiness": burstiness, + "input_scale": input_scale, + "output_scale": output_scale, + "duration_s": duration_s, + "bin_s": fit_.bin_s, + "seed": seed, + } + # A synthetic trace has no bytes on disk, so its identity is the digest of the + # parameters that reproduce it exactly. Same digest, same trace. + digest = hashlib.sha256(json.dumps(params, sort_keys=True).encode()).hexdigest() + + requests = [ + CanonicalRequest(arrival_s=float(a), input_tokens=int(i), output_tokens=int(o)) + for a, i, o in zip(arrivals, inp, out, strict=True) + ] + tol = 1e-9 + in_envelope = ( + rate_mult <= 1.0 + tol + and burstiness <= fit_.burstiness + tol + and input_scale <= 1.0 + tol + and output_scale <= 1.0 + tol + ) + notes = [f"sampled from {fit_.trace} envelope", json.dumps(params, sort_keys=True)] + if burstiness < 1.0: + notes.append("burstiness < 1 requested; clamped to a Poisson process") + meta = TraceMeta( + source="parameterized", + path=f"synthetic:{digest[:16]}", + sha256=digest, + rows_read=n, + rows_emitted=n, + span_s=float(arrivals[-1] - arrivals[0]) if n > 1 else 0.0, + raw_time_unit="s", + notes=notes, + ) + trace = Trace(meta=meta, requests=requests) + regime = Regime.from_trace( + trace, + source_kind=SourceKind.SYNTHETIC, + bin_s=fit_.bin_s, + in_envelope=in_envelope, + trace_id=f"{fit_.trace}~{digest[:8]}", + ) + return trace, regime + + +#: Default grid. Deliberately asymmetric: the interesting margin is *above* the +#: observed trace (more load, burstier, longer outputs), because that is where a +#: playbook row gets applied and nobody measured. +DEFAULT_RATE_MULTS = (0.5, 1.0, 2.0, 4.0) +DEFAULT_BURST_TARGETS = (1.0, 4.0, 16.0) +DEFAULT_OUTPUT_SCALES = (1.0, 2.0) + + +def grid( + fit_: RegimeFit, + *, + rate_mults: tuple[float, ...] = DEFAULT_RATE_MULTS, + burst_targets: tuple[float, ...] = DEFAULT_BURST_TARGETS, + output_scales: tuple[float, ...] = DEFAULT_OUTPUT_SCALES, + duration_s: float | None = None, + seed: int = 0, +) -> Iterator[tuple[Trace, Regime]]: + """Sample the whole grid. One seed per cell, derived, so cells are independent + but the sweep as a whole is reproducible from ``seed`` alone.""" + for i, rm in enumerate(rate_mults): + for j, b in enumerate(burst_targets): + for k, os_ in enumerate(output_scales): + yield sample_trace( + fit_, + rate_mult=rm, + burstiness=b, + output_scale=os_, + duration_s=duration_s, + seed=seed + 1000 * i + 100 * j + 10 * k, + ) diff --git a/gitm/traffic/regime.py b/gitm/traffic/regime.py new file mode 100644 index 0000000..fb6f63d --- /dev/null +++ b/gitm/traffic/regime.py @@ -0,0 +1,183 @@ +"""Workload regimes — the coordinates a playbook row is stated in. + +A :class:`Regime` is what turns "this knob won once" into "this knob wins in +decode-heavy bursty traffic". Every generated workload carries one, the harness +writes :meth:`Regime.label` into every result row, and deliverable 4 keys the +playbook on it. + +Two representations, deliberately both: + +* the **raw axes** (floats), which deliverable 4's match semantics measure a + distance on — live traffic never lands exactly on a measured point; +* the **label** (:meth:`Regime.label`), coarse and bucketed, which is what a + result row carries and a human reads. A label built from raw floats would + never match twice. + +The burstiness axis is the **index of dispersion** — variance over mean of the +arrival counts per bin — not the coefficient of variation of interarrival times. +Both are 1 for a Poisson process, but two traces with the same mean rate and +different bunching collapse to one CV under a fixed mean, and bunching is exactly +the axis the customer's traffic varies on. +""" + +from __future__ import annotations + +from enum import Enum + +import numpy as np +from pydantic import BaseModel, ConfigDict, Field + +from gitm.traffic.schema import Trace + +#: Default counting-bin width for the burstiness statistic, seconds. The index of +#: dispersion is only meaningful relative to a bin: 1 s reads sub-second bunching +#: as burst, 60 s reads it as smooth. Pinned here so every regime is comparable. +DEFAULT_BIN_S = 1.0 + + +class SourceKind(str, Enum): + """Where a workload came from — a schema field, not a naming convention. + + ``SCOREBOARD`` exists so Artificial Analysis's fixed-length benchmark can + never be read as production traffic. It is the public scoreboard condition; + a playbook row measured under it says something about the scoreboard, and the + distinction has to survive being copied into a spreadsheet. + """ + + PRODUCTION = "production" # replayed from a real production trace + SYNTHETIC = "synthetic" # sampled from a fitted envelope + SCOREBOARD = "scoreboard" # fixed-length public benchmark condition + + +def _bucket_tokens(n: float) -> str: + """Round a token count down to a power of two and render it compactly.""" + if n < 1: + return "0" + exp = int(np.floor(np.log2(n))) + v = 1 << exp + return f"{v // 1024}k" if v >= 1024 else str(v) + + +def _bucket_ratio(r: float) -> str: + if r <= 0: + return "io0" + exp = int(np.floor(np.log2(r))) + return f"io{2**exp}" if exp >= 0 else f"io1-{2 ** -exp}" + + +def _bucket_burst(d: float) -> str: + if d < 0.8: + return "burst-flat" # more regular than Poisson (paced or rate-limited) + if d < 1.5: + return "burst-poisson" + if d < 5.0: + return "burst-mod" + return "burst-hi" + + +def index_of_dispersion(arrivals: list[float], *, bin_s: float, span_s: float) -> float: + """Variance / mean of arrival counts per ``bin_s`` bin. 1.0 for Poisson. + + Returns 1.0 when the statistic is undefined — fewer than two bins, or a zero + mean — rather than raising, so a degenerate trace still produces a regime. + The caller learns it was degenerate from :attr:`Regime.burstiness_defined`. + """ + if span_s <= 0 or len(arrivals) < 2: + return 1.0 + nbins = max(int(np.ceil(span_s / bin_s)), 1) + if nbins < 2: + return 1.0 + counts = np.bincount( + np.minimum((np.asarray(arrivals) / bin_s).astype(np.int64), nbins - 1), + minlength=nbins, + ) + mean = counts.mean() + return float(counts.var() / mean) if mean > 0 else 1.0 + + +class Regime(BaseModel): + """One point in workload space, plus the identity of what produced it.""" + + model_config = ConfigDict(extra="forbid") + + source_kind: SourceKind + trace: str # trace identity — adapter name, or the synthetic generator's id + requests: int + rate_rps: float + io_ratio: float # total input tokens / total output tokens + input_p50: int + input_p95: int + output_p50: int + output_p95: int + burstiness: float # index of dispersion at bin_s + bin_s: float = DEFAULT_BIN_S + burstiness_defined: bool = True + concurrency: int | None = None # offered concurrency cap; None = open-loop + in_envelope: bool = True # False when sampled beyond any observed trace + notes: list[str] = Field(default_factory=list) + + @classmethod + def from_trace( + cls, + trace: Trace, + *, + source_kind: SourceKind = SourceKind.PRODUCTION, + bin_s: float = DEFAULT_BIN_S, + concurrency: int | None = None, + in_envelope: bool = True, + trace_id: str | None = None, + ) -> Regime: + if not trace.requests: + raise ValueError("cannot tag an empty trace with a regime") + inp = np.asarray(trace.input_tokens, dtype=float) + outs = trace.output_tokens + out = np.asarray(outs, dtype=float) if outs else np.zeros(1) + span = trace.meta.span_s + arrivals = trace.arrivals + return cls( + source_kind=source_kind, + trace=trace_id or trace.meta.source, + requests=len(trace.requests), + rate_rps=trace.rate_rps(), + io_ratio=float(inp.sum() / out.sum()) if out.sum() > 0 else 0.0, + input_p50=int(np.percentile(inp, 50)), + input_p95=int(np.percentile(inp, 95)), + output_p50=int(np.percentile(out, 50)), + output_p95=int(np.percentile(out, 95)), + burstiness=index_of_dispersion(arrivals, bin_s=bin_s, span_s=span), + bin_s=bin_s, + burstiness_defined=span > 0 and len(arrivals) >= 2, + concurrency=concurrency, + in_envelope=in_envelope, + notes=[] if outs else ["no output lengths in the source; io_ratio is 0"], + ) + + def label(self) -> str: + """The stable, coarse string the harness writes into every result row. + + Bucketed on purpose: two runs of the same workload must produce the same + label, and raw quantiles never repeat. An out-of-envelope point is + suffixed ``/xenv`` so a sampled extrapolation can never be mistaken for a + measured production point. + """ + parts = [ + {"production": "prod", "synthetic": "syn", "scoreboard": "board"}[ + self.source_kind.value + ], + _bucket_ratio(self.io_ratio), + f"in{_bucket_tokens(self.input_p50)}", + f"out{_bucket_tokens(self.output_p50)}", + _bucket_burst(self.burstiness), + f"c{self.concurrency}" if self.concurrency else "copen", + ] + if not self.in_envelope: + parts.append("xenv") + return "/".join(parts) + + def summary(self) -> str: + return ( + f"{self.label()} [{self.requests} req, {self.rate_rps:.3f} rps, " + f"in p50/p95 {self.input_p50}/{self.input_p95}, " + f"out p50/p95 {self.output_p50}/{self.output_p95}, " + f"D={self.burstiness:.2f}@{self.bin_s:g}s]" + ) diff --git a/gitm/traffic/replay.py b/gitm/traffic/replay.py new file mode 100644 index 0000000..cf358d7 --- /dev/null +++ b/gitm/traffic/replay.py @@ -0,0 +1,262 @@ +"""Replay mode — fire a canonical trace as-is, through a tool that already exists. + +vLLM's bench-serve has a native ``timed_trace`` dataset that consumes +``{"timestamp", "input_length", "output_length", "hash_ids"}`` JSONL and, under +``--self-timed``, schedules each request at **its own timestamp** rather than at +a synthesized rate. That is faithful replay, already written, already maintained. +So this module writes that file and builds that command line. There is no load +generator here and there should never be one. + +Two things this module exists to get right, both silent failures otherwise: + +**Block coverage.** ``timed_trace`` builds each prompt by expanding ``hash_ids`` +at ``--timed-trace-chunk-hash-size`` tokens per id and stops when the ids run +out. Pass 16 (the vLLM default) for a Mooncake trace whose blocks are 512 tokens +and every prompt comes out 32x short while every count in the output still looks +correct. :func:`write_timed_trace` checks ``len(blocks) * block_tokens >= +input_tokens`` for every request and refuses the whole file if it does not hold. + +**Sources with no prefix identity.** BurstGPT has no ``hash_ids``. Emitting an +empty list produces a zero-length prompt, not a 472-token one. So blocks are +*synthesized*: each request gets its own fresh run of ids, globally unique, so +lengths are honoured and **no prefix sharing is invented that the source never +had**. The plan records that this happened; a prefix-cache experiment reading a +plan with ``prefix_synthesized=True`` must reject it. +""" + +from __future__ import annotations + +import json +import math +from pathlib import Path + +from pydantic import BaseModel, ConfigDict, Field + +from gitm.traffic.schema import Trace, TraceMeta + +#: Block size used when synthesizing prefix ids for a source that has none. +#: Arbitrary — the ids are unique per request either way — but it sets how many +#: ids each row carries, so keep it large enough that the file stays small. +SYNTHETIC_BLOCK_TOKENS = 512 + +#: First vLLM release containing the ``timed_trace`` dataset. The feature landed +#: 2026-05-28 (``bfb9ebc21``), which missed v0.22.0 by one day: v0.22.1 does +#: **not** have it, v0.23.0 does — checked at both tags, not inferred from dates. +#: +#: This matters because the repo's own ``[vllm]`` extra says ``vllm>=0.6`` and +#: ChunkPrefill's Phase B says ``>=0.19.0``. Either would install a vLLM with no +#: ``timed_trace`` at all, and the failure is an argparse complaint about an +#: unknown dataset name — which reads like a typo in our command, not a missing +#: feature. The number lives here rather than in prose so the argv builder can +#: say it out loud. +VLLM_MIN_VERSION = "0.23.0" + + +class ReplayPlan(BaseModel): + """What was written, and the command that fires it. + + Carries the source :class:`~gitm.traffic.schema.TraceMeta` verbatim: the + promotion rule (deliverable 2) requires trace identity on every promoted row, + and a plan that has forgotten which bytes it came from cannot supply it. + """ + + model_config = ConfigDict(extra="forbid") + + path: str + requests: int + span_s: float + #: Token totals of what was written. Here so a *result* can be reconciled + #: against the trace after the trace object is gone: ``bench serve`` reports + #: ``total_input_tokens``, and comparing it to this is what catches a run + #: fired at the wrong ``--timed-trace-chunk-hash-size``. At vLLM's default of + #: 16 against Mooncake's 512-token blocks the run reports ~1/32 of this while + #: every other count still looks right. See :mod:`gitm.traffic.results`. + input_tokens_total: int = 0 + output_tokens_total: int = 0 + chunk_hash_size: int + sec_multiplier: float = 1.0 # we always emit seconds; vLLM's default is 1 + self_timed: bool = True + prefix_synthesized: bool = False + source: TraceMeta + notes: list[str] = Field(default_factory=list) + + def bench_serve_argv( + self, + *, + model: str, + base_url: str = "http://127.0.0.1:8000", + backend: str = "openai", + tokenizer: str | None = None, + max_concurrency: int | None = None, + result_filename: str | None = None, + seed: int = 0, + ) -> list[str]: + """The exact ``vllm bench serve`` invocation for this plan. + + **Requires vLLM >= :data:`VLLM_MIN_VERSION`.** Below that there is no + ``timed_trace`` dataset and the run dies on an argparse error that looks + like a typo in this command rather than a missing feature. + + ``backend`` must be a completions backend (``openai`` or ``vllm``): + ``timed_trace`` passes pre-tokenized prompts, which the chat endpoints + will not take. + + ``tokenizer`` is worth passing whenever the **served** model id is not an + id HuggingFace can resolve. ``bench serve`` builds a tokenizer from + ``--model`` when none is given, so a server started with + ``--served-model-name`` — or any stub — fails inside + ``AutoTokenizer.from_pretrained`` with *"X is not a local folder and is + not a valid model identifier"*, long after the endpoint answered fine. + Pre-tokenized prompts do not save you from this: the tokenizer is still + constructed for output accounting. + + ``result_filename`` is worth passing. What comes back does **not** + identify the workload: ``bench serve``'s result JSON carries + ``backend``, ``model_id``, ``num_prompts``, ``max_concurrency`` and the + metrics, but no trace identity and no regime. Worse, under + ``--self-timed`` it still records the CLI's ``request_rate`` and + ``burstiness`` defaults, which are meaningless — the real values came + from the trace and are on this plan's ``Regime``. Joining the two is the + open work; see the spec's "what the result JSON does not carry". + """ + argv = [ + "vllm", "bench", "serve", + "--backend", backend, + "--model", model, + "--base-url", base_url, + "--dataset-name", "timed_trace", + "--dataset-path", self.path, + "--num-prompts", str(self.requests), + "--timed-trace-sec-multiplier", f"{self.sec_multiplier:g}", + "--timed-trace-chunk-hash-size", str(self.chunk_hash_size), + "--seed", str(seed), + ] + if tokenizer is not None: + argv += ["--tokenizer", tokenizer] + argv.append("--self-timed" if self.self_timed else "--no-self-timed") + if max_concurrency is not None: + argv += ["--max-concurrency", str(max_concurrency)] + if result_filename is not None: + argv += ["--save-result", "--result-filename", result_filename] + return argv + + +def write_timed_trace( + trace: Trace, + out: str | Path, + *, + block_tokens: int | None = None, +) -> ReplayPlan: + """Write ``trace`` as a vLLM ``timed_trace`` JSONL and return the plan. + + ``block_tokens`` defaults to the trace's own + :attr:`TraceMeta.prefix_block_tokens` when it has prefix identity, and to + :data:`SYNTHETIC_BLOCK_TOKENS` when ids are being synthesized. + + Raises if any request lacks an output length (replay-as-is is not defined for + it — use the parameterized mode) or if block coverage would truncate a prompt. + """ + out = Path(out) + missing_out = sum(1 for r in trace.requests if r.output_tokens is None) + if missing_out: + raise ValueError( + f"{missing_out} of {len(trace.requests)} requests have no output length; " + "replay-as-is is undefined for them — use gitm.traffic.parameterize" + ) + + synthesize = not trace.meta.has_prefix_identity + if block_tokens is None: + block_tokens = ( + SYNTHETIC_BLOCK_TOKENS if synthesize else (trace.meta.prefix_block_tokens or 0) + ) + if block_tokens <= 0: + raise ValueError( + "block_tokens must be positive — a trace with prefix identity must " + "record how many tokens one block id stands for" + ) + + lines: list[str] = [] + next_block_id = 0 + for i, req in enumerate(trace.requests): + if synthesize or not req.prefix_blocks: + n = math.ceil(req.input_tokens / block_tokens) + blocks = list(range(next_block_id, next_block_id + n)) + next_block_id += n + else: + blocks = list(req.prefix_blocks) + if len(blocks) * block_tokens < req.input_tokens: + raise ValueError( + f"request {i}: {len(blocks)} blocks x {block_tokens} tokens cannot " + f"cover input_tokens={req.input_tokens}. The prompt would be " + f"silently truncated — check block_tokens against the source " + f"(Mooncake is 512, not vLLM's default 16)." + ) + lines.append( + json.dumps( + { + "timestamp": round(req.arrival_s, 6), + "input_length": req.input_tokens, + "output_length": req.output_tokens, + "hash_ids": blocks, + } + ) + ) + + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text("\n".join(lines) + "\n", encoding="utf-8", newline="\n") + + notes = ["timestamps emitted in seconds, so sec_multiplier is 1"] + if synthesize: + notes.append( + "prefix blocks SYNTHESIZED (source has no prefix identity): ids are unique " + "per request, so no prefix sharing is implied. Not valid for prefix-cache " + "experiments." + ) + if trace.meta.has_session_identity: + # BurstGPT_3 is exactly this case: it knows which requests are turns of + # one conversation, but not what they share. Later turns of a session do + # re-send the conversation so far, so unique-per-request blocks + # UNDERSTATE the real reuse. Understating is the safe direction — it + # never invents a cache hit — but a measured prefix-cache benefit on + # this trace is a floor, not an estimate, and that has to be said. + notes.append( + f"source has session identity ({trace.meta.session_rows} rows in " + f"{trace.meta.sessions} sessions) but no prefix identity: real " + "conversations share a prefix that these unique ids do not reproduce. " + "Prefix-cache reuse is UNDERSTATED, never overstated." + ) + if trace.meta.has_session_identity: + # Said on every session-carrying plan, not just the synthesized ones: the + # timed_trace format has no session field at all, so conversation identity + # stops at this boundary. Deriving prefix blocks from sessions would make + # it flow, but only by asserting how much each turn re-sends — an + # invented cache hit, which is the one thing this module will not do. + notes.append( + "session identity is NOT carried into the replay file: vLLM's timed_trace " + "format has no session field. Sessions are available for analysis and " + "regime characterization, not for session-aware firing." + ) + return ReplayPlan( + path=str(out), + requests=len(trace.requests), + span_s=trace.meta.span_s, + input_tokens_total=sum(r.input_tokens for r in trace.requests), + output_tokens_total=sum(r.output_tokens or 0 for r in trace.requests), + chunk_hash_size=block_tokens, + prefix_synthesized=synthesize, + source=trace.meta, + notes=notes, + ) + + +def read_timed_trace(path: str | Path, *, source: str = "timed_trace") -> Trace: + """Read a ``timed_trace`` JSONL back into canonical form. + + The inverse of :func:`write_timed_trace`, and the reason the replay claim is + evidence: :mod:`gitm.traffic.validate` compares this against the trace the + adapter produced, so "the pipeline preserves the trace" is a measured + statement about the file the benchmark will actually consume. + """ + from gitm.traffic.adapters import read_mooncake + + return read_mooncake(path, time_scale=1.0, source=source) diff --git a/gitm/traffic/results.py b/gitm/traffic/results.py new file mode 100644 index 0000000..a375484 --- /dev/null +++ b/gitm/traffic/results.py @@ -0,0 +1,308 @@ +"""Seam 3 — join what ``bench serve`` writes back to the workload that produced it. + +Deliverable 1's stated purpose includes *"the harness writes the regime label into +every result row"*. That was true of the tagger and false of the pipeline: the +result JSON comes from vLLM and nothing read it. This module reads it. + +**The result JSON is not merely incomplete — two of its fields are wrong.** +Measured, not assumed, against a real 0.28.0 run (all 33 keys are enumerated in +the standup's `verification.md`): + +``` +result.json request_rate "inf" burstiness 1.0 <- the CLI's defaults +the regime rate_rps 2.837 D 6.74 <- what actually happened +``` + +Under ``--self-timed`` the request schedule comes from the trace, so vLLM never +uses those two flags — but it records them anyway, untouched, beside real +metrics. They are exactly the two axes the playbook keys on. So they are +**dropped with a reason** rather than merged, renamed, or silently overwritten: +:attr:`BenchRun.dropped` shows a reader that they were considered and why they +did not survive. A field quietly removed looks like a field nobody thought about. + +**Reconciliation is the other half.** A result that cannot be tied back to its +trace is not evidence, and the tie has to be checked rather than assumed — the +run may have been fired at the wrong file, the wrong block size, or a server that +dropped half of it. :func:`join_result` runs the checks and records them; a +:class:`BenchRun` that does not reconcile still exists, and says so, because a +failed run is a finding. + +The one this exists for: **``total_input_tokens`` against the trace's own total.** +At vLLM's default ``--timed-trace-chunk-hash-size 16`` against Mooncake's +512-token blocks, every prompt is 32x short while completed, duration, throughput +and every percentile still read perfectly. The emitter refuses to write such a +file (:func:`gitm.traffic.replay.write_timed_trace`); this catches the case where +one was fired anyway, from a plan built elsewhere. + +**R1 is not solved here.** Knob and environment fields stay ``pending-adit``. +This module makes the *trace* half of a playbook row's provenance real; the +config-capture half arrives when Adit's types do, and the joiner is where they +land. +""" + +from __future__ import annotations + +import math +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + +from gitm.traffic.regime import Regime +from gitm.traffic.replay import ReplayPlan +from gitm.traffic.schema import TraceMeta + +#: Schema identity, in the style of ``gitm.bench.manifest.SCHEMA``. +SCHEMA = "gitm.traffic.benchrun/v1" + +#: Waiting on the shared config-capture schema (risk R1). Grep-able. +PENDING_ADIT = "pending-adit" + +#: Result fields that are **wrong** under ``--self-timed``, and why. Dropped, not +#: merged: the trace decided the schedule, so these are the CLI defaults vLLM +#: never consulted, sitting next to real metrics on the two axes a playbook row +#: is keyed on. +MISLEADING_UNDER_SELF_TIMED = { + "request_rate": "CLI default, not the trace's; the real value is regime.rate_rps", + "burstiness": "CLI default, not the trace's; the real value is regime.burstiness", +} + +#: Metrics kept from the result JSON. Deliberately a list rather than "everything +#: except the dropped ones": a future vLLM adding a field should not silently +#: enlarge what a playbook row claims to have measured. +KEPT_METRICS = ( + "duration", "completed", "failed", "num_prompts", + "total_input_tokens", "total_output_tokens", + "request_throughput", "output_throughput", "total_token_throughput", + "request_goodput", "max_output_tokens_per_s", "max_concurrent_requests", + "mean_ttft_ms", "median_ttft_ms", "std_ttft_ms", "p99_ttft_ms", + "mean_tpot_ms", "median_tpot_ms", "std_tpot_ms", "p99_tpot_ms", + "mean_itl_ms", "median_itl_ms", "std_itl_ms", "p99_itl_ms", + "model_id", "tokenizer_id", "backend", "endpoint_type", "date", "label", + "max_concurrency", + # Real-time factor. 0.0 on text serving — it is an ASR metric — but it is a + # field vLLM measured, and `unjoined_keys` exists precisely so a measured + # field cannot fall out of the record without someone deciding it should. + "rtfx", +) + +#: How far a paced run may overrun the trace's span before the schedule is +#: considered to have drifted. Startup and teardown cost a little; a saturating +#: server costs a lot. 5 % of the span plus one second, so short traces are not +#: judged by a percentage of nothing. +DRIFT_TOLERANCE = 0.05 +DRIFT_FLOOR_S = 1.0 + + +class Check(BaseModel): + """One reconciliation between the result and the trace it should describe.""" + + model_config = ConfigDict(extra="forbid") + + name: str + expected: float | int | str | None + actual: float | int | str | None + ok: bool + detail: str = "" + + +class BenchRun(BaseModel): + """One measured run, joined to the workload that produced it. + + This is what deliverable 2 promotes from and what deliverable 4's provenance + is built out of. It is deliberately *not* a playbook row: a row is a **delta + between two arms**, and this is one arm. + """ + + model_config = ConfigDict(extra="forbid") + + schema_id: str = SCHEMA + + # --- what workload this was, which the result JSON does not say ------------ + source: TraceMeta + regime: Regime + regime_label: str + + # --- the conditions it was fired under ------------------------------------- + chunk_hash_size: int + self_timed: bool + prefix_synthesized: bool + + # --- what came back --------------------------------------------------------- + metrics: dict[str, Any] = Field(default_factory=dict) + #: Fields removed from ``metrics``, mapped to why. Kept visible on purpose. + dropped: dict[str, str] = Field(default_factory=dict) + #: The values those fields held, so the record shows what was rejected. + dropped_values: dict[str, Any] = Field(default_factory=dict) + + # --- does it describe the trace it claims to? ------------------------------- + checks: list[Check] = Field(default_factory=list) + + # --- R1 --------------------------------------------------------------------- + knobs: dict[str, bool | int | float | str] = Field(default_factory=dict) + config_capture: str = PENDING_ADIT + + #: The result JSON exactly as vLLM wrote it. Nothing is lost by the join — + #: a dropped field is dropped from ``metrics``, not from the record. + raw: dict[str, Any] = Field(default_factory=dict) + + @property + def reconciled(self) -> bool: + """Whether this result actually describes the trace it is joined to.""" + return all(c.ok for c in self.checks) + + @property + def promotable(self) -> bool: + """Whether deliverable 2 may even look at this run. + + Necessary, never sufficient: D2 owns the promotion rule. What this says + is that the run reconciles with its trace and completed without failures + — below that bar there is nothing for a rule to judge. + """ + return self.reconciled and self.metrics.get("failed", 1) == 0 + + def failures(self) -> list[Check]: + return [c for c in self.checks if not c.ok] + + def summary(self) -> str: + head = "reconciled" if self.reconciled else f"NOT RECONCILED ({len(self.failures())})" + return ( + f"{self.source.source} / {self.regime_label} — {head}: " + f"{self.metrics.get('completed')} completed, " + f"{self.metrics.get('duration', 0.0):.2f}s, " + f"p99 TTFT {self.metrics.get('p99_ttft_ms', float('nan')):.1f}ms" + ) + + def render(self) -> str: + lines = [self.summary(), ""] + w = max((len(c.name) for c in self.checks), default=4) + for c in self.checks: + mark = "ok " if c.ok else "FAIL" + lines.append(f" {mark} {c.name.ljust(w)} expected {c.expected} actual {c.actual}" + + (f" — {c.detail}" if c.detail else "")) + if self.dropped: + lines.append("") + lines.append(" dropped from metrics (recorded, not merged):") + for k, why in sorted(self.dropped.items()): + lines.append(f" {k} = {self.dropped_values.get(k)!r}: {why}") + return "\n".join(lines) + + +def _num(v: Any) -> float | None: + """Coerce a result value to a float, or ``None``. + + ``request_rate`` arrives as the **string** ``"inf"`` — ``json.dumps`` cannot + write a bare ``Infinity``, so vLLM stringifies it. A joiner that assumed + float would raise on the one field it is trying to throw away. + """ + if isinstance(v, bool) or v is None: + return None + if isinstance(v, int | float): + return float(v) + try: + return float(v) + except (TypeError, ValueError): + return None + + +def join_result( + result: dict[str, Any], + plan: ReplayPlan, + regime: Regime, + *, + knobs: dict[str, bool | int | float | str] | None = None, + drift_tolerance: float = DRIFT_TOLERANCE, +) -> BenchRun: + """Join a ``bench serve`` result to the plan and regime that produced it. + + Keeps the metrics deliverable 2 needs, drops the two fields that are wrong + under ``--self-timed``, attaches the workload identity the result JSON has no + room for, and checks that the numbers actually describe this trace. + """ + metrics = {k: result[k] for k in KEPT_METRICS if k in result} + dropped, dropped_values = {}, {} + for k, why in MISLEADING_UNDER_SELF_TIMED.items(): + if k in result: + dropped_values[k] = result[k] + dropped[k] = why if plan.self_timed else f"{why} (run was NOT self-timed)" + + checks: list[Check] = [] + + def add(name, expected, actual, ok, detail=""): + checks.append(Check(name=name, expected=expected, actual=actual, ok=ok, detail=detail)) + + # 1. Every request the plan wrote should have been sent. + completed = result.get("completed") + add("requests_completed", plan.requests, completed, completed == plan.requests, + "" if completed == plan.requests else "the run did not replay the whole trace") + + # 2. Failures are disqualifying, not a footnote. + failed = result.get("failed") + add("no_failed_requests", 0, failed, failed == 0, + "" if failed == 0 else "a run with failures measures the failures too") + + # 3. THE one. Input tokens must match the trace, or the prompts were not the + # trace's prompts — the 32x truncation that leaves every other count right. + ti = result.get("total_input_tokens") + expected_in = plan.input_tokens_total + ok_tokens = ti == expected_in + detail = "" + if not ok_tokens and ti and expected_in: + ratio = expected_in / ti + detail = f"{ratio:.1f}x short — check --timed-trace-chunk-hash-size " \ + f"(the plan wrote {plan.chunk_hash_size}-token blocks)" if ratio > 1.5 else \ + "prompt lengths do not match the trace" + add("input_tokens_match_trace", expected_in, ti, ok_tokens, detail) + + # 4. Pacing, and the two directions mean different things. Only meaningful + # when the run was self-timed; a rate-driven run has no schedule to hold. + duration = _num(result.get("duration")) + if plan.self_timed and duration is not None and plan.span_s > 0: + budget = plan.span_s * (1 + drift_tolerance) + DRIFT_FLOOR_S + if duration < plan.span_s * (1 - drift_tolerance): + ok, why = False, ("finished FASTER than the trace's own span — the timestamps " + "were not honoured, so this is not a replay of this trace") + elif duration > budget: + ok, why = False, ("overran the trace's span — the schedule drifted, which usually " + "means the server saturated; the arrival pattern was not delivered") + else: + ok, why = True, "" + add("paced_to_trace_span", round(plan.span_s, 3), round(duration, 3), ok, why) + + # 5. A synthesized-prefix run cannot speak about cache reuse. Not a failure — + # a label, because the number is a floor and D4 has to know. + if plan.prefix_synthesized: + add("prefix_identity", "from source", "synthesized", True, + "source had no prefix identity; blocks were synthesized unique per request, so any " + "cache-reuse reading from this run is a FLOOR, never an estimate") + + return BenchRun( + source=plan.source, + regime=regime, + regime_label=regime.label(), + chunk_hash_size=plan.chunk_hash_size, + self_timed=plan.self_timed, + prefix_synthesized=plan.prefix_synthesized, + metrics=metrics, + dropped=dropped, + dropped_values=dropped_values, + checks=checks, + knobs=dict(knobs or {}), + raw=dict(result), + ) + + +def unjoined_keys(result: dict[str, Any]) -> list[str]: + """Result keys that are neither kept nor deliberately dropped. + + A new vLLM release adding a field should be a decision, not a silent + omission. `check_join_accounts_for_every_result_key` fails when this is + non-empty for the recorded real-run keys. + """ + known = set(KEPT_METRICS) | set(MISLEADING_UNDER_SELF_TIMED) + return sorted(k for k in result if k not in known) + + +def is_infinite(v: Any) -> bool: + """``request_rate`` arrives as ``"inf"``. Named so the string is not a surprise.""" + n = _num(v) + return n is not None and math.isinf(n) diff --git a/gitm/traffic/runner.py b/gitm/traffic/runner.py new file mode 100644 index 0000000..29a747c --- /dev/null +++ b/gitm/traffic/runner.py @@ -0,0 +1,325 @@ +"""Seam 2 — run the argv the replay module builds, and say what happened. + +:meth:`~gitm.traffic.replay.ReplayPlan.bench_serve_argv` builds the command; +nothing ran it. This module does, and it is deliberately thin — the interesting +work was already done, and everything hard about talking to a vLLM server is +already solved in :mod:`gitm.serve.vllm`, which this reuses rather than reimplements. + +Three things it does that a bare ``subprocess.run`` would not: + +**The version guard fires first.** Below :data:`~gitm.traffic.replay.VLLM_MIN_VERSION` +there is no ``timed_trace`` dataset, and vLLM's own failure is an argparse +complaint about an unknown dataset name — which reads like a typo in *our* +command rather than a missing feature, and costs an afternoon. Checking before +launching converts that into a sentence naming the version and the flag. + +**The model id comes from the server, not from the caller.** ``--served-model-name`` +can rename a model, and a completion request with the wrong id is a **404, not a +slow path** — a whole run that looks like it went badly rather than one that never +started. :func:`gitm.serve.vllm.served_model_name` already knows this; asking it +also proves the endpoint is answering before we fire a few thousand requests at it. + +**The result is joined, not just carried.** ``bench serve``'s JSON has the +metrics and **no trace identity, no regime, no config capture** — and under +``--self-timed`` it still records the CLI's ``request_rate`` and ``burstiness`` +defaults, which are wrong because the real values came from the trace. Pass +``regime=`` and :func:`run_replay` hands the result to +:func:`gitm.traffic.results.join_result` (seam 3), which attaches the workload +identity, drops those two fields with a reason, and reconciles the reported +totals against the trace. Without ``regime=`` the raw result is still carried and +a note says what is missing — joining is not silently skipped. + +Verified against real ``vllm bench serve`` 0.28.0: 40/40 requests, paced to +12.008 s against a 12.000 s trace span. The parts that still need a server — +output-length fidelity, and whether the schedule holds when a real server +saturates — are named in the standup's ``verification.md``. :func:`run_replay` +takes ``dry_run`` so the argv and the guard stay exercisable without one. +""" + +from __future__ import annotations + +import json +import re +import shutil +import subprocess +import sys +import time +from pathlib import Path + +from pydantic import BaseModel, ConfigDict, Field + +from gitm.traffic.regime import Regime +from gitm.traffic.replay import VLLM_MIN_VERSION, ReplayPlan +from gitm.traffic.results import BenchRun, join_result +from gitm.traffic.schema import TraceMeta + +#: How much of a failed run's output to keep. Enough to see the argparse line or +#: the traceback; not so much that a result row carries a log file. +TAIL_CHARS = 4000 + + +class VllmUnavailable(RuntimeError): + """vLLM is missing or too old. The message names the version and the flag.""" + + +def installed_vllm_version() -> str | None: + """The installed vLLM version, or ``None`` if it is not importable. + + Reads package metadata rather than importing ``vllm``: the import pulls in + torch and CUDA and takes tens of seconds, and this runs before every launch. + """ + try: + from importlib.metadata import PackageNotFoundError, version + + try: + return version("vllm") + except PackageNotFoundError: + return None + except ImportError: # pragma: no cover - importlib.metadata is stdlib + return None + + +def _release(v: str) -> tuple[int, ...]: + """The numeric release part of a version, for comparison. + + ``0.23.0rc1`` -> ``(0, 23, 0)``; ``0.23.0+cu128`` -> ``(0, 23, 0)``. Not a + full PEP 440 implementation on purpose — this compares a floor, and the only + thing that matters is that a release candidate does not read as newer than + its release, nor a local/build tag as older. + """ + head = re.split(r"[^0-9.]", v.strip(), maxsplit=1)[0].rstrip(".") + return tuple(int(p) for p in head.split(".") if p.isdigit()) or (0,) + + +def vllm_executable() -> str: + """The ``vllm`` console script **belonging to the running interpreter**. + + Not a bare ``"vllm"`` on ``PATH``, for two reasons and the second is the + serious one: + + * ``python -m gitm.traffic`` run with an absolute interpreter — a conda env + invoked without activation, which is the normal case in CI and under WSL — + has that env's ``bin/`` nowhere on ``PATH``, and the launch dies with + ``FileNotFoundError: 'vllm'``. + * **The guard and the run could disagree.** :func:`check_vllm` reads + ``importlib.metadata.version("vllm")``, which is the version installed + *for this interpreter*. Resolving the binary from ``PATH`` could then run a + different environment's vLLM — so the version that was validated and the + version that runs would not be the same install. Deriving both from + ``sys.executable`` makes that impossible. + + Falls back to ``"vllm"`` when no sibling script exists, so an unusual layout + still gets a ``PATH`` lookup rather than a hard failure. + """ + bindir = Path(sys.executable).parent + for name in ("vllm", "vllm.exe"): + cand = bindir / name + if cand.exists(): + return str(cand) + found = shutil.which("vllm") + return found or "vllm" + + +def check_vllm(min_version: str = VLLM_MIN_VERSION) -> str: + """Return the installed version, or raise with what to do about it. + + Called before launching, never after: the whole point is that the failure + arrives as a sentence about ``timed_trace`` rather than as vLLM's argparse + error about an unknown dataset name. + """ + found = installed_vllm_version() + if found is None: + raise VllmUnavailable( + f"vllm is not installed. Firing a replay needs vllm>={min_version} " + f"(the version that added `bench serve --dataset-name timed_trace`). " + f"Everything else in gitm.traffic is CPU-only and needs no vLLM. " + f"Install: pip install 'gitm-labs[vllm]'" + ) + if _release(found) < _release(min_version): + raise VllmUnavailable( + f"vllm {found} is too old: `--dataset-name timed_trace` needs " + f">={min_version}. Below it the run dies on an argparse complaint " + f"about an unknown dataset name, which reads like a typo in our " + f"command rather than a missing feature. Upgrade: " + f"pip install -U 'vllm>={min_version}'" + ) + return found + + +class RunResult(BaseModel): + """What the run did, with the plan's provenance and — given a regime — the join. + + Two levels on purpose: :attr:`result` is vLLM's JSON untouched, and + :attr:`joined` is that result tied to the workload, with the misleading fields + dropped and the reconciliation checked. Keeping the raw one means the join is + auditable rather than something you have to trust. + """ + + model_config = ConfigDict(extra="forbid") + + argv: list[str] + returncode: int + duration_s: float + vllm_version: str | None = None + resolved_model: str | None = None # what the server calls it, not what we asked for + stdout_tail: str = "" + stderr_tail: str = "" + result_path: str | None = None + #: ``bench serve``'s raw result JSON, unmodified. **Do not read + #: ``request_rate`` or ``burstiness`` out of this** — under ``--self-timed`` + #: they are the CLI's untouched defaults, and the true values are on + #: :attr:`source`'s regime. :attr:`joined` is that join already done + #: (:mod:`gitm.traffic.results`); this field is the unmodified original. + result: dict | None = None + #: The trace this came from, verbatim from the plan. A measured number whose + #: workload cannot be identified is not evidence, and this is the field that + #: stops the two being separated between here and a playbook row. + source: TraceMeta + #: The joined record — result metrics tied to the regime that produced them, + #: with vLLM's misleading ``request_rate`` / ``burstiness`` dropped and the + #: reconciliation checked. Present when a ``regime`` was supplied and a result + #: file came back; ``None`` otherwise. See :mod:`gitm.traffic.results`. + joined: BenchRun | None = None + notes: list[str] = Field(default_factory=list) + + @property + def ok(self) -> bool: + return self.returncode == 0 + + def summary(self) -> str: + head = "ok" if self.ok else f"FAILED rc={self.returncode}" + got = f", {len(self.result)} result keys" if self.result else "" + return ( + f"{head} in {self.duration_s:.1f}s — {self.source.source} " + f"({self.source.rows_emitted} requests){got}" + ) + + +def run_replay( + plan: ReplayPlan, + *, + model: str, + base_url: str = "http://127.0.0.1:8000", + backend: str = "openai", + tokenizer: str | None = None, + max_concurrency: int | None = None, + result_dir: str | Path | None = None, + seed: int = 0, + regime: Regime | None = None, + timeout_s: float = 3600.0, + resolve_model: bool = True, + dry_run: bool = False, +) -> RunResult: + """Fire ``plan`` at ``base_url`` with ``vllm bench serve``. + + Order of operations, and the first two are the reason this is not a bare + ``subprocess.run``: + + 1. **Version guard**, before anything is launched. + 2. **Resolve the served model id** off ``/v1/models``, which also proves the + endpoint is answering. ``resolve_model=False`` skips it and uses ``model`` + verbatim — for a server that does not expose the route. + 3. Run, capturing stdout, stderr and the exit code. + 4. Read back the result JSON if one was written, and — when a ``regime`` is + supplied — **join** it to the workload that produced it (seam 3). The + join is what makes the result evidence rather than a number: it attaches + the regime and trace identity the result JSON has no room for, drops the + two fields that are wrong under ``--self-timed``, and reconciles the + reported totals against the trace. + + ``dry_run`` builds and returns everything except the subprocess, so the argv + and the guard are exercisable with no vLLM and no server. + """ + notes: list[str] = [] + joined: BenchRun | None = None + found = None if dry_run else check_vllm() + + resolved = model + if resolve_model and not dry_run: + from gitm.serve.vllm import served_model_name + + resolved = served_model_name(base_url, model) + if resolved != model: + notes.append( + f"server calls the model {resolved!r}, not {model!r} " + "(--served-model-name); using the server's id — the wrong one is a 404" + ) + + result_path: Path | None = None + if result_dir is not None: + result_path = Path(result_dir) / f"benchserve_{plan.source.source}_{int(time.time())}.json" + result_path.parent.mkdir(parents=True, exist_ok=True) + + argv = plan.bench_serve_argv( + model=resolved, + base_url=base_url, + backend=backend, + tokenizer=tokenizer, + max_concurrency=max_concurrency, + result_filename=str(result_path) if result_path else None, + seed=seed, + ) + + if dry_run: + notes.append("dry run: nothing was launched") + return RunResult(argv=argv, returncode=0, duration_s=0.0, resolved_model=resolved, + result_path=str(result_path) if result_path else None, + source=plan.source, notes=notes) + + # argv[0] is the documented command name; run the console script that belongs + # to THIS interpreter, so the install check_vllm() validated is the one that + # runs. See vllm_executable(). + exe = vllm_executable() + if exe != argv[0]: + notes.append(f"running {exe} (the console script beside {sys.executable})") + argv = [exe, *argv[1:]] + + started = time.monotonic() + try: + proc = subprocess.run(argv, capture_output=True, text=True, timeout=timeout_s) + rc, out, err = proc.returncode, proc.stdout, proc.stderr + except subprocess.TimeoutExpired as exc: + # A timeout is a result, not an exception to propagate: the partial output + # is the only evidence of how far it got. + rc, out, err = 124, (exc.stdout or b"").decode(errors="replace") if isinstance( + exc.stdout, bytes) else (exc.stdout or ""), f"timed out after {timeout_s:.0f}s" + notes.append(f"killed at the {timeout_s:.0f}s timeout") + duration = time.monotonic() - started + + result = None + if result_path is not None and result_path.exists(): + try: + result = json.loads(result_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + notes.append(f"result file unreadable: {exc}") + else: + if regime is None: + notes.append( + "result JSON carries no trace identity, no regime and no config " + "capture, and under --self-timed its request_rate and burstiness are " + "the CLI defaults rather than the trace's. Pass `regime=` to join it." + ) + else: + joined = join_result(result, plan, regime) + notes.append( + f"joined to {joined.regime_label}; dropped {sorted(joined.dropped)} " + "as CLI defaults that --self-timed never consulted" + ) + if not joined.reconciled: + notes.append( + "RESULT DOES NOT RECONCILE WITH ITS TRACE: " + + "; ".join( + f"{c.name} expected {c.expected} got {c.actual}" + for c in joined.failures() + ) + ) + elif result_path is not None: + notes.append(f"no result file at {result_path} despite --save-result") + + return RunResult( + joined=joined, + argv=argv, returncode=rc, duration_s=duration, vllm_version=found, + resolved_model=resolved, stdout_tail=out[-TAIL_CHARS:], stderr_tail=err[-TAIL_CHARS:], + result_path=str(result_path) if result_path else None, result=result, + source=plan.source, notes=notes, + ) diff --git a/gitm/traffic/schema.py b/gitm/traffic/schema.py new file mode 100644 index 0000000..87f9280 --- /dev/null +++ b/gitm/traffic/schema.py @@ -0,0 +1,204 @@ +"""Canonical request-trace shapes — one format every load source normalizes into. + +Three contracts: + +* :class:`CanonicalRequest` — one request. The unit every adapter emits and every + output mode consumes. A frozen slotted dataclass, not a pydantic model: + a trace is millions of these and they are hot data, not configuration. The + *validation* that would justify pydantic already happens in the adapter, where + a rejected row can be attributed to a named :class:`DropReason` instead of + raising. +* :class:`TraceMeta` — provenance for a whole trace: where the bytes came from, + their sha256, how many rows were read, how many survived, and **what was + dropped and why**. This is the part that gets serialized into a result row, so + it is a pydantic model with ``extra="forbid"``. +* :class:`Trace` — meta + requests, and the reason the two cannot be separated. + Nothing downstream accepts a bare list of requests: a stream you cannot trace + back to bytes is not evidence, and the playbook (deliverable 4) keys on this + provenance. + +Units, stated once and never re-derived: **``arrival_s`` is seconds offset from +the trace's first row**, not an epoch and not the source's native unit. Adapters +convert; :attr:`TraceMeta.raw_time_unit` records what they converted from. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum + +from pydantic import BaseModel, ConfigDict, Field + +#: Schema identity, in the style of ``gitm.bench.manifest.SCHEMA``. Bump on any +#: field change that a consumer could misread as the old meaning. +SCHEMA = "gitm.traffic.trace/v1" + + +class DropReason(str, Enum): + """Why an adapter refused a raw row. + + Every rejection is one of these — an adapter may not drop a row silently, and + the counts travel with the trace in :attr:`TraceMeta.drops`. The first seven + are *defects* in the source; :attr:`FILTERED_OUT` is a selection the caller + asked for and is counted separately so a narrow filter never reads as dirty + data. + """ + + MALFORMED_ROW = "malformed_row" # unparseable line / wrong column count + MISSING_FIELD = "missing_field" # column present in the header, empty in the row + NON_NUMERIC = "non_numeric" # a length or timestamp that is not a number + NEGATIVE_VALUE = "negative_value" # negative tokens or a negative timestamp + ZERO_INPUT_TOKENS = "zero_input_tokens" # nothing to prefill + ZERO_OUTPUT_TOKENS = "zero_output_tokens" # nothing to decode + NON_MONOTONIC_ARRIVAL = "non_monotonic_arrival" # timestamp went backwards + FILTERED_OUT = "filtered_out" # excluded by a caller-supplied filter, not a defect + + +@dataclass(frozen=True, slots=True) +class CanonicalRequest: + """One request, in the only shape the harness fires. + + ============== ============================= ========================================= + field type / unit when the source lacks it + ============== ============================= ========================================= + arrival_s float, **seconds from trace fatal. No adapter may synthesize arrivals; + start** (not epoch) a source without timing is not a trace. + input_tokens int, tokens, ``> 0`` fatal — the row is dropped, never defaulted. + output_tokens int | None, tokens ``None`` = "generate to the regime's sampled + length". Recorded in ``TraceMeta.notes``; + a trace of ``None`` cannot be replayed + as-is, only parameterized. + session_id str | None ``None`` — no conversation identity. + ``TraceMeta.has_session_identity`` is False + and multi-turn experiments must not use it. + prefix_blocks tuple[int, ...], block ids ``()`` — no prefix identity. + in prompt order ``TraceMeta.has_prefix_identity`` is False + and prefix-cache experiments must not use it. + ============== ============================= ========================================= + + ``prefix_blocks`` is a *chain*, not a single hash, because partial sharing is + the whole point: two requests share a prefix exactly as far as their leading + block ids agree. One hash of the whole chain would only ever match identical + prompts, which is the case that does not need measuring. Each id stands for + :attr:`TraceMeta.prefix_block_tokens` tokens. + + ``source_e2e_latency_s`` is the **source system's** end-to-end + submission-to-final-response time, when the trace records one (BurstGPT_3's + ``Elapsed time``). It is carried so the adapter does not destroy real data at + the boundary, and it is named at length so it cannot be mistaken for + something it is not: + + * it is **end-to-end**, not TTFT, and not ITL; + * it was measured on **someone else's hardware, model and load**. + + **Never compare it against a measured TTFT/ITL, and never promote a playbook + row against it.** Its legitimate use is bounding think-time between turns of + a session, where only the source's own timeline matters. + """ + + arrival_s: float + input_tokens: int + output_tokens: int | None = None + session_id: str | None = None + prefix_blocks: tuple[int, ...] = () + source_e2e_latency_s: float | None = None + + +class TraceMeta(BaseModel): + """Provenance for one trace. Without it a trace cannot be replayed. + + ``rows_read`` counts raw records seen; ``rows_emitted`` counts survivors; + ``drops`` maps :class:`DropReason` values to counts. The three must reconcile + (:meth:`Trace.__post_init__` checks it), so "we dropped some bad rows" is + never a hand-wave. + """ + + model_config = ConfigDict(extra="forbid") + + schema_id: str = SCHEMA + source: str # adapter name, e.g. "burstgpt" + path: str # the file that was read + sha256: str # of the raw bytes — pins the trace to bytes, per gitm.bench.manifest + source_url: str | None = None # where the raw file came from, when known + + rows_read: int = 0 + rows_emitted: int = 0 + drops: dict[str, int] = Field(default_factory=dict) + + span_s: float = 0.0 # last arrival minus first, seconds + raw_time_unit: str = "s" # what the source's timestamps were before conversion + + prefix_block_tokens: int | None = None # tokens each prefix-block id stands for + has_prefix_identity: bool = False + has_session_identity: bool = False + #: How much session identity there actually is. ``has_session_identity`` only + #: says *some* row carried one; in BurstGPT_3 that is true while 90 % of rows + #: are single-shot API traffic with no conversation at all. A multi-turn + #: experiment needs the counts, not the flag, to decide whether the trace can + #: carry it. + session_rows: int = 0 + sessions: int = 0 + + notes: list[str] = Field(default_factory=list) + + @property + def dropped(self) -> int: + return sum(self.drops.values()) + + @property + def defects(self) -> int: + """Dropped rows that were *bad data*, excluding caller-requested filtering.""" + return sum(v for k, v in self.drops.items() if k != DropReason.FILTERED_OUT.value) + + def summary(self) -> str: + drops = ", ".join(f"{k}={v}" for k, v in sorted(self.drops.items())) or "none" + return ( + f"{self.source}: {self.rows_emitted}/{self.rows_read} rows over " + f"{self.span_s:.1f}s (drops: {drops})" + ) + + +@dataclass(frozen=True) +class Trace: + """A normalized trace: provenance plus requests, inseparable by construction. + + ``requests`` are held in arrival order. The whole trace is materialized — + ponytail: fine to a few million rows (BurstGPT_1 is ~1.4 M), and the memory + ceiling is `slots` dataclasses at roughly 100 B each. If a source arrives that + does not fit, make the adapters yield and give ``Trace`` a streaming sibling; + nothing above this line assumes random access except the quantile fits. + """ + + meta: TraceMeta + requests: list[CanonicalRequest] = field(default_factory=list) + + def __post_init__(self) -> None: + if self.meta.rows_emitted != len(self.requests): + raise ValueError( + f"TraceMeta.rows_emitted={self.meta.rows_emitted} disagrees with " + f"{len(self.requests)} requests — provenance must reconcile" + ) + if self.meta.rows_read != self.meta.rows_emitted + self.meta.dropped: + raise ValueError( + f"rows_read={self.meta.rows_read} != emitted={self.meta.rows_emitted} " + f"+ dropped={self.meta.dropped} — a row went missing unattributed" + ) + + def __len__(self) -> int: + return len(self.requests) + + @property + def arrivals(self) -> list[float]: + return [r.arrival_s for r in self.requests] + + @property + def input_tokens(self) -> list[int]: + return [r.input_tokens for r in self.requests] + + @property + def output_tokens(self) -> list[int]: + """Output lengths, with ``None`` excluded — callers must check the count.""" + return [r.output_tokens for r in self.requests if r.output_tokens is not None] + + def rate_rps(self) -> float: + return len(self.requests) / self.meta.span_s if self.meta.span_s > 0 else 0.0 diff --git a/gitm/traffic/validate.py b/gitm/traffic/validate.py new file mode 100644 index 0000000..fb8dbd1 --- /dev/null +++ b/gitm/traffic/validate.py @@ -0,0 +1,275 @@ +"""Validation — prove the pipeline preserves the trace, and show the proof. + +The brief asks for this as a deliverable, not a nicety: the replayed stream's +arrival-rate and length distributions compared against the source's, **shown**, +with any mismatch explained. Everything downstream — every regime label, every +promoted playbook row — inherits whatever distortion this step fails to catch. + +What gets compared is the file the benchmark will actually consume. The replay +emitter writes a vLLM ``timed_trace`` JSONL; +:func:`gitm.traffic.replay.read_timed_trace` reads that same file back into +canonical form, and :func:`compare` puts it beside the trace the adapter +produced. So "the pipeline preserves the trace" is a measurement of the artifact, +not an argument about the code. + +The same function serves the second, looser use: a parameterized sample against +the trace it was fitted on. There the thresholds are wider — a sample is drawn +from the envelope, not copied from it — and :data:`SAMPLED_THRESHOLDS` says so +explicitly rather than leaving a reader to wonder which standard was applied. +""" + +from __future__ import annotations + +import sys + +import numpy as np +from pydantic import BaseModel, ConfigDict, Field + +from gitm.traffic.regime import DEFAULT_BIN_S, index_of_dispersion +from gitm.traffic.schema import Trace + + +class Thresholds(BaseModel): + """What counts as preserved. Every number here is a decision, not a default.""" + + model_config = ConfigDict(extra="forbid") + + name: str + #: Two-sample KS statistic on arrival times, and on each length distribution. + ks: float = 0.02 + #: Relative error on mean request rate. + rate: float = 0.01 + #: Relative error on the burstiness axis (index of dispersion). + burstiness: float = 0.10 + #: Relative error on request count. 0 = exact. + count: float = 0.0 + #: Whether to compare the arrival *timeline* at all. True for a replay, which + #: must reproduce it. False for a parameterized sample, which reproduces the + #: rate and the dispersion by construction and the timeline by nothing — a KS + #: on arrival times there measures only that a sample is not a copy, which is + #: the point of sampling. + check_arrival_times: bool = True + + +#: A replay must reproduce the trace, not resemble it: the emitter is a format +#: change, so every statistic should come back identical and the tolerances are +#: there to absorb float rounding of the timestamps, nothing else. +REPLAY_THRESHOLDS = Thresholds(name="replay", ks=0.001, rate=1e-6, burstiness=1e-6, count=0.0) + +#: A parameterized sample is a *draw* from the fitted envelope. Sampling error at +#: a few hundred requests is real, so these are the finite-sample tolerances — +#: loose enough not to fire on noise, tight enough that a broken inverse-CDF or a +#: mis-set dispersion target does fire. +SAMPLED_THRESHOLDS = Thresholds( + name="sampled", ks=0.15, rate=0.25, burstiness=0.60, count=0.35, check_arrival_times=False +) + +#: Arrival times are compared at microsecond resolution. Two reasons, and both +#: are about not lying: a replay's timing fidelity below 1 us is meaningless next +#: to millisecond network jitter, and the emitted JSONL rounds to 6 decimals, so +#: an un-quantized KS reports the 6e-16 s residue of ``5999 / 1000`` as a real +#: distribution gap. It found exactly that on the Mooncake fixture. +ARRIVAL_RESOLUTION_S = 1e-6 + + +class Check(BaseModel): + model_config = ConfigDict(extra="forbid") + + name: str + statistic: float + threshold: float + passed: bool + detail: str = "" + + +class ValidationReport(BaseModel): + model_config = ConfigDict(extra="forbid") + + standard: str + source: str + replayed: str + checks: list[Check] = Field(default_factory=list) + source_hist: list[int] = Field(default_factory=list) + replayed_hist: list[int] = Field(default_factory=list) + hist_bin_s: float = 0.0 + + @property + def passed(self) -> bool: + return all(c.passed for c in self.checks) + + def render(self) -> str: + """The shown comparison: the check table plus both arrival-rate profiles. + + ASCII on purpose — this lands in a terminal, a markdown log and a commit + message, and none of those render a PNG. A plotting dependency would buy + prettier and lose all three. + """ + w = max((len(c.name) for c in self.checks), default=4) + lines = [ + f"trace validation [{self.standard}] {self.source} -> {self.replayed}", + f"{'check'.ljust(w)} {'value':>12} {'threshold':>12} result", + f"{'-' * w} {'-' * 12} {'-' * 12} ------", + ] + for c in self.checks: + mark = "pass" if c.passed else "FAIL" + lines.append( + f"{c.name.ljust(w)} {c.statistic:>12.6g} {c.threshold:>12.6g} {mark}" + ) + lines.append("") + lines.append(f"arrival rate, {self.hist_bin_s:g}s bins (S = source, R = replayed)") + lines.append(_sparkbars(self.source_hist, self.replayed_hist)) + lines.append("") + lines.append("PASS — the pipeline preserves the trace" if self.passed else self.explain()) + return "\n".join(lines) + + def explain(self) -> str: + """Prose for every failed check. A printed number is not an explanation.""" + bad = [c for c in self.checks if not c.passed] + if not bad: + return "no mismatch to explain" + out = ["FAIL — the replayed stream differs from the source:"] + for c in bad: + out.append(f" * {c.name}: {c.statistic:.6g} exceeds {c.threshold:.6g}. {c.detail}") + return "\n".join(out) + + +_BLOCKS = " ▁▂▃▄▅▆▇█" +_ASCII = " .:-=+*#@" + + +def _ramp() -> str: + """Block characters when the console can encode them, ASCII when it cannot. + + A Windows console on cp1252 raises ``UnicodeEncodeError`` on U+2588, which + would turn "show the comparison" into a crash at exactly the moment someone + is looking at a failure. + """ + enc = getattr(sys.stdout, "encoding", None) or "utf-8" + try: + _BLOCKS.encode(enc) + except (UnicodeEncodeError, LookupError): + return _ASCII + return _BLOCKS + + +def _spark(counts: list[int], peak: int) -> str: + ramp = _ramp() + if peak <= 0: + return " " * len(counts) + return "".join( + ramp[min(int(c / peak * (len(ramp) - 1) + 0.5), len(ramp) - 1)] for c in counts + ) + + +def _sparkbars(a: list[int], b: list[int]) -> str: + peak = max([*a, *b, 1]) + return f" S |{_spark(a, peak)}| peak {max(a, default=0)}\n R |{_spark(b, peak)}| peak {max(b, default=0)}" + + +def ks_statistic(a: list[float] | np.ndarray, b: list[float] | np.ndarray) -> float: + """Two-sample Kolmogorov-Smirnov statistic — the max CDF gap. + + Eight lines of numpy rather than a scipy dependency: only the statistic is + needed, never the p-value, because the thresholds here are operational + ("this replay is faithful enough to run on") and not a hypothesis test. + """ + a = np.sort(np.asarray(a, dtype=float)) + b = np.sort(np.asarray(b, dtype=float)) + if a.size == 0 or b.size == 0: + return 1.0 + grid = np.concatenate([a, b]) + ca = np.searchsorted(a, grid, side="right") / a.size + cb = np.searchsorted(b, grid, side="right") / b.size + return float(np.max(np.abs(ca - cb))) + + +def _quantize(xs: list[float]) -> np.ndarray: + """Snap arrival times to :data:`ARRIVAL_RESOLUTION_S` before comparing them.""" + return np.rint(np.asarray(xs, dtype=float) / ARRIVAL_RESOLUTION_S) + + +def _hist(trace: Trace, *, nbins: int, span: float) -> list[int]: + if span <= 0 or not trace.requests: + return [len(trace.requests)] + [0] * (nbins - 1) + idx = np.minimum((np.asarray(trace.arrivals) / span * nbins).astype(int), nbins - 1) + return [int(v) for v in np.bincount(idx, minlength=nbins)] + + +def _rel(a: float, b: float) -> float: + """Relative error of ``b`` against ``a``, with a zero-safe denominator.""" + return abs(a - b) / abs(a) if a else (0.0 if not b else 1.0) + + +def compare( + source: Trace, + replayed: Trace, + *, + thresholds: Thresholds = REPLAY_THRESHOLDS, + bin_s: float = DEFAULT_BIN_S, + hist_bins: int = 60, +) -> ValidationReport: + """Compare a replayed (or sampled) trace against its source.""" + checks: list[Check] = [] + + def add(name: str, stat: float, thr: float, detail: str) -> None: + checks.append( + Check(name=name, statistic=stat, threshold=thr, passed=stat <= thr, detail=detail) + ) + + add( + "request_count", + _rel(len(source), len(replayed)), + thresholds.count, + f"{len(source)} source vs {len(replayed)} replayed — requests were lost or " + "invented between the adapter and the emitted file.", + ) + if thresholds.check_arrival_times: + add( + "arrival_ks", + ks_statistic(_quantize(source.arrivals), _quantize(replayed.arrivals)), + thresholds.ks, + "the replayed arrival times do not follow the source's; timing fidelity " + "is lost, so every burstiness-conditioned result is suspect.", + ) + add( + "input_len_ks", + ks_statistic(source.input_tokens, replayed.input_tokens), + thresholds.ks, + "prompt lengths differ — check block coverage: vLLM expands hash_ids at " + "--timed-trace-chunk-hash-size tokens each and truncates silently when the " + "size is too small (Mooncake is 512, the vLLM default is 16).", + ) + add( + "output_len_ks", + ks_statistic(source.output_tokens, replayed.output_tokens), + thresholds.ks, + "decode lengths differ, which moves the prefill/decode ratio the regime " + "axis is defined on.", + ) + add( + "rate_rps", + _rel(source.rate_rps(), replayed.rate_rps()), + thresholds.rate, + "mean offered rate differs from the source's.", + ) + add( + "burstiness", + _rel( + index_of_dispersion(source.arrivals, bin_s=bin_s, span_s=source.meta.span_s), + index_of_dispersion(replayed.arrivals, bin_s=bin_s, span_s=replayed.meta.span_s), + ), + thresholds.burstiness, + "the index of dispersion moved, so the two traces sit in different regime " + "buckets even where their means agree.", + ) + + span = max(source.meta.span_s, replayed.meta.span_s) + return ValidationReport( + standard=thresholds.name, + source=f"{source.meta.source}({source.meta.sha256[:12]})", + replayed=f"{replayed.meta.source}({replayed.meta.sha256[:12]})", + checks=checks, + source_hist=_hist(source, nbins=hist_bins, span=span), + replayed_hist=_hist(replayed, nbins=hist_bins, span=span), + hist_bin_s=span / hist_bins if span > 0 else 0.0, + ) diff --git a/pyproject.toml b/pyproject.toml index 40bb220..d707c25 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,10 +29,27 @@ nvidia = ["pynvml>=11.5"] # fallback (pandas). On a GPU box the harness runs on cuDF/CuPy instead — those # install from the RAPIDS channel, not PyPI, so they are not listed here. bench = ["pyarrow>=15", "pandas>=2.0"] -# The vllm-decode workload. Deliberately NOT folded into [gpu]: vLLM is a heavy +# Two consumers: the vllm-decode workload, and *firing* a traffic replay +# (gitm/traffic/replay.py). Deliberately NOT folded into [gpu]: vLLM is a heavy # install that pins its own torch, and both must be built for a CUDA major the # HOST driver supports — see gitm/cuda_env.py, which checks exactly that. -vllm = ["vllm>=0.6"] +# +# Floor is 0.23.0 because `bench serve --dataset-name timed_trace` landed in +# bfb9ebc21 (2026-05-28), one day AFTER v0.22.0 shipped — checked at the tags, +# not inferred from dates: v0.22.0 ❌, v0.22.1 ❌, v0.23.0 ✅. Below it the run +# dies on an argparse complaint about an unknown dataset name, which reads like a +# typo in our command rather than a missing feature. A too-high floor fails +# loudly at install time; a too-low one fails quietly at run time, so the floor +# goes up. gitm.traffic.replay.VLLM_MIN_VERSION carries the same number for the +# runtime guard. +# +# NOT split into a separate [traffic] extra: the traffic library is CPU-only +# except for firing — adapters, regime, replay-file writing, validation and the +# viewer all run with no vLLM at all — so an extra named for it would say you +# need a multi-gigabyte CUDA install to use a library that does not need one. +# Phase B of the chunked-prefill experiment needs only >=0.19.0 for its own +# features; it is covered by this floor, not constrained to it. +vllm = ["vllm>=0.23.0"] # Full GPU stack for the real HFT path in two commands: cuDF/CuPy (RAPIDS) + # the CUPTI/runtime wheels the tracer shim links + NVML telemetry + the Parquet # generator deps. cuDF/CuPy resolve from NVIDIA's index, so install with: diff --git a/tests/test_traffic.py b/tests/test_traffic.py new file mode 100644 index 0000000..9d8e5b6 --- /dev/null +++ b/tests/test_traffic.py @@ -0,0 +1,30 @@ +"""Traffic replay library — the pytest face of ``python -m gitm.traffic --selftest``. + +The assertions live in :mod:`gitm.traffic._selftest` and are called from both +places, so the runnable check a reader is told about in the spec and the check CI +runs are the *same* check, not two that can drift apart. + +Each case is one function from ``_selftest.CHECKS``; a failure names the check +that broke rather than the whole library. +""" + +from __future__ import annotations + +import pytest + +from gitm.traffic import _selftest + + +@pytest.mark.parametrize("check", _selftest.CHECKS, ids=lambda f: f.__name__) +def test_traffic_check(check) -> None: + if not _selftest.FIXTURES.exists(): + pytest.skip(f"fixtures not present at {_selftest.FIXTURES}") + check() + + +def test_every_check_is_registered() -> None: + """A check that exists but is never run is worse than no check.""" + defined = { + name for name in dir(_selftest) if name.startswith("check_") and callable(getattr(_selftest, name)) + } + assert {f.__name__ for f in _selftest.CHECKS} == defined From f11062b8d482d77f690db7db7a1e83c4f606faf1 Mon Sep 17 00:00:00 2001 From: Rahul-Ganesan Date: Thu, 3 Sep 2026 19:18:13 -0400 Subject: [PATCH 2/8] playbook: the row schema and its match semantics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `gitm/playbook/` — what a promoted tuning result *is*, and when it may be applied to live traffic. The contract between detection and the apply runtime, so it ships as types rather than as a doc two implementations read differently. `match.py` is separate from `schema.py` because the match semantics, not the field list, is the part that gets argued about. Identity is a split. Exact equality on model+revision, GPU SKU, engine+version, source_kind, concurrency and the knob key set; distance on the numeric regime axes, because live traffic never lands on a measured point. Each gate is a named MatchPolicy field, so loosening one is an edit visible in a diff rather than an accident inside a comparison. The source_kind gate is asserted with every numeric axis identical, so only the gate can be doing the work — a scoreboard result never satisfies a production query. Distance is |log2(a/b)| on the raw axes: scale-free, symmetric, zero at equality, inf when one side is zero, because "no output tokens" is not a small version of "some". Burstiness uses the shifted ratio so a perfectly paced trace stays comparable instead of returning infinity. Six axes — the p95s are in because the p50s alone cannot separate a long-context workload from a short one with a heavy tail, which is exactly where prefix caching and chunked prefill live. Combined with L-infinity, not a mean: a row is as far away as its worst axis. Asserted on the case it exists for — identical on five axes and 8x off on input_p95 reads 0.5 as a mean, which applies the row, and 3.0 as L-inf, which does not. For scale the two real traces are 4.929 apart, limited by input_p95. rate_rps is deliberately not an axis and the exclusion is material rather than an omission: two regimes identical but for an 8x rate difference are 0.0 apart by default and 3.0 with the axis on. The inclusion rule and the experiment that would settle it are written down. No distance threshold ships. AxisTolerance refuses a max_distance without the run that produced it, and the shipped policy carries none. An exact regime match still returns a row, so the schema is usable today; any nonzero distance returns UNCALIBRATED, names the limiting axis, and routes to discovery. Calibration needs the same knob measured across nearby regimes, which needs a GPU — the cost of the open state is a discovery run, the cost of an invented threshold is a knob applied to traffic nobody measured it on. A row cannot be constructed without provenance: no knobs, repeats < 2, a delta missing its latency percentiles, or an empty trace_sha256 all fail. MeasuredDelta carries throughput and TTFT/ITL percentiles, so the schema cannot express a throughput-only row. Invalidation is a field with a reason, never a deletion. `evidence` separates measured from illustrative, because the worked examples ship in the same format as real rows. delta_is_floor falls out of the traces: BurstGPT has no prefix identity, so a replay synthesizes unique blocks per request and a prefix-cache knob measured there saw the least reuse the real traffic could have had. Its delta is a lower bound, never quotable as the gain. ex6 is that case; ex2 is the control. row_from_runs builds a row from two joined runs — one BenchRun is one arm, a row is the difference — refusing arms that did not run the same trace, any run that did not reconcile, unequal repeat counts, and a missing latency percentile. Medians, never means. The shipped examples are 6 rows, 0 selectable: regimes real off the pinned fixtures, deltas invented, every row saying so. A perfectly matching query returns nothing and names the reason per row. The largest claimed delta in the file is the scoreboard row, gated out by equality rather than distance. 19 selftest checks, the same assertions as 20 pytest cases, ruff clean. --- benchmarks/playbook/examples.json | 400 +++++++++++++++++++ benchmarks/playbook/make_examples.py | 140 +++++++ docs/playbook-schema.md | 343 ++++++++++++++++ gitm/playbook/__init__.py | 82 ++++ gitm/playbook/__main__.py | 89 +++++ gitm/playbook/_selftest.py | 568 +++++++++++++++++++++++++++ gitm/playbook/match.py | 452 +++++++++++++++++++++ gitm/playbook/schema.py | 470 ++++++++++++++++++++++ tests/test_playbook.py | 29 ++ 9 files changed, 2573 insertions(+) create mode 100644 benchmarks/playbook/examples.json create mode 100644 benchmarks/playbook/make_examples.py create mode 100644 docs/playbook-schema.md create mode 100644 gitm/playbook/__init__.py create mode 100644 gitm/playbook/__main__.py create mode 100644 gitm/playbook/_selftest.py create mode 100644 gitm/playbook/match.py create mode 100644 gitm/playbook/schema.py create mode 100644 tests/test_playbook.py diff --git a/benchmarks/playbook/examples.json b/benchmarks/playbook/examples.json new file mode 100644 index 0000000..cbe5864 --- /dev/null +++ b/benchmarks/playbook/examples.json @@ -0,0 +1,400 @@ +{ + "schema_id": "gitm.playbook.row/v1", + "rows": [ + { + "schema_id": "gitm.playbook.row/v1", + "row_id": "ex1-prefix-cache-mooncake", + "identity": { + "model": "Qwen/Qwen3.6-35B-A3B-FP8", + "model_revision": "95a723d0", + "gpu_sku": "NVIDIA H100 80GB", + "env": { + "schema_id": "pending-adit/env-capture", + "engine": "vllm", + "engine_version": "0.11.0", + "driver_version": null, + "torch_version": null, + "cuda_version": null + }, + "regime": { + "source_kind": "production", + "trace": "mooncake", + "requests": 400, + "rate_rps": 2.8368794326241136, + "io_ratio": 39.09367234191124, + "input_p50": 9075, + "input_p95": 49904, + "output_p50": 370, + "output_p95": 662, + "burstiness": 6.738120567375886, + "bin_s": 1.0, + "burstiness_defined": true, + "concurrency": null, + "in_envelope": true, + "notes": [] + }, + "knobs": { + "enable_prefix_caching": true + } + }, + "delta": { + "throughput_pct": 14.5, + "ttft_p99_ms": -18.0, + "itl_p99_ms": 0.3, + "repeats": 5, + "throughput_ci95_pct": [ + 9.1, + 19.4 + ], + "latency_blowout": null + }, + "provenance": { + "trace_source": "mooncake", + "trace_sha256": "8090d6a38401d280fd88112ba22007ff4354be59c986f4f6bcc250edd45a52dc", + "trace_drops": {}, + "regime_label": "prod/io32/in8k/out256/burst-hi/copen", + "repeat_raw_data": [], + "promotion_rule": "pending-adit/promotion-rule", + "config_capture": "pending-adit", + "verified_at": "2026-09-02T00:00:00Z", + "replay_chunk_hash_size": 512, + "replay_self_timed": true, + "prefix_synthesized": false + }, + "evidence": "illustrative", + "invalidated": null, + "notes": [ + "Regime is real (measured off the pinned Mooncake fixture). The delta is not: no run against a live endpoint has happened. Shape to build against, not a claim." + ] + }, + { + "schema_id": "gitm.playbook.row/v1", + "row_id": "ex2-max-num-seqs-burstgpt", + "identity": { + "model": "Qwen/Qwen3.6-35B-A3B-FP8", + "model_revision": "95a723d0", + "gpu_sku": "NVIDIA H100 80GB", + "env": { + "schema_id": "pending-adit/env-capture", + "engine": "vllm", + "engine_version": "0.11.0", + "driver_version": null, + "torch_version": null, + "cuda_version": null + }, + "regime": { + "source_kind": "production", + "trace": "burstgpt", + "requests": 383, + "rate_rps": 0.010276637419839545, + "io_ratio": 1.8821556431490254, + "input_p50": 353, + "input_p95": 1638, + "output_p50": 238, + "output_p95": 841, + "burstiness": 1.0106110910396904, + "bin_s": 1.0, + "burstiness_defined": true, + "concurrency": null, + "in_envelope": true, + "notes": [] + }, + "knobs": { + "max_num_seqs": 64 + } + }, + "delta": { + "throughput_pct": 8.1, + "ttft_p99_ms": -4.2, + "itl_p99_ms": 1.0, + "repeats": 5, + "throughput_ci95_pct": [ + 3.3, + 12.6 + ], + "latency_blowout": null + }, + "provenance": { + "trace_source": "burstgpt", + "trace_sha256": "7865fda5dac50938a84dfccb06c405e2f5202969ecfbd671c8399de00f4335f0", + "trace_drops": { + "zero_input_tokens": 17 + }, + "regime_label": "prod/io1/in256/out128/burst-poisson/copen", + "repeat_raw_data": [], + "promotion_rule": "pending-adit/promotion-rule", + "config_capture": "pending-adit", + "verified_at": "2026-09-02T00:00:00Z", + "replay_chunk_hash_size": 512, + "replay_self_timed": true, + "prefix_synthesized": true + }, + "evidence": "illustrative", + "invalidated": null, + "notes": [ + "The counter-example to ex1: same model, same GPU, no prefix identity in the source at all. A prefix-caching row must never be selected for this regime." + ] + }, + { + "schema_id": "gitm.playbook.row/v1", + "row_id": "ex3-chunked-prefill-qwen", + "identity": { + "model": "Qwen/Qwen3.6-35B-A3B-FP8", + "model_revision": "95a723d0", + "gpu_sku": "NVIDIA H100 80GB", + "env": { + "schema_id": "pending-adit/env-capture", + "engine": "vllm", + "engine_version": "0.11.0", + "driver_version": null, + "torch_version": null, + "cuda_version": null + }, + "regime": { + "source_kind": "production", + "trace": "mooncake", + "requests": 400, + "rate_rps": 2.8368794326241136, + "io_ratio": 39.09367234191124, + "input_p50": 9075, + "input_p95": 49904, + "output_p50": 370, + "output_p95": 662, + "burstiness": 6.738120567375886, + "bin_s": 1.0, + "burstiness_defined": true, + "concurrency": null, + "in_envelope": true, + "notes": [] + }, + "knobs": { + "enable_chunked_prefill": true, + "max_num_batched_tokens": 8192 + } + }, + "delta": { + "throughput_pct": 6.4, + "ttft_p99_ms": -31.0, + "itl_p99_ms": 2.1, + "repeats": 5, + "throughput_ci95_pct": null, + "latency_blowout": null + }, + "provenance": { + "trace_source": "mooncake", + "trace_sha256": "8090d6a38401d280fd88112ba22007ff4354be59c986f4f6bcc250edd45a52dc", + "trace_drops": {}, + "regime_label": "prod/io32/in8k/out256/burst-hi/copen", + "repeat_raw_data": [], + "promotion_rule": "pending-adit/promotion-rule", + "config_capture": "pending-adit", + "verified_at": "2026-09-02T00:00:00Z", + "replay_chunk_hash_size": 512, + "replay_self_timed": true, + "prefix_synthesized": false + }, + "evidence": "illustrative", + "invalidated": null, + "notes": [ + "This is the row deliverable 3 would produce. It stays illustrative until Phase B runs -- that needs one 80 GB card, which we do not have." + ] + }, + { + "schema_id": "gitm.playbook.row/v1", + "row_id": "ex4-retired-engine-bump", + "identity": { + "model": "Qwen/Qwen3.6-35B-A3B-FP8", + "model_revision": "95a723d0", + "gpu_sku": "NVIDIA H100 80GB", + "env": { + "schema_id": "pending-adit/env-capture", + "engine": "vllm", + "engine_version": "0.12.0", + "driver_version": null, + "torch_version": null, + "cuda_version": null + }, + "regime": { + "source_kind": "production", + "trace": "mooncake", + "requests": 400, + "rate_rps": 2.8368794326241136, + "io_ratio": 39.09367234191124, + "input_p50": 9075, + "input_p95": 49904, + "output_p50": 370, + "output_p95": 662, + "burstiness": 6.738120567375886, + "bin_s": 1.0, + "burstiness_defined": true, + "concurrency": null, + "in_envelope": true, + "notes": [] + }, + "knobs": { + "enable_prefix_caching": true + } + }, + "delta": { + "throughput_pct": 11.0, + "ttft_p99_ms": -12.0, + "itl_p99_ms": 0.4, + "repeats": 4, + "throughput_ci95_pct": null, + "latency_blowout": null + }, + "provenance": { + "trace_source": "mooncake", + "trace_sha256": "8090d6a38401d280fd88112ba22007ff4354be59c986f4f6bcc250edd45a52dc", + "trace_drops": {}, + "regime_label": "prod/io32/in8k/out256/burst-hi/copen", + "repeat_raw_data": [], + "promotion_rule": "pending-adit/promotion-rule", + "config_capture": "pending-adit", + "verified_at": "2026-09-02T00:00:00Z", + "replay_chunk_hash_size": 512, + "replay_self_timed": true, + "prefix_synthesized": false + }, + "evidence": "illustrative", + "invalidated": { + "reason": "vLLM 0.11 -> 0.12 scheduler rewrite; the measured delta is against a scheduler that no longer exists", + "at": "2026-09-02T00:00:00Z", + "by": "validation" + }, + "notes": [ + "Kept, not deleted. A deleted row leaves no record that the claim was made, which is the first thing a reviewer asks for." + ] + }, + { + "schema_id": "gitm.playbook.row/v1", + "row_id": "ex6-prefix-cache-on-a-synthesized-trace", + "identity": { + "model": "Qwen/Qwen3.6-35B-A3B-FP8", + "model_revision": "95a723d0", + "gpu_sku": "NVIDIA H100 80GB", + "env": { + "schema_id": "pending-adit/env-capture", + "engine": "vllm", + "engine_version": "0.11.0", + "driver_version": null, + "torch_version": null, + "cuda_version": null + }, + "regime": { + "source_kind": "production", + "trace": "burstgpt", + "requests": 383, + "rate_rps": 0.010276637419839545, + "io_ratio": 1.8821556431490254, + "input_p50": 353, + "input_p95": 1638, + "output_p50": 238, + "output_p95": 841, + "burstiness": 1.0106110910396904, + "bin_s": 1.0, + "burstiness_defined": true, + "concurrency": null, + "in_envelope": true, + "notes": [] + }, + "knobs": { + "enable_prefix_caching": true + } + }, + "delta": { + "throughput_pct": 1.2, + "ttft_p99_ms": -0.4, + "itl_p99_ms": 0.0, + "repeats": 5, + "throughput_ci95_pct": null, + "latency_blowout": null + }, + "provenance": { + "trace_source": "burstgpt", + "trace_sha256": "7865fda5dac50938a84dfccb06c405e2f5202969ecfbd671c8399de00f4335f0", + "trace_drops": { + "zero_input_tokens": 17 + }, + "regime_label": "prod/io1/in256/out128/burst-poisson/copen", + "repeat_raw_data": [], + "promotion_rule": "pending-adit/promotion-rule", + "config_capture": "pending-adit", + "verified_at": "2026-09-02T00:00:00Z", + "replay_chunk_hash_size": 512, + "replay_self_timed": true, + "prefix_synthesized": true + }, + "evidence": "illustrative", + "invalidated": null, + "notes": [ + "delta_is_floor: BurstGPT has no prefix identity, so D1 synthesized unique blocks per request and the replay saw the LEAST reuse the real traffic could have had. +1.2% is a lower bound, never quotable as the gain." + ] + }, + { + "schema_id": "gitm.playbook.row/v1", + "row_id": "ex5-scoreboard-not-production", + "identity": { + "model": "Qwen/Qwen3.6-35B-A3B-FP8", + "model_revision": "95a723d0", + "gpu_sku": "NVIDIA H100 80GB", + "env": { + "schema_id": "pending-adit/env-capture", + "engine": "vllm", + "engine_version": "0.11.0", + "driver_version": null, + "torch_version": null, + "cuda_version": null + }, + "regime": { + "source_kind": "scoreboard", + "trace": "artificialanalysis-fixed", + "requests": 1000, + "rate_rps": 1.0, + "io_ratio": 4.0, + "input_p50": 1024, + "input_p95": 1024, + "output_p50": 256, + "output_p95": 256, + "burstiness": 0.0, + "bin_s": 1.0, + "burstiness_defined": true, + "concurrency": null, + "in_envelope": true, + "notes": [ + "fixed-length scoreboard condition; not production traffic" + ] + }, + "knobs": { + "max_num_seqs": 64 + } + }, + "delta": { + "throughput_pct": 22.0, + "ttft_p99_ms": -9.0, + "itl_p99_ms": 0.1, + "repeats": 3, + "throughput_ci95_pct": null, + "latency_blowout": null + }, + "provenance": { + "trace_source": "mooncake", + "trace_sha256": "8090d6a38401d280fd88112ba22007ff4354be59c986f4f6bcc250edd45a52dc", + "trace_drops": {}, + "regime_label": "board/io4/in1k/out256/burst-flat/copen", + "repeat_raw_data": [], + "promotion_rule": "pending-adit/promotion-rule", + "config_capture": "pending-adit", + "verified_at": "2026-09-02T00:00:00Z", + "replay_chunk_hash_size": 512, + "replay_self_timed": true, + "prefix_synthesized": false + }, + "evidence": "illustrative", + "invalidated": null, + "notes": [ + "source_kind=scoreboard. The biggest claimed delta in the file, and it is gated out of every production lookup by equality, not by distance." + ] + } + ] +} diff --git a/benchmarks/playbook/make_examples.py b/benchmarks/playbook/make_examples.py new file mode 100644 index 0000000..906ce04 --- /dev/null +++ b/benchmarks/playbook/make_examples.py @@ -0,0 +1,140 @@ +"""Generate benchmarks/playbook/examples.json from the REAL D1 regimes. + +The regimes are measured off the committed traffic fixtures; the deltas are not +measured and every row says so (evidence=illustrative). +""" +import json +from datetime import datetime, timezone +from pathlib import Path + +from gitm.bench.manifest import sha256_file +from gitm.playbook.schema import ( + EnvCapture, + Evidence, + Invalidation, + MeasuredDelta, + Playbook, + PlaybookRow, + Provenance, + RowIdentity, +) +from gitm.traffic import Regime, SourceKind, read_burstgpt, read_mooncake + +F = Path("benchmarks/traffic_replay/fixtures") +bg_path, mc_path = F / "burstgpt_slice.csv", F / "mooncake_slice.jsonl" +bg = read_burstgpt(bg_path) +mc = read_mooncake(mc_path) +bg_r = Regime.from_trace(bg) +mc_r = Regime.from_trace(mc) + +H100 = "NVIDIA H100 80GB" +MODEL = "Qwen/Qwen3.6-35B-A3B-FP8" +REV = "95a723d0" +ENV = EnvCapture(engine="vllm", engine_version="0.11.0") +ENV_NEXT = EnvCapture(engine="vllm", engine_version="0.12.0") +T = datetime(2026, 9, 2, tzinfo=timezone.utc) + + +def prov(trace, path, regime, **kw): + """Provenance as D1 would emit it: real checksum, real drops, real replay + conditions. BurstGPT has no prefix identity, so a replay of it always + synthesizes blocks -- which is what makes ex6 a floor and not a measurement.""" + kw.setdefault("replay_chunk_hash_size", 512) + kw.setdefault("replay_self_timed", True) + kw.setdefault("prefix_synthesized", not trace.meta.has_prefix_identity) + return Provenance( + trace_source=trace.meta.source, + trace_sha256=sha256_file(path)[0], + trace_drops=dict(trace.meta.drops), + regime_label=regime.label(), + repeat_raw_data=[], + verified_at=T, + **kw, + ) + + +# A scoreboard regime: Artificial Analysis' fixed-length condition, as its own +# named source_kind. Numbers are the published fixed shape, not a measurement of +# ours -- which is exactly why it must never match a production query. +board_r = mc_r.model_copy(update={ + "source_kind": SourceKind.SCOREBOARD, "trace": "artificialanalysis-fixed", + "requests": 1000, "rate_rps": 1.0, "io_ratio": 4.0, + "input_p50": 1024, "input_p95": 1024, "output_p50": 256, "output_p95": 256, + "burstiness": 0.0, "notes": ["fixed-length scoreboard condition; not production traffic"], +}) + +rows = [ + PlaybookRow( + row_id="ex1-prefix-cache-mooncake", + identity=RowIdentity(model=MODEL, model_revision=REV, gpu_sku=H100, env=ENV, + regime=mc_r, knobs={"enable_prefix_caching": True}), + delta=MeasuredDelta(throughput_pct=14.5, ttft_p99_ms=-18.0, itl_p99_ms=0.3, + repeats=5, throughput_ci95_pct=(9.1, 19.4)), + provenance=prov(mc, mc_path, mc_r), + evidence=Evidence.ILLUSTRATIVE, + notes=["Regime is real (measured off the pinned Mooncake fixture). The delta is not: " + "no run against a live endpoint has happened. Shape to build against, not a claim."], + ), + PlaybookRow( + row_id="ex2-max-num-seqs-burstgpt", + identity=RowIdentity(model=MODEL, model_revision=REV, gpu_sku=H100, env=ENV, + regime=bg_r, knobs={"max_num_seqs": 64}), + delta=MeasuredDelta(throughput_pct=8.1, ttft_p99_ms=-4.2, itl_p99_ms=1.0, + repeats=5, throughput_ci95_pct=(3.3, 12.6)), + provenance=prov(bg, bg_path, bg_r), + evidence=Evidence.ILLUSTRATIVE, + notes=["The counter-example to ex1: same model, same GPU, no prefix identity in the " + "source at all. A prefix-caching row must never be selected for this regime."], + ), + PlaybookRow( + row_id="ex3-chunked-prefill-qwen", + identity=RowIdentity(model=MODEL, model_revision=REV, gpu_sku=H100, env=ENV, + regime=mc_r, knobs={"enable_chunked_prefill": True, + "max_num_batched_tokens": 8192}), + delta=MeasuredDelta(throughput_pct=6.4, ttft_p99_ms=-31.0, itl_p99_ms=2.1, repeats=5), + provenance=prov(mc, mc_path, mc_r, promotion_rule="pending-adit/promotion-rule"), + evidence=Evidence.ILLUSTRATIVE, + notes=["This is the row deliverable 3 would produce. It stays illustrative until " + "Phase B runs -- that needs one 80 GB card, which we do not have."], + ), + PlaybookRow( + row_id="ex4-retired-engine-bump", + identity=RowIdentity(model=MODEL, model_revision=REV, gpu_sku=H100, env=ENV_NEXT, + regime=mc_r, knobs={"enable_prefix_caching": True}), + delta=MeasuredDelta(throughput_pct=11.0, ttft_p99_ms=-12.0, itl_p99_ms=0.4, repeats=4), + provenance=prov(mc, mc_path, mc_r), + evidence=Evidence.ILLUSTRATIVE, + invalidated=Invalidation(reason="vLLM 0.11 -> 0.12 scheduler rewrite; the measured " + "delta is against a scheduler that no longer exists", + at=datetime(2026, 9, 2, tzinfo=timezone.utc), by="validation"), + notes=["Kept, not deleted. A deleted row leaves no record that the claim was made, " + "which is the first thing a reviewer asks for."], + ), + PlaybookRow( + row_id="ex6-prefix-cache-on-a-synthesized-trace", + identity=RowIdentity(model=MODEL, model_revision=REV, gpu_sku=H100, env=ENV, + regime=bg_r, knobs={"enable_prefix_caching": True}), + delta=MeasuredDelta(throughput_pct=1.2, ttft_p99_ms=-0.4, itl_p99_ms=0.0, repeats=5), + provenance=prov(bg, bg_path, bg_r), + evidence=Evidence.ILLUSTRATIVE, + notes=["delta_is_floor: BurstGPT has no prefix identity, so D1 synthesized unique " + "blocks per request and the replay saw the LEAST reuse the real traffic could " + "have had. +1.2% is a lower bound, never quotable as the gain."], + ), + PlaybookRow( + row_id="ex5-scoreboard-not-production", + identity=RowIdentity(model=MODEL, model_revision=REV, gpu_sku=H100, env=ENV, + regime=board_r, knobs={"max_num_seqs": 64}), + delta=MeasuredDelta(throughput_pct=22.0, ttft_p99_ms=-9.0, itl_p99_ms=0.1, repeats=3), + provenance=prov(mc, mc_path, board_r, config_capture="pending-adit"), + evidence=Evidence.ILLUSTRATIVE, + notes=["source_kind=scoreboard. The biggest claimed delta in the file, and it is " + "gated out of every production lookup by equality, not by distance."], + ), +] + +out = Path("benchmarks/playbook/examples.json") +out.parent.mkdir(parents=True, exist_ok=True) +out.write_text(json.dumps(Playbook(rows=rows).model_dump(mode="json"), indent=2) + "\n", + encoding="utf-8") +print(f"wrote {out} — {len(rows)} rows, {sum(r.selectable for r in rows)} selectable") diff --git a/docs/playbook-schema.md b/docs/playbook-schema.md new file mode 100644 index 0000000..ea5aa84 --- /dev/null +++ b/docs/playbook-schema.md @@ -0,0 +1,343 @@ +# Playbook schema + +The contract between detection and apply. A **row** is one promoted tuning +result; a **lookup** decides whether a past row applies to the workload in front +of you. Types in `gitm/playbook/schema.py`, matching in `gitm/playbook/match.py`, +worked rows in `benchmarks/playbook/examples.json`. + +> **Read this first.** The distance *metric* below is designed and implemented. +> The distance *threshold* is **not calibrated**, no measurement exists that +> could calibrate it yet, and the shipped policy carries no number at all. Under +> `UNCALIBRATED_POLICY` the only automatic match is an **exact** regime match; +> everything else routes to conservative discovery. That is the honest state, and +> §5 says exactly what would change it. + +```bash +python -m gitm.playbook --selftest # 17 checks +python -m gitm.playbook --show benchmarks/playbook/examples.json +python -m gitm.playbook --distance benchmarks/playbook/examples.json ex1-… ex2-… +python -m gitm.playbook --lookup benchmarks/playbook/examples.json ex1-… +``` + +--- + +## 1. What a row is + +``` +(model + revision, GPU SKU, environment, workload regime, knob set) + -> measured delta + provenance +``` + +| part | type | matched how | +|---|---|---| +| `model`, `model_revision` | `str` | **exact** | +| `gpu_sku` | `str` | **exact** | +| `env` (`EnvCapture`) | engine + engine version | **exact** (§3) | +| `regime.source_kind` | `production \| synthetic \| scoreboard` | **exact** | +| `regime.concurrency` | `int \| None` | **exact**, by policy | +| `knobs` | `dict[str, bool\|int\|float\|str]` | **exact**, key set | +| `regime` numeric axes | `Regime` | **distance** (§2) | + +`Regime` is deliverable 1's type, **imported**. There is no second copy — a +second one would drift inside a week and the distance would be measured in two +different coordinate systems. `check_regime_is_imported_not_redeclared` asserts +the field's annotation is literally `gitm.traffic.regime.Regime`. + +### What the types refuse to hold + +A type cannot see how a number was produced, so it cannot enforce "came through +the promotion rule". What it does instead is refuse rows that **could not** have: + +| refused | why | +|---|---| +| `knobs={}` | a row with no knob set says nothing | +| `repeats < 2` | a single run has no variance and cannot clear D2-1 | +| a delta with throughput but no TTFT/ITL | D2 criterion 3 would be unenforceable | +| `trace_sha256=""` | a claim that cannot be traced back to bytes | +| any unknown field | `extra="forbid"` everywhere except `EnvCapture` (§3) | + +### Two fields that are states, not absences + +- **`evidence`** — `measured` or `illustrative`. The worked examples ship in the + same format as real rows, so a field has to separate them. Every row in + `examples.json` is `illustrative` and **none of them is selectable**; a lookup + against the example file returns `no_match` with that as the stated reason. +- **`invalidated`** — an `Invalidation(reason, at, by)`, never a deletion. A + deleted row leaves no record that the claim was ever made, which is the first + thing a reviewer asks for. + +### `delta_is_floor` — the D1-11 guard + +BurstGPT has no prefix identity. When D1 replays it, `write_timed_trace` +**synthesizes** prefix blocks, unique per request, so lengths hold and *no prefix +sharing is invented that the source never had*. A prefix-cache knob measured on +such a trace therefore saw the **least** reuse the real traffic could have had. + +`PlaybookRow.delta_is_floor` is `True` when `provenance.prefix_synthesized` is +set **and** the knob set touches prefix caching. Such a row's delta is a lower +bound — usable as "at least this much", never quotable as the gain — and +`summary()` prints `[FLOOR: prefixes synthesized]`. `ex6` in the examples is that +case; `ex2` is the control (same synthesized trace, a knob that does not depend +on reuse, so not a floor). + +### Replay conditions in provenance + +`replay_chunk_hash_size` is a field because it is deliverable 1's finding with +the worst failure mode: at vLLM's default of 16 against Mooncake's 512-token +blocks, every prompt is 32× short **while every count in the result still reads +correctly**. A row that does not record the block size cannot be checked for it. + +### Where a row comes from — `row_from_runs` + +A `BenchRun` (deliverable 1, seam 3) is **one arm**. A row is the **difference +between two**, so the two are joined here and nowhere else: + +```python +from gitm.playbook import row_from_runs + +row = row_from_runs( + "prefix-cache-mooncake-h100", baseline_runs, treatment_runs, + model="Qwen/Qwen3.6-35B-A3B-FP8", model_revision="95a723d0", + gpu_sku="NVIDIA H100 80GB", env=EnvCapture(engine="vllm", engine_version="0.28.0"), + knobs={"enable_prefix_caching": True}, +) +``` + +Provenance is lifted off the runs, never retyped by the caller: the trace +checksum, drop counts, regime label and replay conditions come from the arms that +actually ran, which is the only way the row's checksum and the run's checksum +cannot drift apart. + +**It refuses, rather than producing a row, when:** + +| refusal | why | +|---|---| +| the arms did not run the same trace bytes or label to the same regime | a delta across two workloads measures the workloads | +| any run is not `promotable` (did not reconcile, or had failures) | below that bar there is nothing to take a difference of | +| the arms have unequal repeat counts | D2 interleaves A/B/A/B; unequal means the interleave broke | +| `repeats < 2` | delegated to `MeasuredDelta` — a single run has no variance | +| either arm is missing `output_throughput`, `p99_ttft_ms` or `p99_itl_ms` | D2's criterion 3 is unenforceable without both sides | + +**What it deliberately does not do.** `throughput_pct` is a percentage of +`output_throughput` (`THROUGHPUT_METRIC`, stated once so two readers cannot mean +different numbers), computed on **medians, never means**. `throughput_ci95_pct` +and `latency_blowout` stay `None`: D2 owns the variance rule and the blowout +predicate, and inventing either here is the mistake `AxisTolerance` refuses to +make with a distance threshold. And "same config minus exactly one knob" is D2's, +enforced by diffing two config-capture records that do not exist yet (**R1**) — +until they do, the caller asserts it and the row carries a note saying so, which +disappears on its own the moment `config_capture` is real. + +--- + +## 2. Regime distance + +### The metric + +Numeric axes are compared as **log2 ratios**, because what matters for token +counts and rates is the *factor*, not the difference: + +``` +|log2(a / b)| + +1,024 vs 2,048 tokens -> 1.00 (a 2x change) +1,024 vs 1,536 tokens -> 0.58 +1,024 vs 1,024 tokens -> 0.00 +``` + +Scale-free by construction: 100 vs 200 and 10,000 vs 20,000 are the same +distance, which is the property the axis needs. Symmetric. Two exact zeros are +equal; **one** zero is `inf` — "no output tokens at all" is not a small version +of "some output tokens", and infinity is what makes the combination say so +without a special case. + +**Burstiness** uses a shifted ratio, `|log2((1+a)/(1+b))|`. A perfectly paced +trace has `D = 0` and a bare ratio would make it incomparable to everything +including another paced trace. The shift anchors the axis on the Poisson +reference: + +``` +flat (0.00) vs poisson (1.00) -> 1.00 +burstgpt (1.01) vs mooncake (6.74) -> 1.94 +moderate (5.00) vs mooncake (6.74) -> 0.37 +``` + +### Combination: L-infinity + +``` +regime_distance = max(input_p50, input_p95, output_p50, output_p95, + io_ratio, burstiness) +``` + +Not a mean, not a Euclidean norm. **A row is as far away as its worst axis.** The +case this exists for, asserted in `check_linf_is_the_worst_axis`: a candidate +identical on five axes and 8× off on `input_p95`. The mean calls that a 0.5 +mismatch and would apply the row; L-inf calls it 3.0 and does not. That is a +long-context workload against a short-context row. + +For reference, the two real traces deliverable 1 measured are **L-inf 4.929 +apart, limited by `input_p95`** (49,904 vs 1,638). If those two collapsed to a +small distance, the axes would be decoration. + +### `rate_rps` is deliberately not an axis + +It exists on `Regime` and is **not** in `DEFAULT_AXES`. Adding it because it is +there would be exactly the mistake this module is written to avoid. + +| | | +|---|---| +| **include it if** | knob outcomes are shown to depend materially on offered load *after* concurrency and burstiness are accounted for | +| **omit it if** | the selected knobs are insensitive to rate once those two are fixed | +| **either way** | the decision is recorded with the experiment that settled it | + +The decision is material, not cosmetic: two regimes identical except for an 8× +difference in offered rate are distance **0.0** by default and **3.0** with the +axis on (`check_rate_is_not_in_the_default_axes`). Turning it on is one field: + +```python +MatchPolicy(name="with-rate", axes=(*DEFAULT_AXES, "rate_rps"), tolerances=…) +``` + +--- + +## 3. The exact-match gates + +Checked **before** any distance is computed, and each returns a stated reason +rather than a large number. + +| gate | rule | why not a distance | +|---|---|---| +| `model`, `model_revision` | exact | "nearly the same weights" is not a thing | +| `gpu_sku` | exact | ditto for silicon | +| `env.engine`, `env.engine_version` | exact | scheduler rewrites ship in point releases; a version bump is the most common way a knob's effect changes with nothing in the workload changing | +| `source_kind` | exact | a `scoreboard` row can never satisfy a production query, even when every numeric axis is identical (asserted) | +| `concurrency` | exact, `match_concurrency=True` | open-loop and a capped-concurrency run are different experiments | +| knob key set | exact | a lookup asks about a *specific* knob | + +Loosening any of them is an edit to a named field on `MatchPolicy`, not an +accident inside a comparison. `EnvCapture` is the one model with `extra="allow"`, +so a capture record from a newer engine round-trips without being truncated — +but `compatible_with` reads the **named** fields only, so an unknown extra key +can never change a match decision. + +When the shared config-capture schema lands (**R1**), `EnvCapture` is deleted and +Adit's types are **imported verbatim**. No translation layer: two schemas that +translate into each other are two schemas that drift. Everything waiting on this +is marked `pending-adit` and is grep-able. + +--- + +## 4. Lookup, precedence, and the handoff + +```python +result = lookup(playbook, query_identity, UNCALIBRATED_POLICY) +if result.route_to_discovery: + ... # conservative discovery +else: + apply(result.row.identity.knobs) +``` + +| status | meaning | returns a row? | +|---|---|---| +| `exact_regime` | distance 0.0 on every axis | ✅ | +| `near_regime` | inside every calibrated tolerance | ✅ | +| `uncalibrated` | candidates exist; some nonzero axis has no calibrated tolerance | ❌ → discovery | +| `no_match` | nothing passed the gates, or the nearest exceeds its tolerance | ❌ → discovery | + +`route_to_discovery` is `True` for everything that is not a returned row. A +status that is sometimes a row and sometimes a suggestion is how a wrong row gets +applied inside a 72-hour window. + +A miss carries what it rejected and why — `rejected: {row_id -> reason}` plus the +ranked `candidates` with their per-axis distances. Discovery starts warm, and a +human reading a miss learns which axis was the problem, not just that there was +one. **Discovery mode itself is not designed here**; only the handoff. + +### Precedence + +1. **Distance** — nearest first. +2. **In envelope** — among equally near rows, a row measured inside the observed + envelope beats one sampled beyond it (`/xenv`). *This sits below distance, + where `todo.md` had it above*: a nearby extrapolated point was still genuinely + run, and preferring a 4×-away in-envelope row over it answers the wrong + question. +3. **Recency** — most recently verified. +4. **Smallest claim** — conservative. A wrong row applied in the live window + costs more than a missed opportunity. + +Each of the four is asserted separately. + +--- + +## 5. Calibration — what would remove `uncalibrated` + +`AxisTolerance` **refuses a number without the experiment that produced it**: + +```python +AxisTolerance(max_distance=1.0) # ValidationError +AxisTolerance(max_distance=1.0, calibration="prereg E4, 2026-09-14: " + "sign flip at 1.4 on input_p95") # ok +``` + +That is the enforcement behind "the threshold is currently unknown". A +placeholder cannot quietly become a production constant. + +**Why `PLAYBOOK_MATCH_MAX_DISTANCE = 1.0` was not shipped.** Under log2 it means +"accept up to a 2× mismatch on every axis at once". That may well be safe for +`output_p50`. It is not obviously safe for long-context `input_p95`, for +prefix-cache reuse, or for a queue-sensitive scheduling policy — and nothing +measured says which. A number that reads as derived when it was picked is the +kind of thing that gets defended in front of a customer and then collapses. + +### The procedure + +1. Run the same knob across nearby regimes, varying **one** axis at a time. +2. Find where the effect changes sign, or where D2's latency-percentile criterion + flips from pass to fail. +3. Set that axis's tolerance strictly inside the distance at which it flipped, + and cite the run in `calibration`. +4. The L-inf limit is then **the strictest relevant per-axis tolerance, by + construction** — there is no separate global number to choose. + +### What blocks it today + +- No run against a live endpoint **with a real knob** has happened. Seam 3 is + closed and `row_from_runs` (§ 6) builds a row from two arms, but calibration + needs the *same knob measured across nearby regimes*, which needs a GPU. +- D2 does not exist, so "the effect flipped" has no agreed predicate. + +Until then the cost of `uncalibrated` is a discovery run, which is the cheap +failure. `MatchPolicy.uncalibrated_axes` reports the current state; today it is +all six. + +--- + +## 6. Worked examples + +`benchmarks/playbook/examples.json` — 6 rows, **0 selectable**, regenerated by +`python benchmarks/playbook/make_examples.py` from the repo root, with `gitm` +importable (`pip install -e .`, or `PYTHONPATH=.` — a bare script path does not +put the root on `sys.path` the way `python -m` does). The **regimes are real**, measured +off deliverable 1's sha256-pinned fixtures. The **deltas are invented** and every +row says so. + +| row | what it demonstrates | +|---|---| +| `ex1-prefix-cache-mooncake` | the ordinary case: long-input, prefix-sharing production traffic | +| `ex2-max-num-seqs-burstgpt` | the counter-example — same model and GPU, no prefix identity in the source. Synthesized prefixes but **not** a floor, because the knob does not depend on reuse | +| `ex3-chunked-prefill-qwen` | the row D3 would produce; stays illustrative until Phase B runs (needs one 80 GB card) | +| `ex4-retired-engine-bump` | invalidated by an engine bump — **kept, with a reason** | +| `ex5-scoreboard-not-production` | the biggest claimed delta in the file, gated out of every production lookup **by equality, not by distance** | +| `ex6-prefix-cache-on-a-synthesized-trace` | `delta_is_floor` — a prefix-cache knob on a source with no prefix identity | + +--- + +## 7. Not in scope + +- **Discovery mode.** The handoff is defined; the mode is not designed here. +- **The apply runtime** (Seojun's) and **env capture** (Adit's). +- **The promotion rule** (D2). D4 says what a row *is*; D2 says when one may be + created. `MeasuredDelta.latency_blowout` is the stored field where D2's + criterion-3 verdict lands, so the promotion gate and the live revert trigger + read the same value instead of each re-deriving it. +- **A calibrated threshold.** See §5. diff --git a/gitm/playbook/__init__.py b/gitm/playbook/__init__.py new file mode 100644 index 0000000..b9f9e11 --- /dev/null +++ b/gitm/playbook/__init__.py @@ -0,0 +1,82 @@ +"""Playbook schema — the contract between detection and apply. + +Deliverable 4 of the validation infrastructure. A playbook row says *this knob, +on this model revision and this GPU, under this workload regime, moved these +numbers by this much, and here is everything needed to check it.* + + from gitm.playbook import Playbook, lookup, UNCALIBRATED_POLICY + + result = lookup(playbook, query_identity, UNCALIBRATED_POLICY) + if result.route_to_discovery: + ... # conservative discovery; the handoff is defined, not the mode + else: + apply(result.row.identity.knobs) + +Matching is a **split**: exact equality on the categorical fields (model, +revision, GPU, environment, source_kind, concurrency), log2-ratio distance +combined with L-infinity on the numeric regime axes. The distance threshold is +**not calibrated yet** and the shipped policy says so — see +:mod:`gitm.playbook.match`. + +CPU-only. ``python -m gitm.playbook --selftest`` is the check. +""" + +from gitm.playbook.match import ( + AXIS_METRICS, + DEFAULT_AXES, + RATE_AXIS_DECISION, + UNCALIBRATED_POLICY, + AxisTolerance, + Candidate, + MatchPolicy, + MatchResult, + MatchStatus, + RegimeDistance, + dispersion_distance, + log2_ratio, + lookup, + regime_distance, +) +from gitm.playbook.schema import ( + PENDING_ADIT, + SCHEMA, + THROUGHPUT_METRIC, + EnvCapture, + Evidence, + Invalidation, + MeasuredDelta, + Playbook, + PlaybookRow, + Provenance, + RowIdentity, + row_from_runs, +) + +__all__ = [ + "AXIS_METRICS", + "DEFAULT_AXES", + "PENDING_ADIT", + "RATE_AXIS_DECISION", + "SCHEMA", + "THROUGHPUT_METRIC", + "UNCALIBRATED_POLICY", + "AxisTolerance", + "Candidate", + "EnvCapture", + "Evidence", + "Invalidation", + "MatchPolicy", + "MatchResult", + "MatchStatus", + "MeasuredDelta", + "Playbook", + "PlaybookRow", + "Provenance", + "RegimeDistance", + "RowIdentity", + "dispersion_distance", + "log2_ratio", + "lookup", + "regime_distance", + "row_from_runs", +] diff --git a/gitm/playbook/__main__.py b/gitm/playbook/__main__.py new file mode 100644 index 0000000..839b866 --- /dev/null +++ b/gitm/playbook/__main__.py @@ -0,0 +1,89 @@ +"""CLI for the playbook: inspect a file, measure a distance, run a lookup. + + python -m gitm.playbook --selftest + python -m gitm.playbook --show benchmarks/playbook/examples.json + python -m gitm.playbook --distance benchmarks/playbook/examples.json ex1-... ex2-... + python -m gitm.playbook --lookup benchmarks/playbook/examples.json ex2-... + +``--lookup`` takes a row id and asks the playbook what it would select *for that +row's own workload*, which is the honest way to demo a lookup without a live +server: the query is a real regime, and the answer is whatever the shipped policy +says. On the example file the answer is always "nothing" — every row is +illustrative — and that is the demonstration. + +All CPU-only. Nothing here applies a knob to anything. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +from gitm._banner import add_banner_argument, show_banner +from gitm.playbook._selftest import run_all +from gitm.playbook.match import UNCALIBRATED_POLICY, lookup, regime_distance +from gitm.playbook.schema import Playbook + + +def _load(path: str) -> Playbook: + return Playbook.model_validate(json.loads(Path(path).read_text(encoding="utf-8"))) + + +def _find(book: Playbook, row_id: str): + for row in book.rows: + if row.row_id == row_id: + return row + raise SystemExit(f"no row {row_id!r}; have: {', '.join(r.row_id for r in book.rows)}") + + +def main(argv: list[str] | None = None) -> int: + if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding="utf-8", errors="replace") + + p = argparse.ArgumentParser(prog="python -m gitm.playbook") + add_banner_argument(p) + p.add_argument("--selftest", action="store_true", help="run every check and exit") + p.add_argument("--show", metavar="PLAYBOOK") + p.add_argument("--distance", nargs=3, metavar=("PLAYBOOK", "ROW_A", "ROW_B")) + p.add_argument("--lookup", nargs=2, metavar=("PLAYBOOK", "ROW_ID")) + a = p.parse_args(argv) + show_banner(suppressed=a.no_banner) + + if a.selftest: + return run_all() + + if a.show: + book = _load(a.show) + print(f"{a.show}: {len(book.rows)} rows, {len(book.selectable())} selectable") + for row in book.rows: + print(f" {row.summary()}") + for note in row.notes: + print(f" note: {note}") + return 0 + + if a.distance: + path, a_id, b_id = a.distance + book = _load(path) + d = regime_distance( + _find(book, a_id).identity.regime, _find(book, b_id).identity.regime, UNCALIBRATED_POLICY + ) + print(f"{a_id} vs {b_id}") + print(f" {d.render()}") + return 0 + + if a.lookup: + path, row_id = a.lookup + book = _load(path) + result = lookup(book, _find(book, row_id).identity, UNCALIBRATED_POLICY) + print(result.render()) + print(f"\nroute_to_discovery: {result.route_to_discovery}") + return 0 + + p.print_help() + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/gitm/playbook/_selftest.py b/gitm/playbook/_selftest.py new file mode 100644 index 0000000..b888568 --- /dev/null +++ b/gitm/playbook/_selftest.py @@ -0,0 +1,568 @@ +"""The check that fails if the playbook schema or its match semantics break. + +One runnable thing, ``python -m gitm.playbook --selftest``, and the same +functions are the pytest cases in ``tests/test_playbook.py`` — the check a reader +is told about and the check CI runs are the *same* check. + +The regime coordinates below are the **real** ones measured off deliverable 1's +committed fixtures, so the distances asserted here describe the gap between two +real production traces rather than between two made-up ones. The *deltas* are +invented and every shipped example row says so. +""" + +from __future__ import annotations + +import json +import math +import os +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from gitm.playbook.match import ( + DEFAULT_AXES, + UNCALIBRATED_POLICY, + AxisTolerance, + MatchPolicy, + MatchStatus, + dispersion_distance, + log2_ratio, + lookup, + regime_distance, +) +from gitm.playbook.schema import ( + EnvCapture, + Evidence, + Invalidation, + MeasuredDelta, + Playbook, + PlaybookRow, + Provenance, + RowIdentity, +) +from gitm.traffic.regime import Regime, SourceKind + +#: The shipped worked examples. Data, not package content — same rule as the +#: traffic fixtures. ``$GITM_PLAYBOOK_EXAMPLES`` overrides for an installed +#: checkout. +EXAMPLES = Path( + os.environ.get( + "GITM_PLAYBOOK_EXAMPLES", + Path(__file__).resolve().parents[2] / "benchmarks" / "playbook" / "examples.json", + ) +) + +EXAMPLE_ROWS = 6 + +# --- real regime coordinates, measured off the D1 fixtures ------------------- +MOONCAKE = Regime( + source_kind=SourceKind.PRODUCTION, trace="mooncake", requests=400, + rate_rps=2.8368794326241136, io_ratio=39.09367234191124, + input_p50=9075, input_p95=49904, output_p50=370, output_p95=662, + burstiness=6.738120567375886, +) +BURSTGPT = Regime( + source_kind=SourceKind.PRODUCTION, trace="burstgpt", requests=383, + rate_rps=0.010276637419839545, io_ratio=1.8821556431490254, + input_p50=353, input_p95=1638, output_p50=238, output_p95=841, + burstiness=1.0106110910396904, +) + +#: How far apart the two real traces are, on the default axes. Pinned because it +#: is the sanity check on the whole metric: if these two collapsed to a small +#: distance the axes would not be separating anything. +BURSTGPT_VS_MOONCAKE_LINF = 4.929 # limited by input_p95 (49904 vs 1638, ~30x) + +H100 = "NVIDIA H100 80GB" +MODEL = "Qwen/Qwen3.6-35B-A3B-FP8" +REV = "95a723d0" +ENV = EnvCapture(engine="vllm", engine_version="0.11.0") +T0 = datetime(2026, 9, 2, tzinfo=timezone.utc) + + +def _identity(regime: Regime, **kw) -> RowIdentity: + base = dict( + model=MODEL, model_revision=REV, gpu_sku=H100, env=ENV, + regime=regime, knobs={"enable_prefix_caching": True}, + ) + base.update(kw) + return RowIdentity(**base) + + +def _row(row_id: str, regime: Regime, *, tput: float = 10.0, verified: datetime | None = T0, + **kw) -> PlaybookRow: + """A *measured* row — the selftest needs selectable rows, which the shipped + examples deliberately are not.""" + identity_kw = kw.pop("identity_kw", {}) + return PlaybookRow( + row_id=row_id, + identity=_identity(regime, **identity_kw), + delta=MeasuredDelta(throughput_pct=tput, ttft_p99_ms=-5.0, itl_p99_ms=0.2, repeats=5), + provenance=Provenance( + trace_source=regime.trace, trace_sha256="0" * 64, + regime_label=regime.label(), verified_at=verified, + ), + evidence=Evidence.MEASURED, + **kw, + ) + + +def _calibrated(max_distance: float, axes: tuple[str, ...] = DEFAULT_AXES) -> MatchPolicy: + return MatchPolicy( + name="test-calibrated", + axes=axes, + tolerances={ + a: AxisTolerance(max_distance=max_distance, calibration="selftest fixture, not a real run") + for a in axes + }, + ) + + +# --- the checks -------------------------------------------------------------- + +def check_log2_ratio_is_scale_free() -> None: + """The three worked numbers from the design, plus the properties behind them.""" + assert log2_ratio(1024, 2048) == 1.0 # a 2x change is exactly 1 + assert abs(log2_ratio(1024, 1536) - 0.5849625007211562) < 1e-12 + assert log2_ratio(1024, 1024) == 0.0 + + # scale-free: the same factor at any magnitude is the same distance. This is + # why a raw difference would be the wrong metric — 100 vs 200 tokens and + # 10,000 vs 20,000 are the same *kind* of mismatch. + assert log2_ratio(100, 200) == log2_ratio(10_000, 20_000) == 1.0 + # symmetric + assert log2_ratio(2048, 1024) == log2_ratio(1024, 2048) + # zero is not a small number: one side zero is incomparable, both zero is equal + assert math.isinf(log2_ratio(0, 256)) + assert log2_ratio(0, 0) == 0.0 + + +def check_dispersion_distance_handles_flat_traces() -> None: + """``D = 0`` is a real trace (perfectly paced), not an incomparable one.""" + assert dispersion_distance(0.0, 0.0) == 0.0 + assert dispersion_distance(0.0, 1.0) == 1.0 # flat vs Poisson, one unit + # the two real traces are far apart on this axis, correctly + d = dispersion_distance(BURSTGPT.burstiness, MOONCAKE.burstiness) + assert 1.9 < d < 2.0, d + # and a nearby dispersion is near + assert dispersion_distance(5.0, MOONCAKE.burstiness) < 0.4 + # a raw log2 ratio would have blown up here; the shift is what prevents it + assert math.isinf(log2_ratio(0.0, 1.0)) + + +def check_linf_is_the_worst_axis() -> None: + """A mean would let four close axes hide one that breaks the row.""" + d = regime_distance(BURSTGPT, MOONCAKE, UNCALIBRATED_POLICY) + assert d.limiting_axis == "input_p95", d.per_axis + assert abs(d.linf - BURSTGPT_VS_MOONCAKE_LINF) < 0.001, d.render() + assert d.linf == max(d.per_axis.values()) + + # The case L-inf exists for: identical on five axes, 8x off on the sixth. + # The mean calls that a 0.5 mismatch; L-inf calls it a 3.0 mismatch. The + # workload is a long-context one against a short-context row, and the mean + # is the reading that would apply the row. + long_ctx = MOONCAKE.model_copy(update={"input_p95": MOONCAKE.input_p95 * 8}) + one_axis = regime_distance(MOONCAKE, long_ctx, UNCALIBRATED_POLICY) + assert one_axis.limiting_axis == "input_p95" and one_axis.linf == 3.0 + assert sum(one_axis.per_axis.values()) / len(one_axis.per_axis) == 0.5 + + # a regime against itself is exactly zero on every axis + same = regime_distance(MOONCAKE, MOONCAKE, UNCALIBRATED_POLICY) + assert same.exact and set(same.per_axis.values()) == {0.0} + + +def check_rate_is_not_in_the_default_axes() -> None: + """``rate_rps`` is excluded by decision, and the decision is material.""" + assert "rate_rps" not in DEFAULT_AXES + assert "rate_rps" in regime_distance( + BURSTGPT, MOONCAKE, MatchPolicy(name="with-rate", axes=(*DEFAULT_AXES, "rate_rps")) + ).per_axis + + # Two regimes identical except for offered rate: distance 0 by default, and + # a large distance the moment rate is included. If including it were a + # no-op the decision would not need making. + slower = MOONCAKE.model_copy(update={"rate_rps": MOONCAKE.rate_rps / 8}) + assert regime_distance(MOONCAKE, slower, UNCALIBRATED_POLICY).exact + with_rate = MatchPolicy(name="with-rate", axes=(*DEFAULT_AXES, "rate_rps")) + assert abs(regime_distance(MOONCAKE, slower, with_rate).linf - 3.0) < 1e-9 + + +def check_tolerance_requires_calibration() -> None: + """A threshold cannot enter a policy without the experiment that set it.""" + with pytest.raises(ValidationError, match="max_distance without calibration"): + AxisTolerance(max_distance=1.0) + with pytest.raises(ValidationError): + AxisTolerance(max_distance=-0.5, calibration="whatever") + ok = AxisTolerance(max_distance=1.0, calibration="prereg E4: sign flip at 1.4 on input_p95") + assert ok.calibrated + # and the shipped policy has none of them + assert UNCALIBRATED_POLICY.uncalibrated_axes == DEFAULT_AXES + assert not any(UNCALIBRATED_POLICY.tolerance(a).calibrated for a in DEFAULT_AXES) + + +def check_exact_gates_reject_before_distance() -> None: + """Categorical mismatches are rejections, never a large distance.""" + book = Playbook(rows=[_row("r-mooncake", MOONCAKE)]) + for label, kw in [ + ("gpu_sku", {"gpu_sku": "NVIDIA A100 80GB"}), + ("model", {"model": "meta-llama/Llama-4-70B"}), + ("model_revision", {"model_revision": "deadbeef"}), + ("engine_version", {"env": EnvCapture(engine="vllm", engine_version="0.12.0")}), + ("knob set", {"knobs": {"max_num_seqs": 64}}), + ]: + res = lookup(book, _identity(MOONCAKE, **kw), UNCALIBRATED_POLICY) + assert res.status is MatchStatus.NO_MATCH, (label, res.render()) + assert res.route_to_discovery + assert label.split("_")[0] in res.rejected["r-mooncake"], (label, res.rejected) + + # source_kind: a scoreboard row is gated out of a production query even + # though every numeric axis is identical. + board = MOONCAKE.model_copy(update={"source_kind": SourceKind.SCOREBOARD}) + assert regime_distance(MOONCAKE, board, UNCALIBRATED_POLICY).exact + res = lookup(Playbook(rows=[_row("r-board", board)]), _identity(MOONCAKE), UNCALIBRATED_POLICY) + assert res.status is MatchStatus.NO_MATCH + assert "scoreboard" in res.rejected["r-board"] + + # concurrency: exact by policy, and turning the gate off is a named edit + capped = MOONCAKE.model_copy(update={"concurrency": 64}) + book_capped = Playbook(rows=[_row("r-capped", capped)]) + assert lookup(book_capped, _identity(MOONCAKE), UNCALIBRATED_POLICY).status is MatchStatus.NO_MATCH + loose = MatchPolicy(name="ignore-concurrency", match_concurrency=False) + assert lookup(book_capped, _identity(MOONCAKE), loose).status is MatchStatus.EXACT_REGIME + + +def check_uncalibrated_policy_routes_to_discovery() -> None: + """The headline: a near miss is not a match until an axis is calibrated.""" + near = MOONCAKE.model_copy(update={"output_p50": 400}) # ~0.11 away, one axis + res = lookup(Playbook(rows=[_row("r-near", near)]), _identity(MOONCAKE), UNCALIBRATED_POLICY) + assert res.status is MatchStatus.UNCALIBRATED + assert res.route_to_discovery and res.row is None + assert "not yet calibrated" in res.reason + assert "output_p50" in res.reason # the reason names the axis, not just a number + # the candidate is still reported — a miss has to be actionable + assert [c.row.row_id for c in res.candidates] == ["r-near"] + assert res.candidates[0].distance.limiting_axis == "output_p50" + + +def check_exact_regime_matches_without_any_calibration() -> None: + """Distance 0 needs no threshold, so the schema is usable on day one.""" + res = lookup(Playbook(rows=[_row("r-exact", MOONCAKE)]), _identity(MOONCAKE), UNCALIBRATED_POLICY) + assert res.status is MatchStatus.EXACT_REGIME + assert not res.route_to_discovery + assert res.row is not None and res.row.row_id == "r-exact" + assert res.distance is not None and res.distance.exact + + +def check_calibrated_policy_matches_inside_and_rejects_outside() -> None: + """Once an axis is calibrated, near means near — and far still means no.""" + near = MOONCAKE.model_copy(update={"output_p50": 400}) # log2(400/370) = 0.112 + far = MOONCAKE.model_copy(update={"output_p50": 1480}) # log2(1480/370) = 2.0 + book_near = Playbook(rows=[_row("r-near", near)]) + book_far = Playbook(rows=[_row("r-far", far)]) + policy = _calibrated(0.5) + + ok = lookup(book_near, _identity(MOONCAKE), policy) + assert ok.status is MatchStatus.NEAR_REGIME and ok.row is not None + assert not ok.route_to_discovery + + no = lookup(book_far, _identity(MOONCAKE), policy) + assert no.status is MatchStatus.NO_MATCH and no.route_to_discovery + assert "output_p50" in no.reason and "2.0" in no.reason + + # the two real traces are far outside any plausible tolerance + assert lookup( + Playbook(rows=[_row("r-burstgpt", BURSTGPT)]), _identity(MOONCAKE), _calibrated(2.0) + ).status is MatchStatus.NO_MATCH + + +def check_precedence_exact_then_recent_then_conservative() -> None: + """Three tie-breaks, in order, each demonstrated on its own.""" + near = MOONCAKE.model_copy(update={"output_p50": 400}) + policy = _calibrated(0.5) + + # 1. exact regime beats nearest regime, even with a smaller claimed delta + res = lookup( + Playbook(rows=[_row("near-big", near, tput=99.0), _row("exact-small", MOONCAKE, tput=1.0)]), + _identity(MOONCAKE), policy, + ) + assert res.row is not None and res.row.row_id == "exact-small" + assert res.status is MatchStatus.EXACT_REGIME + + # 2. among equals, the most recently verified wins + res = lookup( + Playbook(rows=[ + _row("stale", MOONCAKE, verified=T0 - timedelta(days=30)), + _row("fresh", MOONCAKE, verified=T0), + ]), + _identity(MOONCAKE), policy, + ) + assert res.row is not None and res.row.row_id == "fresh" + + # 3. equally close and equally fresh: the smaller claim wins, because a + # wrong row inside the live window costs more than a missed one + res = lookup( + Playbook(rows=[_row("bold", MOONCAKE, tput=40.0), _row("modest", MOONCAKE, tput=3.0)]), + _identity(MOONCAKE), policy, + ) + assert res.row is not None and res.row.row_id == "modest" + + +def check_synthesized_prefixes_make_a_prefix_cache_delta_a_floor() -> None: + """D1-11: BurstGPT has no prefix identity, so its reuse number is a bound. + + D1 synthesizes unique blocks per request, which invents no sharing the source + never had. A prefix-cache knob measured there saw the *least* reuse possible, + and the row has to say so or the floor gets quoted as the gain. + """ + row = _row("r-synth", BURSTGPT) + assert not row.delta_is_floor # prefix_synthesized defaults False + synth = row.model_copy(update={ + "provenance": row.provenance.model_copy(update={"prefix_synthesized": True}) + }) + assert synth.delta_is_floor + assert "FLOOR" in synth.summary() + + # a non-prefix knob on the same synthesized trace is unaffected — the + # synthesis only distorts what depends on reuse + other = synth.model_copy(update={ + "identity": _identity(BURSTGPT, knobs={"max_num_seqs": 64}) + }) + assert not other.delta_is_floor + + # the marker is a substring, so a renamed flag still trips it + renamed = synth.model_copy(update={ + "identity": _identity(BURSTGPT, knobs={"enable-prefix-caching-v2": True}) + }) + assert renamed.delta_is_floor + + +def check_extrapolated_rows_lose_ties_but_not_distance() -> None: + """``/xenv`` is a tie-break below distance, and the ordering is asserted.""" + xenv = MOONCAKE.model_copy(update={"in_envelope": False}) + policy = _calibrated(1.0) + + # equal distance: the in-envelope row wins + res = lookup( + Playbook(rows=[_row("extrapolated", xenv), _row("measured", MOONCAKE)]), + _identity(MOONCAKE), policy, + ) + assert res.row is not None and res.row.row_id == "measured" + + # nearer-but-extrapolated still beats a far in-envelope row: distance is the + # workload question and comes first (this is where we depart from todo.md) + far = MOONCAKE.model_copy(update={"output_p50": 640}) # 0.79 away + res = lookup( + Playbook(rows=[_row("near-xenv", xenv), _row("far-inenv", far)]), + _identity(MOONCAKE), policy, + ) + assert res.row is not None and res.row.row_id == "near-xenv" + + +def check_replay_conditions_are_carried_into_provenance() -> None: + """The 512-vs-16 finding is a field, so a wrong-block-size row is checkable.""" + row = _row("r-mooncake", MOONCAKE) + assert row.provenance.replay_chunk_hash_size is None # unrecorded, not assumed + pinned = row.model_copy(update={ + "provenance": row.provenance.model_copy( + update={"replay_chunk_hash_size": 512, "replay_self_timed": True} + ) + }) + assert pinned.provenance.replay_chunk_hash_size == 512 + # and D2's stored verdict has somewhere to go without being recomputed + assert pinned.delta.latency_blowout is None + assert pinned.delta.model_copy(update={"latency_blowout": True}).latency_blowout is True + + +def check_invalidated_row_is_kept_and_never_selected() -> None: + """Retirement is a field with a reason, not a deletion.""" + dead = _row("r-dead", MOONCAKE) + dead = dead.model_copy(update={ + "invalidated": Invalidation(reason="vLLM 0.11 -> 0.12 scheduler rewrite", at=T0) + }) + book = Playbook(rows=[dead]) + assert len(book.rows) == 1 and book.selectable() == [] + res = lookup(book, _identity(MOONCAKE), UNCALIBRATED_POLICY) + assert res.status is MatchStatus.NO_MATCH + assert "scheduler rewrite" in res.rejected["r-dead"] # the reason survives the miss + + +def check_a_row_cannot_exist_without_its_evidence() -> None: + """The type refuses rows that could not have come through the promotion rule.""" + with pytest.raises(ValidationError, match="knobs is required"): + _identity(MOONCAKE, knobs={}) + with pytest.raises(ValidationError, match="single run has no variance"): + MeasuredDelta(throughput_pct=8.0, ttft_p99_ms=-1.0, itl_p99_ms=0.0, repeats=1) + with pytest.raises(ValidationError): # trace_sha256 is required + Provenance(trace_source="mooncake", trace_sha256="", regime_label="x") + with pytest.raises(ValidationError): # throughput alone cannot be promoted (D2 criterion 3) + MeasuredDelta(throughput_pct=8.0, repeats=5) + + +def check_examples_ship_nothing_selectable() -> None: + """Every worked example is labelled, and none of them can be applied.""" + book = Playbook.model_validate(json.loads(EXAMPLES.read_text(encoding="utf-8"))) + assert len(book.rows) == EXAMPLE_ROWS + assert all(r.evidence is Evidence.ILLUSTRATIVE for r in book.rows) + assert book.selectable() == [] # an example in a live file is the failure mode + + # round-trips byte-identically: the file is the schema, not a rendering of it + assert json.loads(json.dumps(book.model_dump(mode="json"))) == json.loads( + EXAMPLES.read_text(encoding="utf-8") + ) + + # the illustrative rows carry real regimes and real trace checksums + by_id = {r.row_id: r for r in book.rows} + assert by_id["ex1-prefix-cache-mooncake"].identity.regime.input_p50 == MOONCAKE.input_p50 + assert by_id["ex2-max-num-seqs-burstgpt"].identity.regime.input_p50 == BURSTGPT.input_p50 + assert all(len(r.provenance.trace_sha256) == 64 for r in book.rows) + # the biggest claimed delta in the file is the scoreboard row, gated by equality + assert max(book.rows, key=lambda r: r.delta.throughput_pct).row_id == ( + "ex5-scoreboard-not-production" + ) + # exactly one example is a floor, and it is the prefix-cache knob on the + # source with no prefix identity — both halves of D1-11 have an example + floors = [r.row_id for r in book.rows if r.delta_is_floor] + assert floors == ["ex6-prefix-cache-on-a-synthesized-trace"], floors + assert by_id["ex2-max-num-seqs-burstgpt"].provenance.prefix_synthesized + assert not by_id["ex2-max-num-seqs-burstgpt"].delta_is_floor # synthesized != floor + assert all(r.provenance.replay_chunk_hash_size == 512 for r in book.rows) + + # and lookup refuses all of them even for a perfectly matching query + res = lookup(book, _identity(MOONCAKE), UNCALIBRATED_POLICY) + assert res.status is MatchStatus.NO_MATCH + assert all("never selectable" in w or "invalidated" in w for w in res.rejected.values()) + + +def check_regime_is_imported_not_redeclared() -> None: + """One coordinate system. A second Regime here would drift within a week.""" + import gitm.traffic.regime as d1 + + assert RowIdentity.model_fields["regime"].annotation is d1.Regime + # and every distance axis is a real field on it, by name + from gitm.playbook.match import AXIS_METRICS + + assert set(AXIS_METRICS) <= set(d1.Regime.model_fields) + + +# --- the last mile: seam 3's records become a row ---------------------------- +#: Deltas injected into the treatment arm's copy of the real result JSON. The +#: *runs* are real (the committed 0.28.0 result); the difference between the arms +#: is constructed, because a second real arm needs a knob and a GPU. What is +#: under test is the arithmetic and the refusals, not the numbers. +ARM_TPUT_GAIN = 1.10 +ARM_TTFT_DELTA_MS = -40.0 + + +def _runs(**bump): + """N ``BenchRun`` records off the committed real result, optionally bumped.""" + from gitm.traffic._selftest import _real_run + from gitm.traffic.results import join_result + + result, plan, reg, _ = _real_run() + return result, plan, reg, (lambda n=2, **kw: [join_result({**result, **kw}, plan, reg) + for _ in range(n)]) + + +def check_two_bench_runs_become_a_row() -> None: + """The last mile. Seam 3 makes one arm; a row is the difference between two. + + Before this existed, nothing could populate a row end to end no matter how + complete the join was — which is why every shipped example is illustrative. + """ + from gitm.playbook.schema import PENDING_ADIT, row_from_runs + + result, _, reg, mk = _runs() + base = mk(2) + treat = mk(2, + output_throughput=result["output_throughput"] * ARM_TPUT_GAIN, + p99_ttft_ms=result["p99_ttft_ms"] + ARM_TTFT_DELTA_MS) + + row = row_from_runs( + "r-from-real-runs", base, treat, + model=MODEL, model_revision=REV, gpu_sku=H100, env=ENV, + knobs={"enable_prefix_caching": True}, + ) + + # the arithmetic + assert row.delta.repeats == 2 + assert math.isclose(row.delta.throughput_pct, (ARM_TPUT_GAIN - 1) * 100, rel_tol=1e-9) + assert math.isclose(row.delta.ttft_p99_ms, ARM_TTFT_DELTA_MS, rel_tol=1e-9) + + # provenance came off the runs, not off the caller — the whole point + assert row.provenance.trace_sha256 == base[0].source.sha256 + assert len(row.provenance.trace_sha256) == 64 + assert row.provenance.regime_label == reg.label() + assert row.provenance.replay_chunk_hash_size == 512 + assert row.provenance.replay_self_timed is True + assert row.identity.regime == reg + + # measured, and therefore selectable — the first row in the repo that is + assert row.evidence is Evidence.MEASURED and row.selectable + + # ...and it still says what it does not know + assert row.provenance.config_capture == PENDING_ADIT + assert row.provenance.promotion_rule.startswith(PENDING_ADIT) + assert any("R1" in n for n in row.notes), row.notes + assert row.delta.latency_blowout is None # D2 owns the predicate + assert row.delta.throughput_ci95_pct is None # D2 owns the variance rule + + +def check_a_row_refuses_arms_that_are_not_one_experiment() -> None: + """Every way a pair of runs is not a delta. Each is a wrong row prevented.""" + from gitm.playbook.schema import row_from_runs + + _, _, reg, mk = _runs() + ident = dict(model=MODEL, model_revision=REV, gpu_sku=H100, env=ENV, + knobs={"enable_prefix_caching": True}) + + def refused(base, treat, needle): + with pytest.raises((ValueError, ValidationError)) as e: + row_from_runs("r-bad", base, treat, **ident) + assert needle in str(e.value), str(e.value) + + refused(mk(2), mk(1), "unequal arms") # the interleave broke + refused(mk(1), mk(1), "repeats=1") # delegated to MeasuredDelta + refused(mk(2), mk(2, failed=1), "promotable") # a run with failures + refused(mk(2), mk(2, completed=39), "promotable") # did not replay the trace + refused([], mk(2), "both arms are required") + + # two different workloads: same result JSON, a regime that labels differently + from gitm.traffic.results import join_result + result, plan, _, _ = _runs() + other = reg.model_copy(update={"input_p50": 100, "input_p95": 200}) + assert other.label() != reg.label() + refused(mk(2), [join_result(result, plan, other) for _ in range(2)], + "did not run the same workload") + + +CHECKS = ( + check_log2_ratio_is_scale_free, + check_dispersion_distance_handles_flat_traces, + check_linf_is_the_worst_axis, + check_rate_is_not_in_the_default_axes, + check_tolerance_requires_calibration, + check_exact_gates_reject_before_distance, + check_uncalibrated_policy_routes_to_discovery, + check_exact_regime_matches_without_any_calibration, + check_calibrated_policy_matches_inside_and_rejects_outside, + check_precedence_exact_then_recent_then_conservative, + check_extrapolated_rows_lose_ties_but_not_distance, + check_synthesized_prefixes_make_a_prefix_cache_delta_a_floor, + check_replay_conditions_are_carried_into_provenance, + check_invalidated_row_is_kept_and_never_selected, + check_a_row_cannot_exist_without_its_evidence, + check_examples_ship_nothing_selectable, + check_regime_is_imported_not_redeclared, + check_two_bench_runs_become_a_row, + check_a_row_refuses_arms_that_are_not_one_experiment, +) + + +def run_all() -> int: + for fn in CHECKS: + fn() + print(f" ok {fn.__name__}") + print(f"selftest ok -- {len(CHECKS)} checks, 2 real regimes, 0 calibrated axes") + return 0 diff --git a/gitm/playbook/match.py b/gitm/playbook/match.py new file mode 100644 index 0000000..2e81044 --- /dev/null +++ b/gitm/playbook/match.py @@ -0,0 +1,452 @@ +"""Match semantics — which past result applies to the workload in front of you. + +The whole deliverable turns on this module, because "close enough" is where a +wrong row gets applied. The design is a **split**, not a single score: + + exact equality model + revision, GPU SKU, environment, source_kind, + concurrency policy -> a gate: pass or reject + distance the numeric regime axes -> a ranking among survivors + +Categorical fields are gated because "nearly an H100" is not a thing, and a +distance that mixed a GPU mismatch into the same number as a token-count +mismatch would let a large enough workload similarity outvote running on +different silicon. + +**The numeric axes are compared as log2 ratios**, which is the natural metric for +token counts and rates — what matters is the *factor*, not the difference: + + 1,024 vs 2,048 tokens -> 1.0 (a 2x change) + 1,024 vs 1,536 tokens -> 0.58 + 1,024 vs 1,024 tokens -> 0.0 + +and combined with **L-infinity** (the max across axes), not a mean or a Euclidean +norm. A mean lets a close match on four axes hide a 4x mismatch on the fifth, and +the fifth is the one that breaks the row. L-inf says: a row is as far away as its +worst axis. + +**The threshold is not known, and this module says so rather than picking one.** +A number like ``max_distance = 1.0`` reads as calibrated and is not: under log2 +it means "accept up to a 2x mismatch on every axis at once", which may well be +safe for output p50 and is certainly not safe for long-context input p95, for +prefix-cache reuse, or for a queue-sensitive scheduling policy. So +:class:`AxisTolerance` **refuses to hold a number without the experiment that +produced it**, the shipped :data:`UNCALIBRATED_POLICY` has no numbers at all, and +a lookup that would need one returns :attr:`MatchStatus.UNCALIBRATED` and routes +to conservative discovery. + +Calibrating an axis, which is what removes that status: + +1. Run the same knob across nearby regimes, varying **one** axis at a time. +2. Find where the effect changes sign, or where the latency percentile criterion + from deliverable 2 flips from pass to fail. +3. Set that axis's tolerance strictly inside the distance at which it flipped. +4. The L-inf limit is then the strictest relevant per-axis tolerance, by + construction — there is no separate global number to choose. + +Until step 1 has data, the honest state is "not yet calibrated", and the cost of +that state is a discovery run, which is the cheap failure. +""" + +from __future__ import annotations + +import math +from datetime import datetime, timezone +from enum import Enum + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from gitm.playbook.schema import Playbook, PlaybookRow, RowIdentity +from gitm.traffic.regime import Regime + +#: Distance returned when an axis cannot be compared at all — one side is zero, +#: the other is not. Not 0.0 and not "skip": an incomparable axis is a reason to +#: reject a row, and infinity is what makes L-inf say so without a special case. +INCOMPARABLE = math.inf + + +def log2_ratio(a: float, b: float) -> float: + """``|log2(a / b)|`` — the distance between two positive magnitudes. + + Symmetric, zero at equality, 1.0 at a factor of two, and scale-free: 100 vs + 200 tokens is the same distance as 10,000 vs 20,000, which is the property a + token-count axis needs. Two exact zeros are equal; one zero is + :data:`INCOMPARABLE`, because "no output tokens at all" is not a small + version of "some output tokens". + """ + if a == b: + return 0.0 + if a <= 0 or b <= 0: + return INCOMPARABLE + return abs(math.log2(a / b)) + + +def dispersion_distance(a: float, b: float) -> float: + """Distance between two index-of-dispersion values, ``|log2((1+a)/(1+b))|``. + + Burstiness cannot use :func:`log2_ratio` directly: a perfectly paced trace + has ``D = 0`` and a bare ratio would make it incomparable to everything, + including another paced trace. The ``1 +`` shift anchors the axis so that the + Poisson reference ``D = 1`` sits one unit from flat ``D = 0``, and the two + real traces deliverable 1 measured land where intuition puts them: + + flat (0.0) vs poisson (1.0) -> 1.00 + burstgpt (1.01) vs mooncake (6.74) -> 1.95 (far apart, correctly) + moderate (5.0) vs mooncake (6.74) -> 0.37 (near, correctly) + """ + return log2_ratio(1.0 + a, 1.0 + b) + + +#: The numeric axes, and how each is compared. Names match +#: :class:`~gitm.traffic.regime.Regime` fields exactly — an axis that cannot be +#: read off a Regime by name is an axis that will silently stop being computed. +AXIS_METRICS = { + "input_p50": log2_ratio, + "input_p95": log2_ratio, + "output_p50": log2_ratio, + "output_p95": log2_ratio, + "io_ratio": log2_ratio, + "burstiness": dispersion_distance, + "rate_rps": log2_ratio, +} + +#: Axes on by default. **``rate_rps`` is deliberately absent.** It exists on +#: ``Regime`` and adding it to the distance because it is there would be exactly +#: the mistake this module is written to avoid: offered rate is largely captured +#: by burstiness plus the concurrency gate, and a knob that is insensitive to +#: rate would then be rejected for a workload it fits. Turning it on is a +#: decision with evidence behind it — see :data:`RATE_AXIS_DECISION`. +DEFAULT_AXES = ("input_p50", "input_p95", "output_p50", "output_p95", "io_ratio", "burstiness") + +RATE_AXIS_DECISION = """\ +rate_rps is not in the distance by default. Include it when knob outcomes are +shown to depend materially on offered load *after* concurrency and burstiness are +accounted for; omit it when they are not. Either way the decision is recorded +with the experiment that settled it, not inferred from the field existing.""" + + +class AxisTolerance(BaseModel): + """How far this axis may differ, and the experiment that says so. + + ``max_distance=None`` means **uncalibrated** — the axis is compared and + reported, but no nonzero distance on it can be accepted automatically. + + A number without ``calibration`` is rejected at construction. That is the + enforcement behind "the threshold is currently unknown": the only way to get + a tolerance into a policy is to name the run that produced it, so a + placeholder can never quietly become a production constant. + """ + + model_config = ConfigDict(extra="forbid") + + max_distance: float | None = None + #: What measured it. Free text pointing at a run or a spec section, e.g. + #: "prereg_rank1 E4, 2026-09-14: sign flip at 1.4 on input_p95". + calibration: str | None = None + + @model_validator(mode="after") + def _a_number_needs_a_reason(self) -> AxisTolerance: + if self.max_distance is not None and not self.calibration: + raise ValueError( + "max_distance without calibration: a tolerance is a measured " + "quantity, not a default. Run the knob across nearby regimes, " + "find where the effect flips, and cite it here." + ) + if self.max_distance is not None and self.max_distance < 0: + raise ValueError("max_distance must be >= 0") + return self + + @property + def calibrated(self) -> bool: + return self.max_distance is not None + + +class MatchPolicy(BaseModel): + """Which axes count, how far each may stray, and what is gated exactly.""" + + model_config = ConfigDict(extra="forbid") + + name: str + axes: tuple[str, ...] = DEFAULT_AXES + tolerances: dict[str, AxisTolerance] = Field(default_factory=dict) + #: Exact-match gates. Each is a policy choice, listed so that loosening one + #: is an edit to a named field rather than an accident in a comparison. + match_source_kind: bool = True + match_concurrency: bool = True + match_env: bool = True + + @model_validator(mode="after") + def _axes_are_real(self) -> MatchPolicy: + unknown = [a for a in self.axes if a not in AXIS_METRICS] + if unknown: + raise ValueError(f"unknown regime axes {unknown}; known: {sorted(AXIS_METRICS)}") + stray = [a for a in self.tolerances if a not in self.axes] + if stray: + raise ValueError(f"tolerance set for axes not in the policy: {stray}") + return self + + def tolerance(self, axis: str) -> AxisTolerance: + return self.tolerances.get(axis, AxisTolerance()) + + @property + def uncalibrated_axes(self) -> tuple[str, ...]: + return tuple(a for a in self.axes if not self.tolerance(a).calibrated) + + +#: The policy that ships. Every axis uncalibrated, so the only automatic match is +#: an **exact** regime match and everything else routes to discovery. This is not +#: a placeholder to be edited in passing — replacing it means calibrating the +#: axes, and :class:`AxisTolerance` will not let a number in without the run. +UNCALIBRATED_POLICY = MatchPolicy(name="uncalibrated") + + +class MatchStatus(str, Enum): + """The outcome of a lookup. Four states, and three of them are not a row.""" + + EXACT_REGIME = "exact_regime" # distance 0 on every axis; safe to apply + NEAR_REGIME = "near_regime" # within calibrated tolerances + UNCALIBRATED = "uncalibrated" # candidates exist, but no axis is calibrated + NO_MATCH = "no_match" # nothing passed the exact gates + + +class RegimeDistance(BaseModel): + """Per-axis distances and the L-inf that summarizes them. + + Both halves are kept. The L-inf is what a threshold compares against; the + per-axis dict is what tells a human *which* axis put the row out of range, + which is the only actionable half when a lookup misses. + """ + + model_config = ConfigDict(extra="forbid") + + per_axis: dict[str, float] + linf: float + limiting_axis: str | None + + @property + def exact(self) -> bool: + return self.linf == 0.0 + + def render(self) -> str: + parts = " ".join( + f"{a}={'inf' if math.isinf(d) else format(d, '.3f')}" + for a, d in sorted(self.per_axis.items()) + ) + lim = f" (limited by {self.limiting_axis})" if self.limiting_axis else "" + return f"L-inf {self.linf:.3f}{lim} [{parts}]" + + +def regime_distance(a: Regime, b: Regime, policy: MatchPolicy = UNCALIBRATED_POLICY) -> RegimeDistance: + """Distance between two regimes on the policy's axes, combined with L-inf.""" + per_axis = {axis: AXIS_METRICS[axis](getattr(a, axis), getattr(b, axis)) for axis in policy.axes} + if not per_axis: + return RegimeDistance(per_axis={}, linf=0.0, limiting_axis=None) + limiting = max(per_axis, key=lambda k: per_axis[k]) + # An exact match has no limiting axis. Naming one would read as "this is the + # axis that nearly failed", which is the opposite of what a 0.0 means. + return RegimeDistance( + per_axis=per_axis, + linf=per_axis[limiting], + limiting_axis=limiting if per_axis[limiting] > 0 else None, + ) + + +def _gate(query: RowIdentity, row: RowIdentity, policy: MatchPolicy) -> str: + """Exact-match gate. Returns ``""`` when the row passes, else why it did not.""" + if query.model != row.model: + return f"model {row.model!r} != {query.model!r}" + if query.model_revision != row.model_revision: + return f"model_revision {row.model_revision} != {query.model_revision}" + if query.gpu_sku != row.gpu_sku: + return f"gpu_sku {row.gpu_sku!r} != {query.gpu_sku!r}" + if policy.match_env: + ok, why = query.env.compatible_with(row.env) + if not ok: + return why + if policy.match_source_kind and query.regime.source_kind is not row.regime.source_kind: + return ( + f"source_kind {row.regime.source_kind.value} != {query.regime.source_kind.value}" + " — a scoreboard result is not evidence about production traffic" + ) + if policy.match_concurrency and query.regime.concurrency != row.regime.concurrency: + return f"concurrency {row.regime.concurrency} != {query.regime.concurrency}" + if query.knobs.keys() != row.knobs.keys(): + return f"knob set {sorted(row.knobs)} != {sorted(query.knobs)}" + return "" + + +class Candidate(BaseModel): + """A row that passed the gates, with how far its regime is from the query.""" + + model_config = ConfigDict(extra="forbid") + + row: PlaybookRow + distance: RegimeDistance + + +class MatchResult(BaseModel): + """What a lookup found, and — when it found nothing — exactly why. + + ``rejected`` is not diagnostics-for-later. A miss that cannot say which gate + it failed sends someone to read the whole playbook by hand, and a miss is the + common case for a schema this young. + """ + + model_config = ConfigDict(extra="forbid") + + status: MatchStatus + row: PlaybookRow | None = None + distance: RegimeDistance | None = None + candidates: list[Candidate] = Field(default_factory=list) + rejected: dict[str, str] = Field(default_factory=dict) # row_id -> why + policy: str = UNCALIBRATED_POLICY.name + reason: str = "" + + @property + def route_to_discovery(self) -> bool: + """Whether the caller must fall back to conservative discovery mode. + + True for everything that is not a returned row. Deliverable 4 defines + this handoff and **not** discovery itself: what a caller needs from the + schema is an unambiguous "I have nothing for you", and a status that is + sometimes a row and sometimes a suggestion is how a wrong row gets + applied in a 72-hour window. + """ + return self.row is None + + def render(self) -> str: + head = f"{self.status.value} (policy: {self.policy})" + if self.row is not None and self.distance is not None: + return f"{head}\n {self.row.summary()}\n {self.distance.render()}" + lines = [head, f" {self.reason}"] if self.reason else [head] + for c in self.candidates: + lines.append(f" candidate {c.row.row_id}: {c.distance.render()}") + for row_id, why in sorted(self.rejected.items()): + lines.append(f" rejected {row_id}: {why}") + return "\n".join(lines) + + +def _precedence(c: Candidate) -> tuple[float, int, float, float]: + """Sort key: nearest, then measured-in-envelope, then freshest, then smallest. + + Four terms, in this order and for these reasons: + + 1. **Distance.** The workload question comes first; everything else is a + tie-break among rows that answer it equally well. + 2. **In envelope.** A row measured at a point deliberately sampled *beyond* + any observed trace (D1 marks these ``/xenv``) is weaker evidence than one + measured inside it. This sits below distance rather than above it — + ``todo.md`` had it above — because a nearby extrapolated point was still + genuinely run, and preferring a 4x-away in-envelope row over it answers + the wrong question. + 3. **Recency.** Among equals, the most recently verified. + 4. **Smallest claim.** The conservative tie-break from the plan: prefer the + row claiming **less**, because a wrong row applied inside the live window + costs more than a missed opportunity. + """ + verified = c.row.provenance.verified_at + recency = -(verified or datetime.min.replace(tzinfo=timezone.utc)).timestamp() + extrapolated = 0 if c.row.identity.regime.in_envelope else 1 + return (c.distance.linf, extrapolated, recency, abs(c.row.delta.throughput_pct)) + + +def lookup( + playbook: Playbook, + query: RowIdentity, + policy: MatchPolicy = UNCALIBRATED_POLICY, +) -> MatchResult: + """Find the row that applies to ``query``, or say why none does. + + Order of operations, and each step can only ever *reject*: + + 1. **Gate** on the exact fields. Model, revision, GPU, environment, + source_kind, concurrency, and the knob set being asked about. + 2. **Measure** the regime distance for the survivors. + 3. **Decide.** Distance 0 on every axis is an exact regime match and is + returned. A nonzero distance needs a calibrated tolerance on every axis it + is nonzero along; without one the result is + :attr:`MatchStatus.UNCALIBRATED` and the caller goes to discovery. + 4. **Break ties** by :func:`_precedence`. + + Unselectable rows — invalidated, or worked examples — never reach step 2. + """ + result_rejected: dict[str, str] = {} + candidates: list[Candidate] = [] + + for row in playbook.rows: + if not row.selectable: + result_rejected[row.row_id] = ( + "invalidated: " + row.invalidated.reason + if row.invalidated + else f"evidence={row.evidence.value}; never selectable" + ) + continue + why = _gate(query, row.identity, policy) + if why: + result_rejected[row.row_id] = why + continue + candidates.append(Candidate(row=row, distance=regime_distance(query.regime, row.identity.regime, policy))) + + if not candidates: + return MatchResult( + status=MatchStatus.NO_MATCH, + rejected=result_rejected, + policy=policy.name, + reason="no row passed the exact-match gates", + ) + + candidates.sort(key=_precedence) + best = candidates[0] + + if best.distance.exact: + return MatchResult( + status=MatchStatus.EXACT_REGIME, + row=best.row, + distance=best.distance, + candidates=candidates, + rejected=result_rejected, + policy=policy.name, + ) + + # Nonzero distance: every axis it is nonzero along must have a calibrated + # tolerance, and must be inside it. An uncalibrated axis is not "probably + # fine" — it is an axis nobody has measured the knob across. + uncalibrated = [ + a for a, d in best.distance.per_axis.items() if d > 0 and not policy.tolerance(a).calibrated + ] + if uncalibrated: + return MatchResult( + status=MatchStatus.UNCALIBRATED, + candidates=candidates, + rejected=result_rejected, + policy=policy.name, + reason=( + f"nearest row {best.row.row_id} is {best.distance.linf:.3f} away, limited by " + f"{best.distance.limiting_axis}; no calibrated tolerance for {sorted(uncalibrated)}. " + "Automatic regime matching is not yet calibrated — routing to discovery." + ), + ) + + over = [ + (a, d) for a, d in best.distance.per_axis.items() if d > (policy.tolerance(a).max_distance or 0.0) + ] + if over: + axis, dist = max(over, key=lambda t: t[1]) + return MatchResult( + status=MatchStatus.NO_MATCH, + candidates=candidates, + rejected=result_rejected, + policy=policy.name, + reason=( + f"nearest row {best.row.row_id} exceeds its tolerance on {axis}: " + f"{dist:.3f} > {policy.tolerance(axis).max_distance}" + ), + ) + + return MatchResult( + status=MatchStatus.NEAR_REGIME, + row=best.row, + distance=best.distance, + candidates=candidates, + rejected=result_rejected, + policy=policy.name, + ) diff --git a/gitm/playbook/schema.py b/gitm/playbook/schema.py new file mode 100644 index 0000000..1acb21b --- /dev/null +++ b/gitm/playbook/schema.py @@ -0,0 +1,470 @@ +"""Playbook row contracts — what a promoted tuning result *is*. + +Deliverable 4. One row says: + + (model+revision, GPU SKU, workload regime, knob set, environment) + -> measured delta + the provenance to re-verify it from scratch + +The schema is the contract between Adit's detection and Seojun's apply runtime, +so it ships as **types**, not as a doc that two implementations read differently. + +Three rules the types enforce rather than describe: + +* **A row cannot exist without provenance.** :class:`Provenance` is required and + ``extra="forbid"``, and its trace fields are the ones deliverable 1 already + emits in :class:`~gitm.traffic.schema.TraceMeta`. A tuning claim without the + raw trace checksum, the drop counts and the repeat data is not defensible, and + the type is where that stops being a convention. +* **A row is retired by a field, never by a deletion.** :class:`Invalidation` + carries a reason. A row deleted from a file leaves no record that the claim was + ever made, which is exactly what a reviewer asks for. +* **Regime is imported, never re-declared.** :class:`~gitm.traffic.regime.Regime` + is deliverable 1's type. A second copy here would drift within a week, and the + distance metric in :mod:`gitm.playbook.match` would be measuring two different + coordinate systems. + +**R1, stated in the types:** the shared config-capture schema does not exist yet. +:class:`EnvCapture` below is the *named subset* this deliverable needs, marked +``pending-adit``. When Adit's types land they are **imported verbatim** and this +class is deleted — there is no translation layer, per the brief, because two +schemas that translate into each other are two schemas that drift. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from datetime import datetime +from enum import Enum +from statistics import median + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from gitm.traffic.regime import Regime +from gitm.traffic.results import BenchRun + +#: Schema identity, in the style of ``gitm.bench.manifest.SCHEMA``. Bump on any +#: field change that is not purely additive. +SCHEMA = "gitm.playbook.row/v1" + +#: Marker for every field that is waiting on the shared config-capture schema +#: (risk R1). Grep-able on purpose: when Adit's types land, this string is the +#: work list. +PENDING_ADIT = "pending-adit" + +#: Knob-name fragments that make a row's delta depend on prefix-cache reuse. +#: Substring match, deliberately: engines rename these flags between versions, +#: and a list of exact names would go stale silently while a substring stays +#: right for ``enable_prefix_caching``, ``prefix_caching``, and the next +#: spelling after that. +PREFIX_CACHE_KNOB_MARKERS = ("prefix_cach", "prefix-cach", "kv_reuse") + + +class Evidence(str, Enum): + """Whether a row's *delta* was measured or is an illustration. + + The worked examples ship in the same file format as real rows, so a field + has to separate them. Without it, an example row copied into a live playbook + is indistinguishable from a promoted one — and the whole point of the schema + is that a row's standing is readable from the row. + """ + + MEASURED = "measured" # produced by a real A/B under the promotion rule + ILLUSTRATIVE = "illustrative" # a worked example; never selectable + + +class EnvCapture(BaseModel): + """The environment fields a playbook row must pin. ``pending-adit`` (R1). + + Deliberately thin. These are the values that, if they differ between the run + that produced a row and the box about to apply it, make the row's number + meaningless. Everything else Adit's capture records is welcome and arrives by + *import*, not by being re-typed here. + + ``extra="allow"`` is the one place in this module that permits unknown keys: + a capture record from a newer engine must round-trip through a playbook file + without being silently truncated. Comparison uses :meth:`compatible_with`, + which reads the named fields only — an unknown extra key never changes a + match decision, it just survives the trip. + """ + + model_config = ConfigDict(extra="allow") + + schema_id: str = f"{PENDING_ADIT}/env-capture" + engine: str # e.g. "vllm" + engine_version: str # exact, e.g. "0.11.0" + driver_version: str | None = None + torch_version: str | None = None + cuda_version: str | None = None + + def compatible_with(self, other: EnvCapture) -> tuple[bool, str]: + """Exact on engine and engine version. Returns ``(ok, why_not)``. + + An engine version bump is the single most common way a knob's effect + changes without anything in the workload changing — scheduler rewrites + ship in point releases. So the default policy is **exact**, and anything + looser has to be written down as a policy rather than assumed by a + comparison that used ``startswith``. + """ + if self.engine != other.engine: + return False, f"engine {self.engine!r} != {other.engine!r}" + if self.engine_version != other.engine_version: + return False, f"engine_version {self.engine_version} != {other.engine_version}" + return True, "" + + +class RowIdentity(BaseModel): + """The lookup key. Everything here is matched, nothing here is a result. + + Split deliberately into fields matched **exactly** and one field matched by + **distance** (:attr:`regime`). The split is the design: model, GPU and + environment are categorical — "nearly an H100" is not a thing — while the + workload is continuous and live traffic never lands on a measured point. + :mod:`gitm.playbook.match` implements exactly that split and nothing else. + """ + + # ``protected_namespaces=()`` because the fields really are called ``model`` + # and ``model_revision`` — that is the vocabulary everyone else uses, and + # renaming them to dodge a pydantic warning would make the schema wrong in + # the one place it is read by hand. + model_config = ConfigDict(extra="forbid", protected_namespaces=()) + + model: str # e.g. "Qwen/Qwen3.6-35B-A3B-FP8" + model_revision: str # exact commit/revision, e.g. "95a723d0" + gpu_sku: str # e.g. "NVIDIA H100 80GB" + env: EnvCapture + regime: Regime # deliverable 1's type, imported + #: ``bool`` first in the union on purpose: pydantic coerces left to right, + #: and ``True`` arriving as ``1.0`` turns a boolean knob into a number in + #: every rendering of the row. + knobs: dict[str, bool | int | float | str] = Field(default_factory=dict) + + @model_validator(mode="after") + def _knobs_are_not_empty(self) -> RowIdentity: + if not self.knobs: + raise ValueError("a playbook row with no knob set says nothing; knobs is required") + return self + + def exact_key(self) -> tuple[str, str, str]: + """The part of the key compared by equality, for grouping and printing.""" + return (self.model, self.model_revision, self.gpu_sku) + + +class MeasuredDelta(BaseModel): + """What the knob did, relative to the baseline arm of the same experiment. + + Throughput **and** latency percentiles, always both. Deliverable 2's + criterion 3 is that a knob which gains throughput while blowing out latency + percentiles fails promotion — a schema that let a row carry throughput alone + would make that criterion unenforceable at the point it matters. + + Signs are stated once, here, because a sign error in a playbook is a knob + applied backwards: **throughput is percent, higher is better; the latency + fields are milliseconds of change, negative is better.** + """ + + model_config = ConfigDict(extra="forbid") + + throughput_pct: float # % change vs baseline; + is faster + ttft_p99_ms: float # ms change vs baseline; - is better + itl_p99_ms: float # ms change vs baseline; - is better + repeats: int = Field(ge=1) # per arm, interleaved A/B/A/B per D2 + #: Bootstrap 95 % CI on the median difference in throughput, per D2-1. The + #: promotion rule owns the predicate; the row carries the numbers it needs. + throughput_ci95_pct: tuple[float, float] | None = None + #: D2 criterion 3's verdict, **stored rather than recomputed**: did this knob + #: blow out a latency percentile? The promotion gate and the live-window + #: revert trigger must read the same value, and two callers each re-deriving + #: "blowout" from the raw percentiles is two predicates that drift. D2 owns + #: the predicate and writes the answer here; ``None`` means D2 has not run, + #: which is every row today because D2 does not exist yet. + latency_blowout: bool | None = None + + @model_validator(mode="after") + def _repeats_are_plural(self) -> MeasuredDelta: + if self.repeats < 2: + raise ValueError( + f"repeats={self.repeats}: a single run has no variance and cannot be " + "promoted under D2. Record it as evidence=illustrative if it is an example." + ) + return self + + +class Provenance(BaseModel): + """Everything needed to re-run this row from scratch and get it again. + + The trace fields are deliverable 1's :class:`~gitm.traffic.schema.TraceMeta` + verbatim in meaning: ``trace_sha256`` pins the raw bytes, ``trace_drops`` + says what was rejected getting to them. A row whose trace no longer hashes to + ``trace_sha256`` is not the same experiment, and the field is what lets + anyone find that out. + """ + + model_config = ConfigDict(extra="forbid") + + trace_source: str # adapter name, e.g. "mooncake" + trace_sha256: str # of the raw file, via gitm.bench.manifest + trace_drops: dict[str, int] = Field(default_factory=dict) + regime_label: str # Regime.label() as it stood when measured + #: Where the per-repeat raw numbers live. D2 requires them; a summarized + #: delta with the repeats thrown away cannot be re-analyzed under a different + #: variance rule, and D2-1's thresholds are explicitly not settled yet. + repeat_raw_data: list[str] = Field(default_factory=list) + promotion_rule: str = f"{PENDING_ADIT}/promotion-rule" # D2 doc + version + config_capture: str = PENDING_ADIT # R1 + verified_at: datetime | None = None # last time the row was re-measured + + #: The conditions the replay actually ran under, from D1's ``ReplayPlan``. + #: ``replay_chunk_hash_size`` is here because it is the deliverable-1 finding + #: with the worst failure mode: at vLLM's default of 16 against Mooncake's + #: 512-token blocks every prompt is 32x short while every count in the result + #: still reads correctly. A row that does not record it cannot be checked. + replay_chunk_hash_size: int | None = None + replay_self_timed: bool | None = None + #: True when D1 *synthesized* prefix blocks because the source had none — + #: BurstGPT. See :attr:`PlaybookRow.delta_is_floor`: a prefix-cache knob + #: measured on a synthesized-prefix trace saw no sharing the source never + #: had, so its delta is a lower bound and may not be quoted as a gain. + prefix_synthesized: bool = False + + @model_validator(mode="after") + def _label_matches_nothing_yet(self) -> Provenance: + if not self.trace_sha256: + raise ValueError("trace_sha256 is required — a row that cannot be traced to bytes") + return self + + @classmethod + def from_bench_run(cls, run: BenchRun, **overrides) -> Provenance: + """Build the trace half of a row's provenance from a seam-3 record. + + Every field here already exists on :class:`~gitm.traffic.results.BenchRun` + because that is what seam 3 was for; copying them by hand at each call + site is how the checksum and the label that ran drift apart. What is + *not* here is the config-capture half — ``config_capture`` comes across + still marked ``pending-adit`` (R1), which is the truthful value. + + ``overrides`` covers the fields no single run knows: ``repeat_raw_data``, + ``promotion_rule``, ``verified_at``. + """ + return cls( + trace_source=run.source.source, + trace_sha256=run.source.sha256, + trace_drops=dict(run.source.drops), + regime_label=run.regime_label, + replay_chunk_hash_size=run.chunk_hash_size, + replay_self_timed=run.self_timed, + prefix_synthesized=run.prefix_synthesized, + config_capture=run.config_capture, + **overrides, + ) + + +class Invalidation(BaseModel): + """Why a row stopped being usable. A field, never a deletion.""" + + model_config = ConfigDict(extra="forbid") + + reason: str # free text, e.g. "vLLM 0.11 -> 0.12 scheduler rewrite" + at: datetime + by: str | None = None + + +class PlaybookRow(BaseModel): + """One promoted tuning result. + + Rows enter **only** through deliverable 2's promotion rule. That is not + enforceable in a type — a type cannot see how a number was produced — so what + the type does instead is refuse to hold a row that *could not* have come + through it: no knobs, no provenance, a single repeat, or a delta missing its + latency percentiles all fail construction. + """ + + model_config = ConfigDict(extra="forbid") + + schema_id: str = SCHEMA + row_id: str + identity: RowIdentity + delta: MeasuredDelta + provenance: Provenance + evidence: Evidence = Evidence.MEASURED + invalidated: Invalidation | None = None + notes: list[str] = Field(default_factory=list) + + @property + def delta_is_floor(self) -> bool: + """True when this row's delta is a **lower bound**, not a measurement. + + D1 synthesizes prefix blocks for a source that has none (BurstGPT), with + every request's ids unique so that lengths hold and **no prefix sharing + is invented**. A prefix-cache knob measured on such a trace therefore saw + the *least* reuse the real traffic could have had. The number is a floor: + useful as "at least this much", never quotable as the gain. + + A property rather than a stored flag because it is a function of two + fields already on the row, and a stored copy is a third place to get it + wrong. + """ + if not self.provenance.prefix_synthesized: + return False + return any( + m in k.lower() for k in self.identity.knobs for m in PREFIX_CACHE_KNOB_MARKERS + ) + + @property + def selectable(self) -> bool: + """Whether a lookup may return this row at all. + + Two ways to be unselectable, and both are states rather than absences: + the row was invalidated, or it is a worked example that was never + measured. An example row in a live file is the failure mode this guards. + """ + return self.invalidated is None and self.evidence is Evidence.MEASURED + + def summary(self) -> str: + k = ", ".join(f"{k}={v}" for k, v in sorted(self.identity.knobs.items())) + mark = "" if self.selectable else f" [{'invalid' if self.invalidated else 'example'}]" + mark += " [FLOOR: prefixes synthesized]" if self.delta_is_floor else "" + return ( + f"{self.row_id}: {k} on {self.identity.gpu_sku} / " + f"{self.identity.regime.label()} -> " + f"tput {self.delta.throughput_pct:+.1f}%, " + f"ttft p99 {self.delta.ttft_p99_ms:+.1f}ms, " + f"itl p99 {self.delta.itl_p99_ms:+.1f}ms{mark}" + ) + + +class Playbook(BaseModel): + """A file of rows. Thin on purpose — the matching lives in ``match.py``.""" + + model_config = ConfigDict(extra="forbid") + + schema_id: str = SCHEMA + rows: list[PlaybookRow] = Field(default_factory=list) + + def selectable(self) -> list[PlaybookRow]: + return [r for r in self.rows if r.selectable] + + +#: The throughput a row's ``throughput_pct`` is a percentage of. ``bench serve`` +#: reports three (request, output-token, total-token); a row that does not say +#: which one it means is a row two readers compare differently. Output tokens per +#: second is the serving number, and it is the one deliverable 2's criterion 3 +#: pairs against the latency percentiles. +THROUGHPUT_METRIC = "output_throughput" + +#: Metrics both arms must carry before a delta can be computed. TTFT and ITL are +#: here because deliverable 2's criterion 3 is unenforceable without them — the +#: schema already refuses a throughput-only :class:`MeasuredDelta`, and this +#: refuses to *build* one rather than failing later with a pydantic error. +REQUIRED_METRICS = (THROUGHPUT_METRIC, "p99_ttft_ms", "p99_itl_ms") + + +def row_from_runs( + row_id: str, + baseline: Sequence[BenchRun], + treatment: Sequence[BenchRun], + *, + model: str, + model_revision: str, + gpu_sku: str, + env: EnvCapture, + knobs: dict[str, bool | int | float | str], + repeat_raw_data: Sequence[str] = (), + verified_at: datetime | None = None, + notes: Sequence[str] = (), +) -> PlaybookRow: + """Build one row from the two arms of a real experiment. + + This is the last mile deliverable 4 was missing: seam 3 turns a + ``bench serve`` result into a :class:`~gitm.traffic.results.BenchRun`, and + this turns a *pair* of arms into a row. A ``BenchRun`` is one arm; a row is a + **difference between two**, which is why nothing before this could populate a + row end to end no matter how complete the join was. + + What it enforces, because these are deliverable 4's business: + + * **Both arms ran the same workload** — same trace bytes, same regime label. + A delta across two different traces measures the traces. + * **Every run is** :attr:`~gitm.traffic.results.BenchRun.promotable` — it + reconciles against its trace and had no failures. Below that bar there is + nothing to take a difference of. + * **Equal repeat counts**, because deliverable 2 interleaves A/B/A/B. Unequal + arms mean the interleave broke, and the shorter arm is the one that was + cut short. + * **Medians, never means**, per deliverable 2's criterion 2. + + What it does **not** enforce, and why: + + * **"Same config minus exactly one knob."** That is deliverable 2's, and it + is enforced by diffing two config-capture records — which do not exist + (R1). Until they do, the caller asserts it and the row says so in a note + that disappears on its own the moment ``config_capture`` is real. + * ``throughput_ci95_pct`` and ``latency_blowout`` stay ``None``. Deliverable + 2 owns the variance rule and the blowout predicate; inventing either here + would be the same mistake :class:`~gitm.playbook.match.AxisTolerance` + refuses to make with a distance threshold. + """ + arms = list(baseline) + list(treatment) + if not baseline or not treatment: + raise ValueError("both arms are required — a row is a difference between two") + if len(baseline) != len(treatment): + raise ValueError( + f"unequal arms: {len(baseline)} baseline vs {len(treatment)} treatment. " + "D2 interleaves A/B/A/B, so unequal counts mean the interleave broke." + ) + + unpromotable = [r.summary() for r in arms if not r.promotable] + if unpromotable: + raise ValueError( + "a row cannot be built from runs that are not promotable " + "(reconciled, no failures): " + "; ".join(unpromotable) + ) + + traces = {r.source.sha256 for r in arms} + labels = {r.regime_label for r in arms} + if len(traces) != 1 or len(labels) != 1: + raise ValueError( + f"the arms did not run the same workload: {len(traces)} distinct trace " + f"checksum(s), regime labels {sorted(labels)} — a delta across two " + "workloads measures the workloads" + ) + + for r in arms: + missing = [m for m in REQUIRED_METRICS if r.metrics.get(m) is None] + if missing: + raise ValueError(f"{r.summary()}: result is missing {missing}") + + def med(runs: Sequence[BenchRun], metric: str) -> float: + return median(float(r.metrics[metric]) for r in runs) + + base_tput = med(baseline, THROUGHPUT_METRIC) + if base_tput == 0: + raise ValueError(f"baseline {THROUGHPUT_METRIC} is 0 — no percentage to take") + + notes = list(notes) + prov = Provenance.from_bench_run( + treatment[0], + repeat_raw_data=list(repeat_raw_data), + verified_at=verified_at, + ) + if prov.config_capture == PENDING_ADIT: + notes.append( + "R1: no config-capture record exists, so 'same config minus exactly one knob' " + "is asserted by the caller here, not diffed. D2 owns that check." + ) + + return PlaybookRow( + row_id=row_id, + identity=RowIdentity( + model=model, model_revision=model_revision, gpu_sku=gpu_sku, env=env, + regime=treatment[0].regime, knobs=knobs, + ), + delta=MeasuredDelta( + throughput_pct=(med(treatment, THROUGHPUT_METRIC) / base_tput - 1.0) * 100.0, + ttft_p99_ms=med(treatment, "p99_ttft_ms") - med(baseline, "p99_ttft_ms"), + itl_p99_ms=med(treatment, "p99_itl_ms") - med(baseline, "p99_itl_ms"), + repeats=len(treatment), + ), + provenance=prov, + evidence=Evidence.MEASURED, + notes=notes, + ) diff --git a/tests/test_playbook.py b/tests/test_playbook.py new file mode 100644 index 0000000..77d94d4 --- /dev/null +++ b/tests/test_playbook.py @@ -0,0 +1,29 @@ +"""Playbook schema — the pytest face of ``python -m gitm.playbook --selftest``. + +Same pattern as ``tests/test_traffic.py``: the assertions live in +:mod:`gitm.playbook._selftest` and are called from both places, so the runnable +check a reader is told about and the check CI runs cannot drift apart. +""" + +from __future__ import annotations + +import pytest + +from gitm.playbook import _selftest + + +@pytest.mark.parametrize("check", _selftest.CHECKS, ids=lambda f: f.__name__) +def test_playbook_check(check) -> None: + if not _selftest.EXAMPLES.exists(): + pytest.skip(f"examples not present at {_selftest.EXAMPLES}") + check() + + +def test_every_check_is_registered() -> None: + """A check that exists but is never run is worse than no check.""" + defined = { + name + for name in dir(_selftest) + if name.startswith("check_") and callable(getattr(_selftest, name)) + } + assert {f.__name__ for f in _selftest.CHECKS} == defined From 2f20f617291a96434a52b6721d58277c737d34da Mon Sep 17 00:00:00 2001 From: Rahul-Ganesan Date: Thu, 3 Sep 2026 19:18:22 -0400 Subject: [PATCH 3/8] docs: Parasail POC readme The entry point for the two packages above. Leads with what is measured and what is not, because the distinction is load-bearing: the adapters, replay fidelity, firing at a live endpoint and the result join are measured; no playbook row is true, the regime-distance threshold is uncalibrated, and pacing under a saturating server needs a GPU. Nothing here claims a performance win. It is the machinery that would make one checkable. --- PARASAIL_POC.md | 276 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 276 insertions(+) create mode 100644 PARASAIL_POC.md diff --git a/PARASAIL_POC.md b/PARASAIL_POC.md new file mode 100644 index 0000000..37549d9 --- /dev/null +++ b/PARASAIL_POC.md @@ -0,0 +1,276 @@ +# Parasail POC — traffic replay and the playbook schema + +Two pieces of the validation infrastructure, built to answer one question: **when +we say a knob is worth applying, what makes that statement checkable by someone +who was not in the room?** + +- **`gitm/traffic/`** — replay real production traces through a real serving + endpoint, and tie the result back to the workload that produced it. +- **`gitm/playbook/`** — the row that says *this knob, on this model and GPU, + under this traffic, moved these numbers by this much, and here is everything + needed to re-check it.* + +CPU-only except for firing a replay. No new dependencies. + +--- + +## Five-minute tour + +```bash +# 1. Every claim below, as one runnable check +python -m gitm.traffic --selftest # 24 checks, 3 real traces +python -m gitm.playbook --selftest # 19 checks, 2 real regimes + +# 2. What a real production trace actually contains +python -m gitm.traffic --describe mooncake \ + benchmarks/traffic_replay/fixtures/mooncake_slice.jsonl + +# 3. Replay it, and prove the replay preserved it +python -m gitm.traffic --replay mooncake \ + benchmarks/traffic_replay/fixtures/mooncake_slice.jsonl --out /tmp/mc.jsonl + +# 4. What a playbook row looks like, and why none of these is selectable +python -m gitm.playbook --show benchmarks/playbook/examples.json +python -m gitm.playbook --lookup benchmarks/playbook/examples.json ex1-prefix-cache-mooncake +``` + +Step 3 prints a validation table in which **every statistic is exactly 0.0**, and +the arrival-rate profiles of source and replay drawn one above the other. Step 4 +returns **nothing** and says why, per row. + +--- + +## What is measured, and what is not + +The distinction is load-bearing, so it is stated before anything else. + +| | state | +|---|---| +| The adapters, on real published bytes | ✅ measured | +| Replay fidelity, source vs the file vLLM reads | ✅ measured — every statistic exactly 0.0 | +| Firing at a live vLLM endpoint | ✅ **done**, against real `vllm bench serve` 0.28.0 | +| Joining a result back to its trace | ✅ measured, end to end, exit 0 | +| **Any playbook row being true** | ❌ **no.** Every shipped row is `illustrative` and its delta is invented | +| The regime-distance threshold | ❌ **uncalibrated, and the code says so at runtime** | +| Pacing under a *saturating* server | ❌ needs a GPU | + +**Nothing here claims a performance win.** It is the machinery that would make +such a claim checkable. + +--- + +## `gitm/traffic/` — traffic replay + +### Real traces, not synthetic load + +Two published production traces, pinned by sha256 through +`gitm.bench.manifest`. Measured by the adapters: + +| trace | rows | input p50 / p95 | dispersion `D` | regime label | +|---|---|---|---|---| +| BurstGPT_1 (Azure OpenAI) | 383/400 | 353 / 1,638 | 1.01 | `prod/io1/in256/out128/burst-poisson/copen` | +| BurstGPT_3 (8-column layout) | 399/400 | 309 / 1,497 | 1.01 | `prod/io2/in256/out64/burst-poisson/copen` | +| Mooncake (Kimi serving) | 400/400 | 9,075 / 49,904 | 6.74 | `prod/io32/in8k/out256/burst-hi/copen` | + +The two axes that matter separate cleanly — **D 6.74 vs 1.01, input p50 9,075 vs +353.** If the regime axes could not tell two real production traces apart they +would be decoration; `check_regime_axes_separate_the_traces` asserts they do. + +### Real data is the work, not the parsing + +**7 named drop reasons**, each firing exactly once on both a CSV and a JSONL +dirty fixture. The counts land in `TraceMeta`, and **a trace whose counts do not +reconcile cannot be constructed** — `rows_read == rows_emitted + dropped` is +enforced in `__post_init__`, not by convention. + +The zero-token rows in BurstGPT are real and are the majority defect: **744 of +9,382 `Conversation log` rows carry 0/0**, while all 19 `API log` rows are +well-formed. A caller's own filter counts separately from bad data and still +reconciles. + +### No custom load generator + +Replay goes through vLLM's native `bench serve --dataset-name timed_trace +--self-timed`, which schedules every request at its own timestamp. Fidelity is +checked against **the file vLLM will actually read**, not an in-memory copy: + +``` +trace validation [replay] mooncake(8090d6a38401) -> timed_trace(dfd989ad6d50) +check value threshold result +request_count 0 0 pass +arrival_ks 0 0.001 pass +input_len_ks 0 0.001 pass +output_len_ks 0 0.001 pass +rate_rps 0 1e-06 pass +burstiness 0 1e-06 pass +PASS — the pipeline preserves the trace +``` + +**Fired end to end against real vLLM 0.28.0**: 40/40 completed, paced to +**12.008 s against a 12.000 s trace span** — confirmed by two independent clocks, +vLLM's own and the receiving server's. + +### The silent failure this exists to catch + +vLLM's `--timed-trace-chunk-hash-size` defaults to **16**. Mooncake's cache +blocks are **512**. At the default, every prompt is **32× short** — while +`completed`, `duration`, throughput and every latency percentile still read +perfectly. + +The emitter refuses to write such a file, and the joiner catches one fired from a +plan built elsewhere. Confirmed on the real run: `total_input_tokens` came back +**506,280 against the trace's 506,280** — exact; at 16-token blocks it would have +been ~15,821. + +### The result JSON is not merely incomplete — two fields are wrong + +`bench serve --save-result` writes 34 keys. Under `--self-timed` vLLM still +records the CLI's `request_rate` (`"inf"` — a *string*) and `burstiness` (`1.0`), +against the trace's real 2.837 rps and D 6.74. **They are exactly the two axes a +playbook row keys on.** + +They are **dropped with a stated reason** rather than merged or renamed, with the +values kept visible. A new vLLM field cannot fall out silently either — +`unjoined_keys()` fails on any key neither kept nor deliberately dropped, and it +caught `rtfx` on its first run. + +--- + +## `gitm/playbook/` — the row, and when it may be used + +### Row identity is a split, not a lookup + +``` +(model+revision, GPU SKU, environment, workload regime, knob set) + -> measured delta + provenance +``` + +**Exact equality** on model+revision, GPU SKU, engine+version, `source_kind` and +concurrency. **Distance** on the numeric regime axes, because live traffic never +lands on a measured point. Each gate is a named field on `MatchPolicy`, so +loosening one is an edit visible in a diff rather than an accident inside a +comparison. + +`source_kind` is asserted the hard way — **with every numeric axis identical**, so +only the gate can be doing the work. A scoreboard result never satisfies a +production query. + +### Distance is a log2 ratio, combined with L-infinity + +`|log2(a/b)|` — scale-free, symmetric, zero at equality, and `inf` when one side +is zero, because "no output tokens" is not a small version of "some". + +**L-infinity, not a mean.** A row is as far away as its worst axis. Asserted on +the case it exists for: a candidate identical on five axes and 8× off on +`input_p95` reads **0.5 as a mean** — close enough to apply — and **3.0 as +L-inf**. For scale, the two real traces are **4.929 apart, limited by +`input_p95`** (49,904 vs 1,638). + +### The threshold is uncalibrated, and the type refuses to pretend otherwise + +```python +AxisTolerance(max_distance=1.0) # ValidationError +AxisTolerance(max_distance=1.0, calibration="prereg E4: sign flip at 1.4") # ok +``` + +No tolerance ships. Consequences, all asserted: + +- an **exact** regime match still returns a row, so the schema is usable today; +- any nonzero distance returns `UNCALIBRATED` **naming the limiting axis**, and + routes to discovery; +- a placeholder cannot quietly become a production constant. + +Calibration needs the same knob measured across nearby regimes, which needs a +GPU. **The cost of the open state is a discovery run; the cost of an invented +threshold is a knob applied to traffic nobody measured it on.** + +### A row cannot exist without its evidence + +Refused at construction: no knobs, `repeats < 2`, a delta missing its latency +percentiles, an empty `trace_sha256`. `MeasuredDelta` carries throughput **and** +TTFT/ITL percentiles — the schema cannot express a throughput-only row. + +Invalidation is **a field with a reason, never a deletion**. A deleted row leaves +no record that the claim was made, which is the first thing a reviewer asks for. + +### `delta_is_floor` — the guard that falls out of the traces + +BurstGPT carries no prefix identity, so a replay of it synthesizes unique blocks +per request. A prefix-cache knob measured there saw the **least** reuse the real +traffic could have had, so its delta is a **lower bound** — usable as "at least +this much", never quotable as the gain. `summary()` prints +`[FLOOR: prefixes synthesized]` so it cannot be lost in a paste. + +`ex6` is that case. `ex2` is the control: same synthesized trace, a knob that does +not depend on reuse, so **not** a floor. + +### The shipped examples contain nothing selectable + +``` +$ python -m gitm.playbook --show benchmarks/playbook/examples.json +benchmarks/playbook/examples.json: 6 rows, 0 selectable +``` + +The **regimes are real**, measured off the pinned fixtures. The **deltas are +invented** and every row says so. A perfectly matching query returns nothing and +names the reason per row. The **largest claimed delta in the file (+22%) is the +scoreboard row** — the one most likely to be copied, and the one gated out by +equality rather than distance. + +--- + +## Verification + +``` +python -m gitm.traffic --selftest 24 checks, 3 real traces, 7 drop reasons +python -m gitm.playbook --selftest 19 checks, 2 real regimes, 0 calibrated axes +pytest tests/test_traffic.py tests/test_playbook.py -q 45 passed +ruff check gitm/traffic gitm/playbook All checks passed! +``` + +Assertions are written **once** and run from both entry points, so the check a +reader is told about and the check CI runs are the same check. +`test_every_check_is_registered` fails if a `check_*` function is defined and left +out of the list. + +Full suite: **11 failures, all pre-existing** — module-for-module identical to the +list on clean `b2da5b6`, all missing optional deps or importer goldens. **None in +`gitm/traffic` or `gitm/playbook`.** + +--- + +## Two bugs found by running it rather than reading it + +- **`runner.py` executed a bare `vllm` from `PATH`**, which fails under an + absolute interpreter (a conda env used without activation — normal in CI and + WSL) and, worse, **could have validated one install and run another**: + `check_vllm()` reads metadata for *this* interpreter. The binary is now derived + from `sys.executable`, so the two cannot differ. +- **`--tokenizer` was missing from the argv builder.** `bench serve` constructs a + tokenizer from `--model` even though `timed_trace` sends pre-tokenized prompts, + so any served name HuggingFace cannot resolve dies in + `AutoTokenizer.from_pretrained` — long after the endpoint answered. + +--- + +## Deliberately not built + +- **A custom load generator.** vLLM replays timestamps natively. +- **Adapters beyond BurstGPT and Mooncake.** Two prove the canonical schema. +- **Discovery mode.** The handoff is defined; the mode is not designed here. +- **A calibrated distance threshold.** The type refuses a number without the run + that produced it. +- **Session-aware *firing*.** BurstGPT_3 carries session identity and the adapter + reads it, but `timed_trace` has no session field. Deriving prefix blocks from + sessions would invent cache hits. + +## Known open + +- **Knob and environment fields are `D2`** until the shared + config-capture schema exists. `EnvCapture` is the named subset this needs; it + is **deleted** and those types imported verbatim the moment they land. No + translation layer. +- **Pacing under saturation.** The smoke-test server answers instantly, so pacing + was verified against a server that never queues. +- **The viewer's chart has not been looked at in a browser** — the HTTP layer and + page bytes are checked; the rendering is not. From d427cf643a18286b1419641048fe866f4c71eb0a Mon Sep 17 00:00:00 2001 From: Adit Chawdhary <25533953+aditchawdhary@users.noreply.github.com> Date: Fri, 4 Sep 2026 02:28:57 -0700 Subject: [PATCH 4/8] Delete PARASAIL_POC.md --- PARASAIL_POC.md | 276 ------------------------------------------------ 1 file changed, 276 deletions(-) delete mode 100644 PARASAIL_POC.md diff --git a/PARASAIL_POC.md b/PARASAIL_POC.md deleted file mode 100644 index 37549d9..0000000 --- a/PARASAIL_POC.md +++ /dev/null @@ -1,276 +0,0 @@ -# Parasail POC — traffic replay and the playbook schema - -Two pieces of the validation infrastructure, built to answer one question: **when -we say a knob is worth applying, what makes that statement checkable by someone -who was not in the room?** - -- **`gitm/traffic/`** — replay real production traces through a real serving - endpoint, and tie the result back to the workload that produced it. -- **`gitm/playbook/`** — the row that says *this knob, on this model and GPU, - under this traffic, moved these numbers by this much, and here is everything - needed to re-check it.* - -CPU-only except for firing a replay. No new dependencies. - ---- - -## Five-minute tour - -```bash -# 1. Every claim below, as one runnable check -python -m gitm.traffic --selftest # 24 checks, 3 real traces -python -m gitm.playbook --selftest # 19 checks, 2 real regimes - -# 2. What a real production trace actually contains -python -m gitm.traffic --describe mooncake \ - benchmarks/traffic_replay/fixtures/mooncake_slice.jsonl - -# 3. Replay it, and prove the replay preserved it -python -m gitm.traffic --replay mooncake \ - benchmarks/traffic_replay/fixtures/mooncake_slice.jsonl --out /tmp/mc.jsonl - -# 4. What a playbook row looks like, and why none of these is selectable -python -m gitm.playbook --show benchmarks/playbook/examples.json -python -m gitm.playbook --lookup benchmarks/playbook/examples.json ex1-prefix-cache-mooncake -``` - -Step 3 prints a validation table in which **every statistic is exactly 0.0**, and -the arrival-rate profiles of source and replay drawn one above the other. Step 4 -returns **nothing** and says why, per row. - ---- - -## What is measured, and what is not - -The distinction is load-bearing, so it is stated before anything else. - -| | state | -|---|---| -| The adapters, on real published bytes | ✅ measured | -| Replay fidelity, source vs the file vLLM reads | ✅ measured — every statistic exactly 0.0 | -| Firing at a live vLLM endpoint | ✅ **done**, against real `vllm bench serve` 0.28.0 | -| Joining a result back to its trace | ✅ measured, end to end, exit 0 | -| **Any playbook row being true** | ❌ **no.** Every shipped row is `illustrative` and its delta is invented | -| The regime-distance threshold | ❌ **uncalibrated, and the code says so at runtime** | -| Pacing under a *saturating* server | ❌ needs a GPU | - -**Nothing here claims a performance win.** It is the machinery that would make -such a claim checkable. - ---- - -## `gitm/traffic/` — traffic replay - -### Real traces, not synthetic load - -Two published production traces, pinned by sha256 through -`gitm.bench.manifest`. Measured by the adapters: - -| trace | rows | input p50 / p95 | dispersion `D` | regime label | -|---|---|---|---|---| -| BurstGPT_1 (Azure OpenAI) | 383/400 | 353 / 1,638 | 1.01 | `prod/io1/in256/out128/burst-poisson/copen` | -| BurstGPT_3 (8-column layout) | 399/400 | 309 / 1,497 | 1.01 | `prod/io2/in256/out64/burst-poisson/copen` | -| Mooncake (Kimi serving) | 400/400 | 9,075 / 49,904 | 6.74 | `prod/io32/in8k/out256/burst-hi/copen` | - -The two axes that matter separate cleanly — **D 6.74 vs 1.01, input p50 9,075 vs -353.** If the regime axes could not tell two real production traces apart they -would be decoration; `check_regime_axes_separate_the_traces` asserts they do. - -### Real data is the work, not the parsing - -**7 named drop reasons**, each firing exactly once on both a CSV and a JSONL -dirty fixture. The counts land in `TraceMeta`, and **a trace whose counts do not -reconcile cannot be constructed** — `rows_read == rows_emitted + dropped` is -enforced in `__post_init__`, not by convention. - -The zero-token rows in BurstGPT are real and are the majority defect: **744 of -9,382 `Conversation log` rows carry 0/0**, while all 19 `API log` rows are -well-formed. A caller's own filter counts separately from bad data and still -reconciles. - -### No custom load generator - -Replay goes through vLLM's native `bench serve --dataset-name timed_trace ---self-timed`, which schedules every request at its own timestamp. Fidelity is -checked against **the file vLLM will actually read**, not an in-memory copy: - -``` -trace validation [replay] mooncake(8090d6a38401) -> timed_trace(dfd989ad6d50) -check value threshold result -request_count 0 0 pass -arrival_ks 0 0.001 pass -input_len_ks 0 0.001 pass -output_len_ks 0 0.001 pass -rate_rps 0 1e-06 pass -burstiness 0 1e-06 pass -PASS — the pipeline preserves the trace -``` - -**Fired end to end against real vLLM 0.28.0**: 40/40 completed, paced to -**12.008 s against a 12.000 s trace span** — confirmed by two independent clocks, -vLLM's own and the receiving server's. - -### The silent failure this exists to catch - -vLLM's `--timed-trace-chunk-hash-size` defaults to **16**. Mooncake's cache -blocks are **512**. At the default, every prompt is **32× short** — while -`completed`, `duration`, throughput and every latency percentile still read -perfectly. - -The emitter refuses to write such a file, and the joiner catches one fired from a -plan built elsewhere. Confirmed on the real run: `total_input_tokens` came back -**506,280 against the trace's 506,280** — exact; at 16-token blocks it would have -been ~15,821. - -### The result JSON is not merely incomplete — two fields are wrong - -`bench serve --save-result` writes 34 keys. Under `--self-timed` vLLM still -records the CLI's `request_rate` (`"inf"` — a *string*) and `burstiness` (`1.0`), -against the trace's real 2.837 rps and D 6.74. **They are exactly the two axes a -playbook row keys on.** - -They are **dropped with a stated reason** rather than merged or renamed, with the -values kept visible. A new vLLM field cannot fall out silently either — -`unjoined_keys()` fails on any key neither kept nor deliberately dropped, and it -caught `rtfx` on its first run. - ---- - -## `gitm/playbook/` — the row, and when it may be used - -### Row identity is a split, not a lookup - -``` -(model+revision, GPU SKU, environment, workload regime, knob set) - -> measured delta + provenance -``` - -**Exact equality** on model+revision, GPU SKU, engine+version, `source_kind` and -concurrency. **Distance** on the numeric regime axes, because live traffic never -lands on a measured point. Each gate is a named field on `MatchPolicy`, so -loosening one is an edit visible in a diff rather than an accident inside a -comparison. - -`source_kind` is asserted the hard way — **with every numeric axis identical**, so -only the gate can be doing the work. A scoreboard result never satisfies a -production query. - -### Distance is a log2 ratio, combined with L-infinity - -`|log2(a/b)|` — scale-free, symmetric, zero at equality, and `inf` when one side -is zero, because "no output tokens" is not a small version of "some". - -**L-infinity, not a mean.** A row is as far away as its worst axis. Asserted on -the case it exists for: a candidate identical on five axes and 8× off on -`input_p95` reads **0.5 as a mean** — close enough to apply — and **3.0 as -L-inf**. For scale, the two real traces are **4.929 apart, limited by -`input_p95`** (49,904 vs 1,638). - -### The threshold is uncalibrated, and the type refuses to pretend otherwise - -```python -AxisTolerance(max_distance=1.0) # ValidationError -AxisTolerance(max_distance=1.0, calibration="prereg E4: sign flip at 1.4") # ok -``` - -No tolerance ships. Consequences, all asserted: - -- an **exact** regime match still returns a row, so the schema is usable today; -- any nonzero distance returns `UNCALIBRATED` **naming the limiting axis**, and - routes to discovery; -- a placeholder cannot quietly become a production constant. - -Calibration needs the same knob measured across nearby regimes, which needs a -GPU. **The cost of the open state is a discovery run; the cost of an invented -threshold is a knob applied to traffic nobody measured it on.** - -### A row cannot exist without its evidence - -Refused at construction: no knobs, `repeats < 2`, a delta missing its latency -percentiles, an empty `trace_sha256`. `MeasuredDelta` carries throughput **and** -TTFT/ITL percentiles — the schema cannot express a throughput-only row. - -Invalidation is **a field with a reason, never a deletion**. A deleted row leaves -no record that the claim was made, which is the first thing a reviewer asks for. - -### `delta_is_floor` — the guard that falls out of the traces - -BurstGPT carries no prefix identity, so a replay of it synthesizes unique blocks -per request. A prefix-cache knob measured there saw the **least** reuse the real -traffic could have had, so its delta is a **lower bound** — usable as "at least -this much", never quotable as the gain. `summary()` prints -`[FLOOR: prefixes synthesized]` so it cannot be lost in a paste. - -`ex6` is that case. `ex2` is the control: same synthesized trace, a knob that does -not depend on reuse, so **not** a floor. - -### The shipped examples contain nothing selectable - -``` -$ python -m gitm.playbook --show benchmarks/playbook/examples.json -benchmarks/playbook/examples.json: 6 rows, 0 selectable -``` - -The **regimes are real**, measured off the pinned fixtures. The **deltas are -invented** and every row says so. A perfectly matching query returns nothing and -names the reason per row. The **largest claimed delta in the file (+22%) is the -scoreboard row** — the one most likely to be copied, and the one gated out by -equality rather than distance. - ---- - -## Verification - -``` -python -m gitm.traffic --selftest 24 checks, 3 real traces, 7 drop reasons -python -m gitm.playbook --selftest 19 checks, 2 real regimes, 0 calibrated axes -pytest tests/test_traffic.py tests/test_playbook.py -q 45 passed -ruff check gitm/traffic gitm/playbook All checks passed! -``` - -Assertions are written **once** and run from both entry points, so the check a -reader is told about and the check CI runs are the same check. -`test_every_check_is_registered` fails if a `check_*` function is defined and left -out of the list. - -Full suite: **11 failures, all pre-existing** — module-for-module identical to the -list on clean `b2da5b6`, all missing optional deps or importer goldens. **None in -`gitm/traffic` or `gitm/playbook`.** - ---- - -## Two bugs found by running it rather than reading it - -- **`runner.py` executed a bare `vllm` from `PATH`**, which fails under an - absolute interpreter (a conda env used without activation — normal in CI and - WSL) and, worse, **could have validated one install and run another**: - `check_vllm()` reads metadata for *this* interpreter. The binary is now derived - from `sys.executable`, so the two cannot differ. -- **`--tokenizer` was missing from the argv builder.** `bench serve` constructs a - tokenizer from `--model` even though `timed_trace` sends pre-tokenized prompts, - so any served name HuggingFace cannot resolve dies in - `AutoTokenizer.from_pretrained` — long after the endpoint answered. - ---- - -## Deliberately not built - -- **A custom load generator.** vLLM replays timestamps natively. -- **Adapters beyond BurstGPT and Mooncake.** Two prove the canonical schema. -- **Discovery mode.** The handoff is defined; the mode is not designed here. -- **A calibrated distance threshold.** The type refuses a number without the run - that produced it. -- **Session-aware *firing*.** BurstGPT_3 carries session identity and the adapter - reads it, but `timed_trace` has no session field. Deriving prefix blocks from - sessions would invent cache hits. - -## Known open - -- **Knob and environment fields are `D2`** until the shared - config-capture schema exists. `EnvCapture` is the named subset this needs; it - is **deleted** and those types imported verbatim the moment they land. No - translation layer. -- **Pacing under saturation.** The smoke-test server answers instantly, so pacing - was verified against a server that never queues. -- **The viewer's chart has not been looked at in a browser** — the HTTP layer and - page bytes are checked; the rendering is not. From eb235284651e04326d9fe5a39cd83f69c88a20be Mon Sep 17 00:00:00 2001 From: Adit Chawdhary <25533953+aditchawdhary@users.noreply.github.com> Date: Fri, 4 Sep 2026 02:30:55 -0700 Subject: [PATCH 5/8] Clean up comments in pyproject.toml Removed comments regarding version constraints and traffic library usage. --- pyproject.toml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d707c25..0df83ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,26 +29,9 @@ nvidia = ["pynvml>=11.5"] # fallback (pandas). On a GPU box the harness runs on cuDF/CuPy instead — those # install from the RAPIDS channel, not PyPI, so they are not listed here. bench = ["pyarrow>=15", "pandas>=2.0"] -# Two consumers: the vllm-decode workload, and *firing* a traffic replay -# (gitm/traffic/replay.py). Deliberately NOT folded into [gpu]: vLLM is a heavy # install that pins its own torch, and both must be built for a CUDA major the # HOST driver supports — see gitm/cuda_env.py, which checks exactly that. # -# Floor is 0.23.0 because `bench serve --dataset-name timed_trace` landed in -# bfb9ebc21 (2026-05-28), one day AFTER v0.22.0 shipped — checked at the tags, -# not inferred from dates: v0.22.0 ❌, v0.22.1 ❌, v0.23.0 ✅. Below it the run -# dies on an argparse complaint about an unknown dataset name, which reads like a -# typo in our command rather than a missing feature. A too-high floor fails -# loudly at install time; a too-low one fails quietly at run time, so the floor -# goes up. gitm.traffic.replay.VLLM_MIN_VERSION carries the same number for the -# runtime guard. -# -# NOT split into a separate [traffic] extra: the traffic library is CPU-only -# except for firing — adapters, regime, replay-file writing, validation and the -# viewer all run with no vLLM at all — so an extra named for it would say you -# need a multi-gigabyte CUDA install to use a library that does not need one. -# Phase B of the chunked-prefill experiment needs only >=0.19.0 for its own -# features; it is covered by this floor, not constrained to it. vllm = ["vllm>=0.23.0"] # Full GPU stack for the real HFT path in two commands: cuDF/CuPy (RAPIDS) + # the CUPTI/runtime wheels the tracer shim links + NVML telemetry + the Parquet From f81b10c344906ad57f6799115ed302b9e5437194 Mon Sep 17 00:00:00 2001 From: Adit Chawdhary <25533953+aditchawdhary@users.noreply.github.com> Date: Fri, 4 Sep 2026 02:31:41 -0700 Subject: [PATCH 6/8] Clean up comments in pyproject.toml Removed comments related to CUDA and dependencies. --- pyproject.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0df83ee..69ec59f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,9 +29,6 @@ nvidia = ["pynvml>=11.5"] # fallback (pandas). On a GPU box the harness runs on cuDF/CuPy instead — those # install from the RAPIDS channel, not PyPI, so they are not listed here. bench = ["pyarrow>=15", "pandas>=2.0"] -# install that pins its own torch, and both must be built for a CUDA major the -# HOST driver supports — see gitm/cuda_env.py, which checks exactly that. -# vllm = ["vllm>=0.23.0"] # Full GPU stack for the real HFT path in two commands: cuDF/CuPy (RAPIDS) + # the CUPTI/runtime wheels the tracer shim links + NVML telemetry + the Parquet From 229099ea0547dddda87db2c3cd73a9d5ca68837b Mon Sep 17 00:00:00 2001 From: Adit Chawdhary <25533953+aditchawdhary@users.noreply.github.com> Date: Fri, 4 Sep 2026 02:32:25 -0700 Subject: [PATCH 7/8] Delete gitm/_banner.py --- gitm/_banner.py | 56 ------------------------------------------------- 1 file changed, 56 deletions(-) delete mode 100644 gitm/_banner.py diff --git a/gitm/_banner.py b/gitm/_banner.py deleted file mode 100644 index 74a74d9..0000000 --- a/gitm/_banner.py +++ /dev/null @@ -1,56 +0,0 @@ -"""The ``GitM`` banner, and the three rules that stop it breaking a pipe. - -Decoration on a CLI that also emits JSON and parseable tables is a hazard, not a -feature, so this module is mostly the guard rather than the art: - -1. **stderr, never stdout.** Every byte on stdout belongs to whoever is reading - it — `jq`, a CI step, a `> results.json`. A banner there corrupts the payload, - and the failure shows up far from here as a JSON parse error. -2. **Only when stdout is a TTY.** Redirected stdout means a machine is reading, - and a machine reading stdout usually means stderr is being captured into the - same log. Gate on stdout even though we write to stderr: the question is "is a - human watching this run", and stdout is where the answer is. -3. **Two escapes.** ``--no-banner`` for one invocation, ``GITM_NO_BANNER`` for a - shell, a Makefile or a container that should never show it. - -Cosmetic by intent. The point of writing the guard out is that the cosmetic part -is the part that is safe to change. -""" - -from __future__ import annotations - -import os -import sys - -#: Set to anything non-empty to suppress the banner everywhere. -ENV_VAR = "GITM_NO_BANNER" - -BANNER = r""" - ___ _ _ __ __ - / __(_) |_| \/ | git machines - | (_ | | _| |\/| | inference runtime - \___|_|\__|_| |_| -""" - - -def show_banner(*, suppressed: bool = False, stream=None) -> bool: - """Write the banner to stderr. Returns whether it was actually written. - - ``suppressed`` is the parsed ``--no-banner`` flag. ``stream`` exists for the - selftest; production callers never pass it, and it does not change the TTY - gate — the gate asks about **stdout** regardless of where the banner goes, - because the question is whether a human is watching, not where the bytes land. - """ - if suppressed or os.environ.get(ENV_VAR): - return False - if not (hasattr(sys.stdout, "isatty") and sys.stdout.isatty()): - return False - print(BANNER.strip("\n"), file=stream if stream is not None else sys.stderr) - return True - - -def add_banner_argument(parser) -> None: - """Add ``--no-banner`` to an ``argparse`` parser. One line per entry point.""" - parser.add_argument( - "--no-banner", action="store_true", help="suppress the startup banner" - ) From 331ae134715c79da34174fcb19cdc9cdd05c351f Mon Sep 17 00:00:00 2001 From: Adit Chawdhary <25533953+aditchawdhary@users.noreply.github.com> Date: Fri, 4 Sep 2026 02:35:51 -0700 Subject: [PATCH 8/8] Remove banner argument from CLI Removed banner argument handling from CLI parser. --- gitm/cli.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/gitm/cli.py b/gitm/cli.py index f5e56da..7bd63fc 100644 --- a/gitm/cli.py +++ b/gitm/cli.py @@ -7,7 +7,6 @@ import sys from pathlib import Path -from gitm._banner import add_banner_argument, show_banner from gitm.optimizer.deviation import add_deviate_arguments from gitm.planner.registry import add_plan_arguments @@ -80,7 +79,6 @@ def _parser() -> argparse.ArgumentParser: description="Behavioral compiler and intervention runtime.", ) p.add_argument("--version", action="store_true", help="Print version and exit.") - add_banner_argument(p) sub = p.add_subparsers(dest="cmd") run = sub.add_parser("run", help="Run the autonomous optimization loop.") @@ -320,10 +318,6 @@ def main(argv: list[str] | None = None) -> int: args = _parser().parse_args(argv) - # Before any output, and never on stdout: --version and the JSON-emitting - # subcommands write machine-readable payloads there. - show_banner(suppressed=args.no_banner) - if args.version: from gitm import __version__