diff --git a/benchmarks/playbook/examples.json b/benchmarks/playbook/examples.json new file mode 100644 index 0000000..cbe5864 --- /dev/null +++ b/benchmarks/playbook/examples.json @@ -0,0 +1,400 @@ +{ + "schema_id": "gitm.playbook.row/v1", + "rows": [ + { + "schema_id": "gitm.playbook.row/v1", + "row_id": "ex1-prefix-cache-mooncake", + "identity": { + "model": "Qwen/Qwen3.6-35B-A3B-FP8", + "model_revision": "95a723d0", + "gpu_sku": "NVIDIA H100 80GB", + "env": { + "schema_id": "pending-adit/env-capture", + "engine": "vllm", + "engine_version": "0.11.0", + "driver_version": null, + "torch_version": null, + "cuda_version": null + }, + "regime": { + "source_kind": "production", + "trace": "mooncake", + "requests": 400, + "rate_rps": 2.8368794326241136, + "io_ratio": 39.09367234191124, + "input_p50": 9075, + "input_p95": 49904, + "output_p50": 370, + "output_p95": 662, + "burstiness": 6.738120567375886, + "bin_s": 1.0, + "burstiness_defined": true, + "concurrency": null, + "in_envelope": true, + "notes": [] + }, + "knobs": { + "enable_prefix_caching": true + } + }, + "delta": { + "throughput_pct": 14.5, + "ttft_p99_ms": -18.0, + "itl_p99_ms": 0.3, + "repeats": 5, + "throughput_ci95_pct": [ + 9.1, + 19.4 + ], + "latency_blowout": null + }, + "provenance": { + "trace_source": "mooncake", + "trace_sha256": "8090d6a38401d280fd88112ba22007ff4354be59c986f4f6bcc250edd45a52dc", + "trace_drops": {}, + "regime_label": "prod/io32/in8k/out256/burst-hi/copen", + "repeat_raw_data": [], + "promotion_rule": "pending-adit/promotion-rule", + "config_capture": "pending-adit", + "verified_at": "2026-09-02T00:00:00Z", + "replay_chunk_hash_size": 512, + "replay_self_timed": true, + "prefix_synthesized": false + }, + "evidence": "illustrative", + "invalidated": null, + "notes": [ + "Regime is real (measured off the pinned Mooncake fixture). The delta is not: no run against a live endpoint has happened. Shape to build against, not a claim." + ] + }, + { + "schema_id": "gitm.playbook.row/v1", + "row_id": "ex2-max-num-seqs-burstgpt", + "identity": { + "model": "Qwen/Qwen3.6-35B-A3B-FP8", + "model_revision": "95a723d0", + "gpu_sku": "NVIDIA H100 80GB", + "env": { + "schema_id": "pending-adit/env-capture", + "engine": "vllm", + "engine_version": "0.11.0", + "driver_version": null, + "torch_version": null, + "cuda_version": null + }, + "regime": { + "source_kind": "production", + "trace": "burstgpt", + "requests": 383, + "rate_rps": 0.010276637419839545, + "io_ratio": 1.8821556431490254, + "input_p50": 353, + "input_p95": 1638, + "output_p50": 238, + "output_p95": 841, + "burstiness": 1.0106110910396904, + "bin_s": 1.0, + "burstiness_defined": true, + "concurrency": null, + "in_envelope": true, + "notes": [] + }, + "knobs": { + "max_num_seqs": 64 + } + }, + "delta": { + "throughput_pct": 8.1, + "ttft_p99_ms": -4.2, + "itl_p99_ms": 1.0, + "repeats": 5, + "throughput_ci95_pct": [ + 3.3, + 12.6 + ], + "latency_blowout": null + }, + "provenance": { + "trace_source": "burstgpt", + "trace_sha256": "7865fda5dac50938a84dfccb06c405e2f5202969ecfbd671c8399de00f4335f0", + "trace_drops": { + "zero_input_tokens": 17 + }, + "regime_label": "prod/io1/in256/out128/burst-poisson/copen", + "repeat_raw_data": [], + "promotion_rule": "pending-adit/promotion-rule", + "config_capture": "pending-adit", + "verified_at": "2026-09-02T00:00:00Z", + "replay_chunk_hash_size": 512, + "replay_self_timed": true, + "prefix_synthesized": true + }, + "evidence": "illustrative", + "invalidated": null, + "notes": [ + "The counter-example to ex1: same model, same GPU, no prefix identity in the source at all. A prefix-caching row must never be selected for this regime." + ] + }, + { + "schema_id": "gitm.playbook.row/v1", + "row_id": "ex3-chunked-prefill-qwen", + "identity": { + "model": "Qwen/Qwen3.6-35B-A3B-FP8", + "model_revision": "95a723d0", + "gpu_sku": "NVIDIA H100 80GB", + "env": { + "schema_id": "pending-adit/env-capture", + "engine": "vllm", + "engine_version": "0.11.0", + "driver_version": null, + "torch_version": null, + "cuda_version": null + }, + "regime": { + "source_kind": "production", + "trace": "mooncake", + "requests": 400, + "rate_rps": 2.8368794326241136, + "io_ratio": 39.09367234191124, + "input_p50": 9075, + "input_p95": 49904, + "output_p50": 370, + "output_p95": 662, + "burstiness": 6.738120567375886, + "bin_s": 1.0, + "burstiness_defined": true, + "concurrency": null, + "in_envelope": true, + "notes": [] + }, + "knobs": { + "enable_chunked_prefill": true, + "max_num_batched_tokens": 8192 + } + }, + "delta": { + "throughput_pct": 6.4, + "ttft_p99_ms": -31.0, + "itl_p99_ms": 2.1, + "repeats": 5, + "throughput_ci95_pct": null, + "latency_blowout": null + }, + "provenance": { + "trace_source": "mooncake", + "trace_sha256": "8090d6a38401d280fd88112ba22007ff4354be59c986f4f6bcc250edd45a52dc", + "trace_drops": {}, + "regime_label": "prod/io32/in8k/out256/burst-hi/copen", + "repeat_raw_data": [], + "promotion_rule": "pending-adit/promotion-rule", + "config_capture": "pending-adit", + "verified_at": "2026-09-02T00:00:00Z", + "replay_chunk_hash_size": 512, + "replay_self_timed": true, + "prefix_synthesized": false + }, + "evidence": "illustrative", + "invalidated": null, + "notes": [ + "This is the row deliverable 3 would produce. It stays illustrative until Phase B runs -- that needs one 80 GB card, which we do not have." + ] + }, + { + "schema_id": "gitm.playbook.row/v1", + "row_id": "ex4-retired-engine-bump", + "identity": { + "model": "Qwen/Qwen3.6-35B-A3B-FP8", + "model_revision": "95a723d0", + "gpu_sku": "NVIDIA H100 80GB", + "env": { + "schema_id": "pending-adit/env-capture", + "engine": "vllm", + "engine_version": "0.12.0", + "driver_version": null, + "torch_version": null, + "cuda_version": null + }, + "regime": { + "source_kind": "production", + "trace": "mooncake", + "requests": 400, + "rate_rps": 2.8368794326241136, + "io_ratio": 39.09367234191124, + "input_p50": 9075, + "input_p95": 49904, + "output_p50": 370, + "output_p95": 662, + "burstiness": 6.738120567375886, + "bin_s": 1.0, + "burstiness_defined": true, + "concurrency": null, + "in_envelope": true, + "notes": [] + }, + "knobs": { + "enable_prefix_caching": true + } + }, + "delta": { + "throughput_pct": 11.0, + "ttft_p99_ms": -12.0, + "itl_p99_ms": 0.4, + "repeats": 4, + "throughput_ci95_pct": null, + "latency_blowout": null + }, + "provenance": { + "trace_source": "mooncake", + "trace_sha256": "8090d6a38401d280fd88112ba22007ff4354be59c986f4f6bcc250edd45a52dc", + "trace_drops": {}, + "regime_label": "prod/io32/in8k/out256/burst-hi/copen", + "repeat_raw_data": [], + "promotion_rule": "pending-adit/promotion-rule", + "config_capture": "pending-adit", + "verified_at": "2026-09-02T00:00:00Z", + "replay_chunk_hash_size": 512, + "replay_self_timed": true, + "prefix_synthesized": false + }, + "evidence": "illustrative", + "invalidated": { + "reason": "vLLM 0.11 -> 0.12 scheduler rewrite; the measured delta is against a scheduler that no longer exists", + "at": "2026-09-02T00:00:00Z", + "by": "validation" + }, + "notes": [ + "Kept, not deleted. A deleted row leaves no record that the claim was made, which is the first thing a reviewer asks for." + ] + }, + { + "schema_id": "gitm.playbook.row/v1", + "row_id": "ex6-prefix-cache-on-a-synthesized-trace", + "identity": { + "model": "Qwen/Qwen3.6-35B-A3B-FP8", + "model_revision": "95a723d0", + "gpu_sku": "NVIDIA H100 80GB", + "env": { + "schema_id": "pending-adit/env-capture", + "engine": "vllm", + "engine_version": "0.11.0", + "driver_version": null, + "torch_version": null, + "cuda_version": null + }, + "regime": { + "source_kind": "production", + "trace": "burstgpt", + "requests": 383, + "rate_rps": 0.010276637419839545, + "io_ratio": 1.8821556431490254, + "input_p50": 353, + "input_p95": 1638, + "output_p50": 238, + "output_p95": 841, + "burstiness": 1.0106110910396904, + "bin_s": 1.0, + "burstiness_defined": true, + "concurrency": null, + "in_envelope": true, + "notes": [] + }, + "knobs": { + "enable_prefix_caching": true + } + }, + "delta": { + "throughput_pct": 1.2, + "ttft_p99_ms": -0.4, + "itl_p99_ms": 0.0, + "repeats": 5, + "throughput_ci95_pct": null, + "latency_blowout": null + }, + "provenance": { + "trace_source": "burstgpt", + "trace_sha256": "7865fda5dac50938a84dfccb06c405e2f5202969ecfbd671c8399de00f4335f0", + "trace_drops": { + "zero_input_tokens": 17 + }, + "regime_label": "prod/io1/in256/out128/burst-poisson/copen", + "repeat_raw_data": [], + "promotion_rule": "pending-adit/promotion-rule", + "config_capture": "pending-adit", + "verified_at": "2026-09-02T00:00:00Z", + "replay_chunk_hash_size": 512, + "replay_self_timed": true, + "prefix_synthesized": true + }, + "evidence": "illustrative", + "invalidated": null, + "notes": [ + "delta_is_floor: BurstGPT has no prefix identity, so D1 synthesized unique blocks per request and the replay saw the LEAST reuse the real traffic could have had. +1.2% is a lower bound, never quotable as the gain." + ] + }, + { + "schema_id": "gitm.playbook.row/v1", + "row_id": "ex5-scoreboard-not-production", + "identity": { + "model": "Qwen/Qwen3.6-35B-A3B-FP8", + "model_revision": "95a723d0", + "gpu_sku": "NVIDIA H100 80GB", + "env": { + "schema_id": "pending-adit/env-capture", + "engine": "vllm", + "engine_version": "0.11.0", + "driver_version": null, + "torch_version": null, + "cuda_version": null + }, + "regime": { + "source_kind": "scoreboard", + "trace": "artificialanalysis-fixed", + "requests": 1000, + "rate_rps": 1.0, + "io_ratio": 4.0, + "input_p50": 1024, + "input_p95": 1024, + "output_p50": 256, + "output_p95": 256, + "burstiness": 0.0, + "bin_s": 1.0, + "burstiness_defined": true, + "concurrency": null, + "in_envelope": true, + "notes": [ + "fixed-length scoreboard condition; not production traffic" + ] + }, + "knobs": { + "max_num_seqs": 64 + } + }, + "delta": { + "throughput_pct": 22.0, + "ttft_p99_ms": -9.0, + "itl_p99_ms": 0.1, + "repeats": 3, + "throughput_ci95_pct": null, + "latency_blowout": null + }, + "provenance": { + "trace_source": "mooncake", + "trace_sha256": "8090d6a38401d280fd88112ba22007ff4354be59c986f4f6bcc250edd45a52dc", + "trace_drops": {}, + "regime_label": "board/io4/in1k/out256/burst-flat/copen", + "repeat_raw_data": [], + "promotion_rule": "pending-adit/promotion-rule", + "config_capture": "pending-adit", + "verified_at": "2026-09-02T00:00:00Z", + "replay_chunk_hash_size": 512, + "replay_self_timed": true, + "prefix_synthesized": false + }, + "evidence": "illustrative", + "invalidated": null, + "notes": [ + "source_kind=scoreboard. The biggest claimed delta in the file, and it is gated out of every production lookup by equality, not by distance." + ] + } + ] +} diff --git a/benchmarks/playbook/make_examples.py b/benchmarks/playbook/make_examples.py new file mode 100644 index 0000000..906ce04 --- /dev/null +++ b/benchmarks/playbook/make_examples.py @@ -0,0 +1,140 @@ +"""Generate benchmarks/playbook/examples.json from the REAL D1 regimes. + +The regimes are measured off the committed traffic fixtures; the deltas are not +measured and every row says so (evidence=illustrative). +""" +import json +from datetime import datetime, timezone +from pathlib import Path + +from gitm.bench.manifest import sha256_file +from gitm.playbook.schema import ( + EnvCapture, + Evidence, + Invalidation, + MeasuredDelta, + Playbook, + PlaybookRow, + Provenance, + RowIdentity, +) +from gitm.traffic import Regime, SourceKind, read_burstgpt, read_mooncake + +F = Path("benchmarks/traffic_replay/fixtures") +bg_path, mc_path = F / "burstgpt_slice.csv", F / "mooncake_slice.jsonl" +bg = read_burstgpt(bg_path) +mc = read_mooncake(mc_path) +bg_r = Regime.from_trace(bg) +mc_r = Regime.from_trace(mc) + +H100 = "NVIDIA H100 80GB" +MODEL = "Qwen/Qwen3.6-35B-A3B-FP8" +REV = "95a723d0" +ENV = EnvCapture(engine="vllm", engine_version="0.11.0") +ENV_NEXT = EnvCapture(engine="vllm", engine_version="0.12.0") +T = datetime(2026, 9, 2, tzinfo=timezone.utc) + + +def prov(trace, path, regime, **kw): + """Provenance as D1 would emit it: real checksum, real drops, real replay + conditions. BurstGPT has no prefix identity, so a replay of it always + synthesizes blocks -- which is what makes ex6 a floor and not a measurement.""" + kw.setdefault("replay_chunk_hash_size", 512) + kw.setdefault("replay_self_timed", True) + kw.setdefault("prefix_synthesized", not trace.meta.has_prefix_identity) + return Provenance( + trace_source=trace.meta.source, + trace_sha256=sha256_file(path)[0], + trace_drops=dict(trace.meta.drops), + regime_label=regime.label(), + repeat_raw_data=[], + verified_at=T, + **kw, + ) + + +# A scoreboard regime: Artificial Analysis' fixed-length condition, as its own +# named source_kind. Numbers are the published fixed shape, not a measurement of +# ours -- which is exactly why it must never match a production query. +board_r = mc_r.model_copy(update={ + "source_kind": SourceKind.SCOREBOARD, "trace": "artificialanalysis-fixed", + "requests": 1000, "rate_rps": 1.0, "io_ratio": 4.0, + "input_p50": 1024, "input_p95": 1024, "output_p50": 256, "output_p95": 256, + "burstiness": 0.0, "notes": ["fixed-length scoreboard condition; not production traffic"], +}) + +rows = [ + PlaybookRow( + row_id="ex1-prefix-cache-mooncake", + identity=RowIdentity(model=MODEL, model_revision=REV, gpu_sku=H100, env=ENV, + regime=mc_r, knobs={"enable_prefix_caching": True}), + delta=MeasuredDelta(throughput_pct=14.5, ttft_p99_ms=-18.0, itl_p99_ms=0.3, + repeats=5, throughput_ci95_pct=(9.1, 19.4)), + provenance=prov(mc, mc_path, mc_r), + evidence=Evidence.ILLUSTRATIVE, + notes=["Regime is real (measured off the pinned Mooncake fixture). The delta is not: " + "no run against a live endpoint has happened. Shape to build against, not a claim."], + ), + PlaybookRow( + row_id="ex2-max-num-seqs-burstgpt", + identity=RowIdentity(model=MODEL, model_revision=REV, gpu_sku=H100, env=ENV, + regime=bg_r, knobs={"max_num_seqs": 64}), + delta=MeasuredDelta(throughput_pct=8.1, ttft_p99_ms=-4.2, itl_p99_ms=1.0, + repeats=5, throughput_ci95_pct=(3.3, 12.6)), + provenance=prov(bg, bg_path, bg_r), + evidence=Evidence.ILLUSTRATIVE, + notes=["The counter-example to ex1: same model, same GPU, no prefix identity in the " + "source at all. A prefix-caching row must never be selected for this regime."], + ), + PlaybookRow( + row_id="ex3-chunked-prefill-qwen", + identity=RowIdentity(model=MODEL, model_revision=REV, gpu_sku=H100, env=ENV, + regime=mc_r, knobs={"enable_chunked_prefill": True, + "max_num_batched_tokens": 8192}), + delta=MeasuredDelta(throughput_pct=6.4, ttft_p99_ms=-31.0, itl_p99_ms=2.1, repeats=5), + provenance=prov(mc, mc_path, mc_r, promotion_rule="pending-adit/promotion-rule"), + evidence=Evidence.ILLUSTRATIVE, + notes=["This is the row deliverable 3 would produce. It stays illustrative until " + "Phase B runs -- that needs one 80 GB card, which we do not have."], + ), + PlaybookRow( + row_id="ex4-retired-engine-bump", + identity=RowIdentity(model=MODEL, model_revision=REV, gpu_sku=H100, env=ENV_NEXT, + regime=mc_r, knobs={"enable_prefix_caching": True}), + delta=MeasuredDelta(throughput_pct=11.0, ttft_p99_ms=-12.0, itl_p99_ms=0.4, repeats=4), + provenance=prov(mc, mc_path, mc_r), + evidence=Evidence.ILLUSTRATIVE, + invalidated=Invalidation(reason="vLLM 0.11 -> 0.12 scheduler rewrite; the measured " + "delta is against a scheduler that no longer exists", + at=datetime(2026, 9, 2, tzinfo=timezone.utc), by="validation"), + notes=["Kept, not deleted. A deleted row leaves no record that the claim was made, " + "which is the first thing a reviewer asks for."], + ), + PlaybookRow( + row_id="ex6-prefix-cache-on-a-synthesized-trace", + identity=RowIdentity(model=MODEL, model_revision=REV, gpu_sku=H100, env=ENV, + regime=bg_r, knobs={"enable_prefix_caching": True}), + delta=MeasuredDelta(throughput_pct=1.2, ttft_p99_ms=-0.4, itl_p99_ms=0.0, repeats=5), + provenance=prov(bg, bg_path, bg_r), + evidence=Evidence.ILLUSTRATIVE, + notes=["delta_is_floor: BurstGPT has no prefix identity, so D1 synthesized unique " + "blocks per request and the replay saw the LEAST reuse the real traffic could " + "have had. +1.2% is a lower bound, never quotable as the gain."], + ), + PlaybookRow( + row_id="ex5-scoreboard-not-production", + identity=RowIdentity(model=MODEL, model_revision=REV, gpu_sku=H100, env=ENV, + regime=board_r, knobs={"max_num_seqs": 64}), + delta=MeasuredDelta(throughput_pct=22.0, ttft_p99_ms=-9.0, itl_p99_ms=0.1, repeats=3), + provenance=prov(mc, mc_path, board_r, config_capture="pending-adit"), + evidence=Evidence.ILLUSTRATIVE, + notes=["source_kind=scoreboard. The biggest claimed delta in the file, and it is " + "gated out of every production lookup by equality, not by distance."], + ), +] + +out = Path("benchmarks/playbook/examples.json") +out.parent.mkdir(parents=True, exist_ok=True) +out.write_text(json.dumps(Playbook(rows=rows).model_dump(mode="json"), indent=2) + "\n", + encoding="utf-8") +print(f"wrote {out} — {len(rows)} rows, {sum(r.selectable for r in rows)} selectable") diff --git a/benchmarks/traffic_replay/fixtures/benchserve_result.json b/benchmarks/traffic_replay/fixtures/benchserve_result.json new file mode 100644 index 0000000..54ceb53 --- /dev/null +++ b/benchmarks/traffic_replay/fixtures/benchserve_result.json @@ -0,0 +1,36 @@ +{ + "date": "20260902-173307", + "endpoint_type": "openai", + "backend": "openai", + "label": null, + "model_id": "STUB", + "tokenizer_id": "gpt2", + "num_prompts": 40, + "request_rate": "inf", + "burstiness": 1.0, + "max_concurrency": null, + "duration": 12.00791824299995, + "completed": 40, + "failed": 0, + "total_input_tokens": 506280, + "total_output_tokens": 40, + "request_throughput": 3.3311352717876903, + "request_goodput": null, + "output_throughput": 3.3311352717876903, + "total_token_throughput": 42165.510270288585, + "max_output_tokens_per_s": 108.0, + "max_concurrent_requests": 16, + "rtfx": 0.0, + "mean_ttft_ms": 192.18192199998612, + "median_ttft_ms": 13.159716000018307, + "std_ttft_ms": 388.61726254662835, + "p99_ttft_ms": 1040.9596590099, + "mean_tpot_ms": 0.0, + "median_tpot_ms": 0.0, + "std_tpot_ms": 0.0, + "p99_tpot_ms": 0.0, + "mean_itl_ms": 0.14320468181949764, + "median_itl_ms": 0.004474500087781053, + "std_itl_ms": 0.3848804774613307, + "p99_itl_ms": 1.9144619700205112 +} diff --git a/benchmarks/traffic_replay/fixtures/burstgpt3_dirty.csv b/benchmarks/traffic_replay/fixtures/burstgpt3_dirty.csv new file mode 100644 index 0000000..83bfba1 --- /dev/null +++ b/benchmarks/traffic_replay/fixtures/burstgpt3_dirty.csv @@ -0,0 +1,7 @@ +Timestamp,Session ID,Elapsed time,Model,Request tokens,Response tokens,Total tokens,Log Type +10,3f2a7c11-0000-4000-8000-000000000001,5,ChatGPT,100,50,150,Conversation log +20,,3,ChatGPT,200,60,260,API log +30,3f2a7c11-0000-4000-8000-000000000001,zzz,ChatGPT,120,40,160,Conversation log +40,3f2a7c11-0000-4000-8000-000000000002,7,ChatGPT,0,0,0,Conversation log +50,3f2a7c11-0000-4000-8000-000000000002 +35,3f2a7c11-0000-4000-8000-000000000003,2,ChatGPT,80,20,100,Conversation log diff --git a/benchmarks/traffic_replay/fixtures/burstgpt3_slice.csv b/benchmarks/traffic_replay/fixtures/burstgpt3_slice.csv new file mode 100644 index 0000000..2a2c222 --- /dev/null +++ b/benchmarks/traffic_replay/fixtures/burstgpt3_slice.csv @@ -0,0 +1,401 @@ +Timestamp,Session ID,Elapsed time,Model,Request tokens,Response tokens,Total tokens,Log Type +19440110.0,1722ac82-0a46-4bf0-aa08-89794e7a2b3f,43,GPT-4,906,446,1352,Conversation log +19440161.0,d5983bf9-4b48-497b-892b-a58995247443,2,ChatGPT,36,29,65,Conversation log +19440192.0,a5c69b35-4fbc-45e9-955e-18715e376d74,8,GPT-4,1779,123,1902,Conversation log +19440254.0,8b74c7d8-1643-4bba-8c69-91cb4548c506,3,ChatGPT,935,178,1113,Conversation log +19440301.0,1722ac82-0a46-4bf0-aa08-89794e7a2b3f,26,GPT-4,1631,282,1913,Conversation log +19440371.0,c6f7a15f-4716-4671-b66f-720888a22e68,2,ChatGPT,728,126,854,Conversation log +19440529.0,8b74c7d8-1643-4bba-8c69-91cb4548c506,3,ChatGPT,1136,175,1311,Conversation log +19440537.0,8219e6e1-b261-4978-bf78-0e8703a5fcc1,3,ChatGPT,15,118,133,Conversation log +19440574.0,8219e6e1-b261-4978-bf78-0e8703a5fcc1,2,ChatGPT,147,125,272,Conversation log +19440640.0,8219e6e1-b261-4978-bf78-0e8703a5fcc1,3,ChatGPT,291,163,454,Conversation log +19440655.0,8b74c7d8-1643-4bba-8c69-91cb4548c506,3,ChatGPT,1325,152,1477,Conversation log +19440692.0,8b74c7d8-1643-4bba-8c69-91cb4548c506,2,ChatGPT,1221,185,1406,Conversation log +19440735.0,4c5aaf07-bc43-45eb-a3c3-2c77f12f9ee7,3,ChatGPT,18,141,159,Conversation log +19440845.0,de1987f7-d609-4a7c-a4e4-0deeb4ac66b0,3,ChatGPT,21,103,124,Conversation log +19440980.0,de1987f7-d609-4a7c-a4e4-0deeb4ac66b0,3,ChatGPT,137,152,289,Conversation log +19441216.0,9cfed687-bda4-455b-a400-837e08fce913,2,ChatGPT,42,63,105,Conversation log +19441238.0,7117fb3a-4b85-400b-a429-45adad4fce56,2,ChatGPT,66,7,73,Conversation log +19441263.0,7117fb3a-4b85-400b-a429-45adad4fce56,0,ChatGPT,93,6,99,Conversation log +19441654.0,216d88ba-f25e-47d1-a4b0-b9ee6dda5a58,4,ChatGPT,29,339,368,Conversation log +19441735.0,8c89ec7b-cb6f-4121-9704-2837255d9744,25,GPT-4,37,472,509,Conversation log +19442826.0,56e38018-bb73-4f4f-b233-57edbf97d91d,26,GPT-4,5299,541,5840,Conversation log +19443125.0,977bfbdf-a273-469f-a4be-762f5047c843,3,ChatGPT,16,109,125,Conversation log +19443156.0,977bfbdf-a273-469f-a4be-762f5047c843,2,ChatGPT,157,277,434,Conversation log +19443224.0,aa037ed8-0ee9-4567-af72-25c0fcc67d96,9,GPT-4,12,87,99,Conversation log +19443267.0,aa037ed8-0ee9-4567-af72-25c0fcc67d96,44,GPT-4,151,617,768,Conversation log +19443443.0,aa037ed8-0ee9-4567-af72-25c0fcc67d96,40,GPT-4,812,534,1346,Conversation log +19443733.0,aa037ed8-0ee9-4567-af72-25c0fcc67d96,10,GPT-4,1431,98,1529,Conversation log +19443781.0,aa037ed8-0ee9-4567-af72-25c0fcc67d96,6,GPT-4,1609,79,1688,Conversation log +19443876.0,56e38018-bb73-4f4f-b233-57edbf97d91d,18,GPT-4,5866,359,6225,Conversation log +19443878.0,c404e0bd-29b4-4b8e-b61b-02a47cea83a1,5,ChatGPT,42,340,382,Conversation log +19444826.0,a5722132-b491-41c2-9fda-c0e6e0ad009b,23,GPT-4,19,410,429,Conversation log +19445110.0,0c601e0d-7cd3-40a0-8c76-316913fd02d9,25,GPT-4,984,377,1361,Conversation log +19445406.0,0c601e0d-7cd3-40a0-8c76-316913fd02d9,30,GPT-4,1460,259,1719,Conversation log +19447288.0,712de7cf-7c1c-444b-bf68-33408132b420,70,GPT-4,25,879,904,Conversation log +19447641.0,56e38018-bb73-4f4f-b233-57edbf97d91d,25,GPT-4,5805,403,6208,Conversation log +19448735.0,712de7cf-7c1c-444b-bf68-33408132b420,94,GPT-4,930,787,1717,Conversation log +19448801.0,072315f3-c0d9-4be5-94f3-7767861ff34c,111,GPT-4,25,837,862,Conversation log +19448828.0,a3b80941-f129-4f60-a709-aa9ed5d8c7c2,47,GPT-4,31,477,508,Conversation log +19448842.0,38d2f7f8-3384-40cc-a77a-e0a0f7ee80bc,123,GPT-4,35,862,897,Conversation log +19448853.0,f011f24f-692d-4555-bf84-f1caa1f7db50,101,GPT-4,30,711,741,Conversation log +19448987.0,f011f24f-692d-4555-bf84-f1caa1f7db50,4,ChatGPT,772,394,1166,Conversation log +19449072.0,f011f24f-692d-4555-bf84-f1caa1f7db50,65,GPT-4,1197,406,1603,Conversation log +19450093.0,bda1996c-a6b2-43e4-b9bc-cf02ebb47f78,2,ChatGPT,36,34,70,Conversation log +19450216.0,bda1996c-a6b2-43e4-b9bc-cf02ebb47f78,2,ChatGPT,104,26,130,Conversation log +19450676.0,bda1996c-a6b2-43e4-b9bc-cf02ebb47f78,3,ChatGPT,173,33,206,Conversation log +19450738.0,bda1996c-a6b2-43e4-b9bc-cf02ebb47f78,0,ChatGPT,252,37,289,Conversation log +19450780.0,bda1996c-a6b2-43e4-b9bc-cf02ebb47f78,1,ChatGPT,328,28,356,Conversation log +19450849.0,bda1996c-a6b2-43e4-b9bc-cf02ebb47f78,0,ChatGPT,309,15,324,Conversation log +19451325.0,f6afa413-2768-41d4-8377-f1b5ff823b4f,47,GPT-4,38,537,575,Conversation log +19451346.0,0931bfd2-a988-418e-9f84-5e9e13c0bc8e,67,GPT-4,27,645,672,Conversation log +19452654.0,bda1996c-a6b2-43e4-b9bc-cf02ebb47f78,2,ChatGPT,364,76,440,Conversation log +19452763.0,bda1996c-a6b2-43e4-b9bc-cf02ebb47f78,2,ChatGPT,388,9,397,Conversation log +19453134.0,bda1996c-a6b2-43e4-b9bc-cf02ebb47f78,2,ChatGPT,384,61,445,Conversation log +19453146.0,313b905a-2252-45ca-bf14-46e87f6233dd,5,ChatGPT,69,62,131,Conversation log +19454781.0,,21,GPT-4,264,171,435,API log +19454805.0,,38,GPT-4,265,245,510,API log +19454846.0,,14,GPT-4,283,298,581,API log +19456925.0,313b905a-2252-45ca-bf14-46e87f6233dd,3,ChatGPT,249,98,347,Conversation log +19457291.0,313b905a-2252-45ca-bf14-46e87f6233dd,4,ChatGPT,400,45,445,Conversation log +19457368.0,d0c6aa1e-c64e-4919-9395-12c14c2a71b8,2,ChatGPT,247,50,297,Conversation log +19457398.0,d0c6aa1e-c64e-4919-9395-12c14c2a71b8,2,ChatGPT,560,112,672,Conversation log +19457404.0,313b905a-2252-45ca-bf14-46e87f6233dd,3,ChatGPT,498,43,541,Conversation log +19457413.0,d0c6aa1e-c64e-4919-9395-12c14c2a71b8,3,ChatGPT,684,162,846,Conversation log +19457592.0,313b905a-2252-45ca-bf14-46e87f6233dd,3,ChatGPT,707,155,862,Conversation log +19457808.0,d0c6aa1e-c64e-4919-9395-12c14c2a71b8,4,ChatGPT,1240,124,1364,Conversation log +19457830.0,d0c6aa1e-c64e-4919-9395-12c14c2a71b8,1,ChatGPT,1378,91,1469,Conversation log +19460722.0,313b905a-2252-45ca-bf14-46e87f6233dd,2,ChatGPT,763,23,786,Conversation log +19460763.0,313b905a-2252-45ca-bf14-46e87f6233dd,1,ChatGPT,602,23,625,Conversation log +19460840.0,313b905a-2252-45ca-bf14-46e87f6233dd,0,ChatGPT,560,23,583,Conversation log +19460854.0,313b905a-2252-45ca-bf14-46e87f6233dd,3,GPT-4,520,23,543,Conversation log +19461009.0,313b905a-2252-45ca-bf14-46e87f6233dd,5,GPT-4,273,37,310,Conversation log +19461240.0,313b905a-2252-45ca-bf14-46e87f6233dd,3,GPT-4,289,28,317,Conversation log +19461337.0,313b905a-2252-45ca-bf14-46e87f6233dd,3,GPT-4,306,41,347,Conversation log +19461408.0,313b905a-2252-45ca-bf14-46e87f6233dd,4,GPT-4,316,15,331,Conversation log +19461651.0,70d0486e-d937-479b-ab8f-51b9b0e944a8,2,ChatGPT,82,57,139,Conversation log +19461670.0,70d0486e-d937-479b-ab8f-51b9b0e944a8,5,GPT-4,222,60,282,Conversation log +19461704.0,313b905a-2252-45ca-bf14-46e87f6233dd,14,GPT-4,354,69,423,Conversation log +19461751.0,313b905a-2252-45ca-bf14-46e87f6233dd,6,GPT-4,345,78,423,Conversation log +19461797.0,313b905a-2252-45ca-bf14-46e87f6233dd,3,ChatGPT,370,69,439,Conversation log +19461919.0,313b905a-2252-45ca-bf14-46e87f6233dd,2,ChatGPT,439,75,514,Conversation log +19461995.0,313b905a-2252-45ca-bf14-46e87f6233dd,11,GPT-4,484,82,566,Conversation log +19462283.0,313b905a-2252-45ca-bf14-46e87f6233dd,9,GPT-4,471,55,526,Conversation log +19462532.0,313b905a-2252-45ca-bf14-46e87f6233dd,11,GPT-4,495,60,555,Conversation log +19462658.0,313b905a-2252-45ca-bf14-46e87f6233dd,8,GPT-4,527,50,577,Conversation log +19462739.0,313b905a-2252-45ca-bf14-46e87f6233dd,11,GPT-4,427,61,488,Conversation log +19462794.0,313b905a-2252-45ca-bf14-46e87f6233dd,4,GPT-4,452,59,511,Conversation log +19462830.0,,2,ChatGPT,11,77,88,API log +19462868.0,313b905a-2252-45ca-bf14-46e87f6233dd,2,GPT-4,432,24,456,Conversation log +19462991.0,,2,ChatGPT,10,51,61,API log +19463035.0,,1,ChatGPT,11,112,123,API log +19463117.0,c29c0b68-41af-4e5d-bd23-182d153c7026,1,ChatGPT,129,38,167,Conversation log +19463150.0,c29c0b68-41af-4e5d-bd23-182d153c7026,1,ChatGPT,271,89,360,Conversation log +19463175.0,c29c0b68-41af-4e5d-bd23-182d153c7026,0,ChatGPT,376,34,410,Conversation log +19463219.0,c29c0b68-41af-4e5d-bd23-182d153c7026,3,GPT-4,422,51,473,Conversation log +19464436.0,70d0486e-d937-479b-ab8f-51b9b0e944a8,23,GPT-4,498,243,741,Conversation log +19465371.0,70d0486e-d937-479b-ab8f-51b9b0e944a8,14,GPT-4,819,84,903,Conversation log +19465879.0,70d0486e-d937-479b-ab8f-51b9b0e944a8,13,GPT-4,985,88,1073,Conversation log +19466031.0,70d0486e-d937-479b-ab8f-51b9b0e944a8,57,GPT-4,12149,501,12650,Conversation log +19466436.0,70d0486e-d937-479b-ab8f-51b9b0e944a8,101,GPT-4,12570,551,13121,Conversation log +19466694.0,ba445a59-328f-4f81-bc96-b33024b6f2c9,6,GPT-4,192,30,222,Conversation log +19469449.0,70d0486e-d937-479b-ab8f-51b9b0e944a8,52,GPT-4,13103,499,13602,Conversation log +19469801.0,70d0486e-d937-479b-ab8f-51b9b0e944a8,44,GPT-4,25085,288,25373,Conversation log +19470447.0,35cc1b41-1b59-41f0-a2fa-fcec2b634299,4,ChatGPT,439,234,673,Conversation log +19470521.0,5b5b9d0f-6525-4a12-9815-62822f9225a5,2,ChatGPT,69,318,387,Conversation log +19470605.0,5b5b9d0f-6525-4a12-9815-62822f9225a5,3,ChatGPT,504,326,830,Conversation log +19471220.0,e97a0e91-cb95-41ed-a57c-ec06bacfe694,3,ChatGPT,61,56,117,Conversation log +19471695.0,a0a1380b-0e3e-4977-a689-711747ef8a25,50,GPT-4,57,365,422,Conversation log +19471856.0,a0a1380b-0e3e-4977-a689-711747ef8a25,29,GPT-4,461,207,668,Conversation log +19471919.0,c29c0b68-41af-4e5d-bd23-182d153c7026,4,GPT-4,495,86,581,Conversation log +19472459.0,7e7d6f12-3649-4467-8624-c19a2e58aa9d,2,ChatGPT,577,45,622,Conversation log +19472810.0,e365d716-958f-444d-b446-f50b4d1cdf93,3,ChatGPT,198,282,480,Conversation log +19472958.0,10218b75-e89a-4787-84fe-53d7a1058556,5,ChatGPT,22,311,333,Conversation log +19473110.0,8555dc14-21ad-401d-b33c-816cbb086de5,1,ChatGPT,41,18,59,Conversation log +19473192.0,49d76cf0-2d00-4308-a798-3970f3b93f4f,2,GPT-4,38,15,53,Conversation log +19473219.0,7afcfc41-af8f-49cb-8cda-86acbdbb4ab6,2,ChatGPT,167,143,310,Conversation log +19473253.0,5433ddce-0bb7-4ffc-ab98-14fa1aaec544,0,ChatGPT,0,0,0,Conversation log +19473501.0,e70a0be6-879e-4079-a006-3584f43f23a1,2,ChatGPT,101,101,202,Conversation log +19473786.0,7e7d6f12-3649-4467-8624-c19a2e58aa9d,4,ChatGPT,599,107,706,Conversation log +19473831.0,7599245b-e0bf-49ad-b5d3-77148ccc9bc4,6,ChatGPT,363,536,899,Conversation log +19474039.0,bb0e0d1b-ca5d-49a3-999c-2cb33e7d5b20,6,GPT-4,64,59,123,Conversation log +19474118.0,96056b5c-31d5-4c74-9a3b-9c9b9050f87d,3,ChatGPT,58,142,200,Conversation log +19474155.0,2e0a0c9b-32a0-4ba8-b52f-6779cf9b9ad2,1,ChatGPT,40,29,69,Conversation log +19474230.0,b0410482-18b7-40b6-923e-b878361f2802,2,ChatGPT,20,138,158,Conversation log +19474307.0,b0410482-18b7-40b6-923e-b878361f2802,6,ChatGPT,178,296,474,Conversation log +19474352.0,b0410482-18b7-40b6-923e-b878361f2802,1,ChatGPT,500,156,656,Conversation log +19474363.0,8d3d977a-8acc-431e-a304-646f7ac6b561,2,ChatGPT,61,36,97,Conversation log +19474368.0,b0410482-18b7-40b6-923e-b878361f2802,0,ChatGPT,676,33,709,Conversation log +19474383.0,b0410482-18b7-40b6-923e-b878361f2802,4,ChatGPT,724,342,1066,Conversation log +19474413.0,2e0a0c9b-32a0-4ba8-b52f-6779cf9b9ad2,0,ChatGPT,96,14,110,Conversation log +19474481.0,2e0a0c9b-32a0-4ba8-b52f-6779cf9b9ad2,1,ChatGPT,142,18,160,Conversation log +19474499.0,2e0a0c9b-32a0-4ba8-b52f-6779cf9b9ad2,1,ChatGPT,178,4,182,Conversation log +19474507.0,2e0a0c9b-32a0-4ba8-b52f-6779cf9b9ad2,0,ChatGPT,206,7,213,Conversation log +19474576.0,2e0a0c9b-32a0-4ba8-b52f-6779cf9b9ad2,2,ChatGPT,190,29,219,Conversation log +19474622.0,859aa400-b904-4596-a781-1629f37437d7,1,ChatGPT,49,45,94,Conversation log +19474629.0,95ed3640-36ab-4253-9fad-4afde1b9d00a,145,GPT-4,829,843,1672,Conversation log +19474635.0,23ca3716-d01f-4440-9290-ba9c06c4e4fe,3,ChatGPT,15,101,116,Conversation log +19474640.0,23ca3716-d01f-4440-9290-ba9c06c4e4fe,3,ChatGPT,126,192,318,Conversation log +19474670.0,2e0a0c9b-32a0-4ba8-b52f-6779cf9b9ad2,3,ChatGPT,308,126,434,Conversation log +19474711.0,e1cd025b-2c39-465c-9317-50b5e9325ece,5,ChatGPT,17,420,437,Conversation log +19474721.0,e1cd025b-2c39-465c-9317-50b5e9325ece,5,ChatGPT,456,413,869,Conversation log +19474783.0,23ca3716-d01f-4440-9290-ba9c06c4e4fe,3,ChatGPT,349,189,538,Conversation log +19474836.0,859aa400-b904-4596-a781-1629f37437d7,2,ChatGPT,137,27,164,Conversation log +19474918.0,88941ff8-fdf7-4599-b5f8-fcee623ef50f,4,ChatGPT,30,186,216,Conversation log +19474920.0,7afcfc41-af8f-49cb-8cda-86acbdbb4ab6,3,ChatGPT,596,271,867,Conversation log +19474942.0,88941ff8-fdf7-4599-b5f8-fcee623ef50f,1,ChatGPT,247,120,367,Conversation log +19474983.0,88941ff8-fdf7-4599-b5f8-fcee623ef50f,2,ChatGPT,399,165,564,Conversation log +19474997.0,9286be0f-c848-46af-bcf1-f2ce2c1916b1,2,ChatGPT,30,19,49,Conversation log +19475017.0,88941ff8-fdf7-4599-b5f8-fcee623ef50f,2,ChatGPT,591,185,776,Conversation log +19475019.0,7e7d6f12-3649-4467-8624-c19a2e58aa9d,2,ChatGPT,610,115,725,Conversation log +19475022.0,b29284f7-349c-4002-9c4d-e40f11d1b634,0,ChatGPT,13,3,16,Conversation log +19475062.0,1a311690-1fd2-449d-9833-dbc9444275f0,0,ChatGPT,53,57,110,Conversation log +19475129.0,4ee90b9f-41f6-474b-908c-f4204b50d952,23,GPT-4,1417,515,1932,Conversation log +19475163.0,b3e86cba-e38d-4bb0-a797-e1a270aa6772,1,ChatGPT,30,7,37,Conversation log +19475174.0,00ecf1a2-1bc8-4078-9480-11cce1395f72,2,ChatGPT,24,161,185,Conversation log +19475218.0,6dce0737-5e96-4efe-81cd-7342ee46de64,2,ChatGPT,91,81,172,Conversation log +19475254.0,021a5e07-4d20-4615-9855-229182369c8f,2,ChatGPT,171,153,324,Conversation log +19475307.0,b3e86cba-e38d-4bb0-a797-e1a270aa6772,1,ChatGPT,69,9,78,Conversation log +19475357.0,6338841e-ad62-4ef0-b897-51246061ea19,8,ChatGPT,22,741,763,Conversation log +19475409.0,6dce0737-5e96-4efe-81cd-7342ee46de64,0,ChatGPT,248,29,277,Conversation log +19475452.0,c29c0b68-41af-4e5d-bd23-182d153c7026,8,GPT-4,439,167,606,Conversation log +19475456.0,fa1adb6b-a9a3-49ef-b87b-e13d7d6d1c9a,2,ChatGPT,75,88,163,Conversation log +19475457.0,90d7608a-6f5e-4d49-bd87-06a323bad8bf,0,ChatGPT,93,26,119,Conversation log +19475473.0,3650ec27-6d6a-4492-923b-f8c5b8625b35,0,ChatGPT,42,69,111,Conversation log +19475474.0,3639b4da-8871-4b98-b0cb-365f227d68a3,3,ChatGPT,52,194,246,Conversation log +19475481.0,6dce0737-5e96-4efe-81cd-7342ee46de64,0,ChatGPT,295,2,297,Conversation log +19475490.0,6dce0737-5e96-4efe-81cd-7342ee46de64,0,ChatGPT,314,3,317,Conversation log +19475521.0,c29c0b68-41af-4e5d-bd23-182d153c7026,2,GPT-4,434,58,492,Conversation log +19475609.0,021a5e07-4d20-4615-9855-229182369c8f,4,ChatGPT,563,220,783,Conversation log +19475685.0,e48d6c1f-787e-4d42-ab0d-f095302946aa,0,ChatGPT,49,39,88,Conversation log +19475689.0,021a5e07-4d20-4615-9855-229182369c8f,4,ChatGPT,834,402,1236,Conversation log +19475708.0,e70a0be6-879e-4079-a006-3584f43f23a1,2,ChatGPT,279,104,383,Conversation log +19475729.0,fa1adb6b-a9a3-49ef-b87b-e13d7d6d1c9a,2,ChatGPT,184,93,277,Conversation log +19475773.0,e48d6c1f-787e-4d42-ab0d-f095302946aa,2,ChatGPT,155,59,214,Conversation log +19475796.0,fa1adb6b-a9a3-49ef-b87b-e13d7d6d1c9a,0,ChatGPT,286,5,291,Conversation log +19475839.0,fa1adb6b-a9a3-49ef-b87b-e13d7d6d1c9a,1,ChatGPT,312,10,322,Conversation log +19475859.0,6dce0737-5e96-4efe-81cd-7342ee46de64,2,ChatGPT,352,31,383,Conversation log +19475906.0,b3e86cba-e38d-4bb0-a797-e1a270aa6772,2,ChatGPT,192,104,296,Conversation log +19475917.0,3650ec27-6d6a-4492-923b-f8c5b8625b35,5,ChatGPT,164,213,377,Conversation log +19475993.0,6338841e-ad62-4ef0-b897-51246061ea19,12,ChatGPT,788,1101,1889,Conversation log +19476074.0,926c155f-6cbc-4404-8848-55b7a859a54e,5,ChatGPT,22,5,27,Conversation log +19476077.0,021a5e07-4d20-4615-9855-229182369c8f,4,ChatGPT,1336,78,1414,Conversation log +19476106.0,b29284f7-349c-4002-9c4d-e40f11d1b634,3,ChatGPT,65,38,103,Conversation log +19476180.0,3dbd1af9-4e46-413d-b63c-ae52c55ec66e,155,GPT-4,22,757,779,Conversation log +19476193.0,1e3e8e87-3c87-4600-8ea8-794aabf3d198,3,ChatGPT,25,212,237,Conversation log +19476213.0,95ed3640-36ab-4253-9fad-4afde1b9d00a,108,GPT-4,2457,558,3015,Conversation log +19476216.0,9286be0f-c848-46af-bcf1-f2ce2c1916b1,6,ChatGPT,108,46,154,Conversation log +19476296.0,021a5e07-4d20-4615-9855-229182369c8f,4,ChatGPT,1515,84,1599,Conversation log +19476299.0,00ecf1a2-1bc8-4078-9480-11cce1395f72,2,ChatGPT,207,144,351,Conversation log +19476304.0,0f284f8b-add5-493e-8fda-a57620a580ee,4,ChatGPT,75,69,144,Conversation log +19476323.0,926c155f-6cbc-4404-8848-55b7a859a54e,5,ChatGPT,160,106,266,Conversation log +19476327.0,0f284f8b-add5-493e-8fda-a57620a580ee,1,ChatGPT,182,67,249,Conversation log +19476332.0,a76bb823-3f64-4b7a-8a15-f4eb1253ad7f,3,ChatGPT,471,46,517,Conversation log +19476342.0,0f284f8b-add5-493e-8fda-a57620a580ee,1,ChatGPT,269,44,313,Conversation log +19476366.0,0f284f8b-add5-493e-8fda-a57620a580ee,8,GPT-4,328,80,408,Conversation log +19476371.0,a76bb823-3f64-4b7a-8a15-f4eb1253ad7f,1,ChatGPT,482,26,508,Conversation log +19476481.0,e70a0be6-879e-4079-a006-3584f43f23a1,3,ChatGPT,516,140,656,Conversation log +19476485.0,021a5e07-4d20-4615-9855-229182369c8f,3,ChatGPT,1499,191,1690,Conversation log +19476490.0,00ecf1a2-1bc8-4078-9480-11cce1395f72,4,ChatGPT,370,216,586,Conversation log +19476519.0,90d7608a-6f5e-4d49-bd87-06a323bad8bf,1,ChatGPT,143,15,158,Conversation log +19476557.0,2f9dc6e6-62f8-47ed-bf3c-b55ea0219207,3,ChatGPT,14,221,235,Conversation log +19476581.0,90d7608a-6f5e-4d49-bd87-06a323bad8bf,0,ChatGPT,177,8,185,Conversation log +19476611.0,e7ab5f81-cbef-4543-94fc-226946f1fe0a,58,GPT-4,35,389,424,Conversation log +19476629.0,0a9088f6-3cd2-48fb-85df-d9b5525b812d,2,ChatGPT,116,111,227,Conversation log +19476631.0,6552716e-d54c-485e-9edc-ff9a4baf3273,1,ChatGPT,40,20,60,Conversation log +19476652.0,6552716e-d54c-485e-9edc-ff9a4baf3273,1,ChatGPT,97,22,119,Conversation log +19476659.0,84b8a44a-4b64-4ab6-b784-01a53f7c1126,18,GPT-4,417,48,465,Conversation log +19476680.0,1e3e8e87-3c87-4600-8ea8-794aabf3d198,2,ChatGPT,256,178,434,Conversation log +19476680.0,2f9dc6e6-62f8-47ed-bf3c-b55ea0219207,3,ChatGPT,257,288,545,Conversation log +19476752.0,021a5e07-4d20-4615-9855-229182369c8f,3,ChatGPT,1404,133,1537,Conversation log +19476766.0,021a5e07-4d20-4615-9855-229182369c8f,2,ChatGPT,1094,191,1285,Conversation log +19476789.0,6552716e-d54c-485e-9edc-ff9a4baf3273,2,ChatGPT,161,31,192,Conversation log +19476826.0,021a5e07-4d20-4615-9855-229182369c8f,1,ChatGPT,1134,136,1270,Conversation log +19476858.0,3f9670a6-3e3d-44b4-b3f0-ac1742d3843e,1,ChatGPT,33,34,67,Conversation log +19476868.0,3f9670a6-3e3d-44b4-b3f0-ac1742d3843e,3,ChatGPT,159,113,272,Conversation log +19476897.0,2f9dc6e6-62f8-47ed-bf3c-b55ea0219207,3,ChatGPT,571,113,684,Conversation log +19476930.0,2f9dc6e6-62f8-47ed-bf3c-b55ea0219207,2,ChatGPT,718,140,858,Conversation log +19476960.0,68201869-131c-4de7-b428-7cc34ca426cf,1,ChatGPT,67,55,122,Conversation log +19476997.0,2f9dc6e6-62f8-47ed-bf3c-b55ea0219207,1,ChatGPT,885,15,900,Conversation log +19477082.0,9286be0f-c848-46af-bcf1-f2ce2c1916b1,6,ChatGPT,276,114,390,Conversation log +19477093.0,6552716e-d54c-485e-9edc-ff9a4baf3273,2,ChatGPT,308,103,411,Conversation log +19477103.0,68201869-131c-4de7-b428-7cc34ca426cf,4,ChatGPT,196,35,231,Conversation log +19477153.0,6552716e-d54c-485e-9edc-ff9a4baf3273,2,ChatGPT,533,160,693,Conversation log +19477185.0,1ad91820-0b5f-437b-8e5f-53d86ba39fab,5,ChatGPT,715,435,1150,Conversation log +19477185.0,1a311690-1fd2-449d-9833-dbc9444275f0,3,ChatGPT,216,91,307,Conversation log +19477188.0,210705c1-e47f-42fe-80d8-239f78163ab0,60,GPT-4,15,524,539,Conversation log +19477195.0,0f9ad46b-9f44-4b06-bbb8-1c0fc44591a1,2,ChatGPT,63,36,99,Conversation log +19477207.0,a88af531-4376-4743-9a88-dc78f8b6b901,102,GPT-4,24,760,784,Conversation log +19477223.0,68201869-131c-4de7-b428-7cc34ca426cf,3,ChatGPT,298,47,345,Conversation log +19477281.0,68201869-131c-4de7-b428-7cc34ca426cf,2,ChatGPT,447,86,533,Conversation log +19477334.0,80676dea-b9d2-4347-ae11-fb3008926e06,3,ChatGPT,24,143,167,Conversation log +19477338.0,68201869-131c-4de7-b428-7cc34ca426cf,2,ChatGPT,674,134,808,Conversation log +19477358.0,80676dea-b9d2-4347-ae11-fb3008926e06,1,ChatGPT,180,55,235,Conversation log +19477402.0,701b2d51-cd8d-49e5-a5d8-ff3598cb49c7,4,ChatGPT,163,201,364,Conversation log +19477430.0,f93dadc5-1806-46e3-90f3-c53a79e64fc5,2,ChatGPT,86,56,142,Conversation log +19477458.0,1ad91820-0b5f-437b-8e5f-53d86ba39fab,6,ChatGPT,1272,292,1564,Conversation log +19477574.0,9286be0f-c848-46af-bcf1-f2ce2c1916b1,4,ChatGPT,675,277,952,Conversation log +19477599.0,9286be0f-c848-46af-bcf1-f2ce2c1916b1,5,ChatGPT,1241,235,1476,Conversation log +19477626.0,0f284f8b-add5-493e-8fda-a57620a580ee,30,GPT-4,432,15,447,Conversation log +19477736.0,bd73fbc3-25f5-4e8a-8efa-a63bfbdd7cf3,7,ChatGPT,573,635,1208,Conversation log +19477766.0,80676dea-b9d2-4347-ae11-fb3008926e06,1,ChatGPT,251,146,397,Conversation log +19477772.0,efe52725-b30d-4de8-8c73-56ad121666f0,2,ChatGPT,132,192,324,Conversation log +19477788.0,80676dea-b9d2-4347-ae11-fb3008926e06,3,ChatGPT,421,146,567,Conversation log +19477817.0,181e8a96-674e-44cd-b54d-78dea1b504c7,9,ChatGPT,282,258,540,Conversation log +19477869.0,efe52725-b30d-4de8-8c73-56ad121666f0,0,ChatGPT,348,13,361,Conversation log +19477881.0,b5defce4-fa6b-4b59-9615-bcc1f271e456,11,ChatGPT,84,495,579,Conversation log +19477941.0,efe52725-b30d-4de8-8c73-56ad121666f0,0,ChatGPT,380,11,391,Conversation log +19477964.0,b5defce4-fa6b-4b59-9615-bcc1f271e456,5,ChatGPT,684,418,1102,Conversation log +19477975.0,80676dea-b9d2-4347-ae11-fb3008926e06,3,ChatGPT,585,239,824,Conversation log +19477984.0,efe52725-b30d-4de8-8c73-56ad121666f0,1,ChatGPT,418,17,435,Conversation log +19478156.0,68201869-131c-4de7-b428-7cc34ca426cf,3,ChatGPT,790,77,867,Conversation log +19478181.0,68201869-131c-4de7-b428-7cc34ca426cf,2,ChatGPT,773,74,847,Conversation log +19478184.0,b5defce4-fa6b-4b59-9615-bcc1f271e456,9,ChatGPT,1133,340,1473,Conversation log +19478185.0,c29c0b68-41af-4e5d-bd23-182d153c7026,11,GPT-4,561,125,686,Conversation log +19478186.0,b0d071ef-8d89-47e2-9d3e-6561c8d34b0c,5,ChatGPT,128,158,286,Conversation log +19478223.0,f5c2c846-16b3-4610-978b-5783f80efb48,9,ChatGPT,401,694,1095,Conversation log +19478277.0,c29c0b68-41af-4e5d-bd23-182d153c7026,3,GPT-4,646,25,671,Conversation log +19478282.0,c29c0b68-41af-4e5d-bd23-182d153c7026,7,GPT-4,717,131,848,Conversation log +19478341.0,efe52725-b30d-4de8-8c73-56ad121666f0,2,ChatGPT,464,17,481,Conversation log +19478360.0,b0d071ef-8d89-47e2-9d3e-6561c8d34b0c,1,ChatGPT,341,24,365,Conversation log +19478364.0,c29c0b68-41af-4e5d-bd23-182d153c7026,33,GPT-4,999,316,1315,Conversation log +19478373.0,7e7d6f12-3649-4467-8624-c19a2e58aa9d,0,ChatGPT,738,48,786,Conversation log +19478411.0,68201869-131c-4de7-b428-7cc34ca426cf,2,ChatGPT,798,57,855,Conversation log +19478436.0,3c165d0f-0118-449a-94d5-ec59cb721be5,3,ChatGPT,794,89,883,Conversation log +19478527.0,3c165d0f-0118-449a-94d5-ec59cb721be5,1,ChatGPT,933,96,1029,Conversation log +19478580.0,3f9670a6-3e3d-44b4-b3f0-ac1742d3843e,4,ChatGPT,379,131,510,Conversation log +19478585.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,2,ChatGPT,82,46,128,Conversation log +19478600.0,a9e07ba1-e7c1-4360-a774-6d684273d5b5,1,ChatGPT,32,18,50,Conversation log +19478619.0,efe52725-b30d-4de8-8c73-56ad121666f0,2,ChatGPT,236,71,307,Conversation log +19478624.0,c29c0b68-41af-4e5d-bd23-182d153c7026,22,GPT-4,1581,337,1918,Conversation log +19478631.0,3d1ad40d-66cf-4b64-8c38-2a12326abadb,3,ChatGPT,15,328,343,Conversation log +19478636.0,b834913c-a77c-4360-8c3e-bba69ea65374,0,ChatGPT,79,63,142,Conversation log +19478637.0,3c165d0f-0118-449a-94d5-ec59cb721be5,3,ChatGPT,1078,167,1245,Conversation log +19478647.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,1,ChatGPT,191,34,225,Conversation log +19478699.0,d921e3c1-8cea-4faa-9c44-ea5557fbc116,4,ChatGPT,27,382,409,Conversation log +19478713.0,64fc0e1a-1001-4467-bc1b-5012c5eea2bf,45,GPT-4,26,553,579,Conversation log +19478717.0,1ad91820-0b5f-437b-8e5f-53d86ba39fab,5,ChatGPT,1600,471,2071,Conversation log +19478717.0,3f9670a6-3e3d-44b4-b3f0-ac1742d3843e,3,ChatGPT,528,106,634,Conversation log +19478757.0,b834913c-a77c-4360-8c3e-bba69ea65374,1,ChatGPT,220,68,288,Conversation log +19478778.0,64fc0e1a-1001-4467-bc1b-5012c5eea2bf,47,GPT-4,592,502,1094,Conversation log +19478796.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,2,ChatGPT,380,104,484,Conversation log +19478817.0,bba5d06c-a6ce-4b6e-94db-a2022253e428,1,ChatGPT,92,59,151,Conversation log +19478820.0,1ad91820-0b5f-437b-8e5f-53d86ba39fab,5,ChatGPT,2121,552,2673,Conversation log +19478837.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,1,ChatGPT,498,57,555,Conversation log +19478853.0,1a311690-1fd2-449d-9833-dbc9444275f0,4,ChatGPT,366,55,421,Conversation log +19478886.0,926c155f-6cbc-4404-8848-55b7a859a54e,0,ChatGPT,304,16,320,Conversation log +19478889.0,5589009a-91d4-401f-8de6-e9848eb39c03,5,ChatGPT,189,281,470,Conversation log +19478896.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,2,ChatGPT,572,103,675,Conversation log +19478920.0,68201869-131c-4de7-b428-7cc34ca426cf,2,ChatGPT,852,116,968,Conversation log +19478929.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,1,ChatGPT,565,100,665,Conversation log +19478949.0,7345154f-a066-4d61-8bb4-1a1879d1aa52,1,ChatGPT,149,87,236,Conversation log +19478975.0,0f284f8b-add5-493e-8fda-a57620a580ee,34,GPT-4,563,214,777,Conversation log +19478977.0,bba5d06c-a6ce-4b6e-94db-a2022253e428,0,ChatGPT,163,8,171,Conversation log +19479005.0,bba5d06c-a6ce-4b6e-94db-a2022253e428,1,ChatGPT,184,6,190,Conversation log +19479020.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,1,ChatGPT,653,57,710,Conversation log +19479029.0,35793c3d-a428-48af-b670-3be233737e09,1,ChatGPT,8,9,17,Conversation log +19479030.0,f93dadc5-1806-46e3-90f3-c53a79e64fc5,3,ChatGPT,169,14,183,Conversation log +19479031.0,68201869-131c-4de7-b428-7cc34ca426cf,4,ChatGPT,774,161,935,Conversation log +19479038.0,0f284f8b-add5-493e-8fda-a57620a580ee,50,GPT-4,682,297,979,Conversation log +19479045.0,f93dadc5-1806-46e3-90f3-c53a79e64fc5,1,ChatGPT,201,13,214,Conversation log +19479084.0,68201869-131c-4de7-b428-7cc34ca426cf,1,ChatGPT,930,77,1007,Conversation log +19479086.0,f93dadc5-1806-46e3-90f3-c53a79e64fc5,1,ChatGPT,235,13,248,Conversation log +19479109.0,0524d64c-d5fa-4387-bc4a-e379438ede44,5,ChatGPT,20,339,359,Conversation log +19479110.0,181e8a96-674e-44cd-b54d-78dea1b504c7,2,ChatGPT,609,47,656,Conversation log +19479113.0,f5c2c846-16b3-4610-978b-5783f80efb48,1,ChatGPT,1195,181,1376,Conversation log +19479127.0,3f9670a6-3e3d-44b4-b3f0-ac1742d3843e,1,ChatGPT,669,53,722,Conversation log +19479143.0,35793c3d-a428-48af-b670-3be233737e09,0,ChatGPT,46,16,62,Conversation log +19479182.0,68201869-131c-4de7-b428-7cc34ca426cf,8,ChatGPT,936,157,1093,Conversation log +19479237.0,60cf96c3-b9a3-425d-b8aa-5cb52a83e94c,5,ChatGPT,37,294,331,Conversation log +19479238.0,0f2378f5-84c3-48fe-9355-fb536585b45d,68,GPT-4,12,776,788,Conversation log +19479258.0,35793c3d-a428-48af-b670-3be233737e09,1,ChatGPT,75,15,90,Conversation log +19479271.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,2,ChatGPT,467,54,521,Conversation log +19479280.0,35793c3d-a428-48af-b670-3be233737e09,1,ChatGPT,105,15,120,Conversation log +19479288.0,e4fa8c3a-fe67-44c6-b021-1e0ce99809b9,3,ChatGPT,63,324,387,Conversation log +19479305.0,e4fa8c3a-fe67-44c6-b021-1e0ce99809b9,4,ChatGPT,453,233,686,Conversation log +19479343.0,bd73fbc3-25f5-4e8a-8efa-a63bfbdd7cf3,9,ChatGPT,1234,670,1904,Conversation log +19479381.0,6a13d58e-49da-49b5-829b-ab764f010d6f,3,ChatGPT,22,221,243,Conversation log +19479424.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,1,ChatGPT,467,54,521,Conversation log +19479433.0,6a13d58e-49da-49b5-829b-ab764f010d6f,2,ChatGPT,267,202,469,Conversation log +19479437.0,84b8a44a-4b64-4ab6-b784-01a53f7c1126,8,GPT-4,519,50,569,Conversation log +19479458.0,0f284f8b-add5-493e-8fda-a57620a580ee,16,GPT-4,943,22,965,Conversation log +19479460.0,bd73fbc3-25f5-4e8a-8efa-a63bfbdd7cf3,5,ChatGPT,1927,480,2407,Conversation log +19479474.0,0a4b4191-e4f5-4fde-9b54-9a4b9d0dca25,2,ChatGPT,56,123,179,Conversation log +19479517.0,6a13d58e-49da-49b5-829b-ab764f010d6f,2,ChatGPT,493,200,693,Conversation log +19479542.0,0f284f8b-add5-493e-8fda-a57620a580ee,40,GPT-4,1096,191,1287,Conversation log +19479544.0,6a13d58e-49da-49b5-829b-ab764f010d6f,2,ChatGPT,722,181,903,Conversation log +19479562.0,bcf6c528-e342-421a-b04e-a1991e272945,2,ChatGPT,21,10,31,Conversation log +19479564.0,b5defce4-fa6b-4b59-9615-bcc1f271e456,5,ChatGPT,1497,315,1812,Conversation log +19479577.0,11c0c733-3220-4deb-b5fd-d6366fe54bef,3,ChatGPT,28,272,300,Conversation log +19479584.0,ed956d87-4593-41ff-9270-37c02b668487,14,ChatGPT,1552,928,2480,Conversation log +19479596.0,48ebee3e-57a7-4411-aa64-309820af4510,2,ChatGPT,121,94,215,Conversation log +19479601.0,8f5263df-e41d-4360-8f9a-7423cd08950e,1,ChatGPT,167,96,263,Conversation log +19479633.0,11c0c733-3220-4deb-b5fd-d6366fe54bef,46,GPT-4,334,326,660,Conversation log +19479644.0,a9e07ba1-e7c1-4360-a774-6d684273d5b5,2,ChatGPT,116,66,182,Conversation log +19479647.0,160dfb66-bc81-4228-9902-cdf0203d0868,6,ChatGPT,19,297,316,Conversation log +19479648.0,6a13d58e-49da-49b5-829b-ab764f010d6f,3,ChatGPT,945,185,1130,Conversation log +19479670.0,0f284f8b-add5-493e-8fda-a57620a580ee,54,GPT-4,1271,195,1466,Conversation log +19479674.0,3dda1072-611f-4497-8afa-0cd4e30553fb,3,ChatGPT,23,331,354,Conversation log +19479725.0,7345154f-a066-4d61-8bb4-1a1879d1aa52,12,GPT-4,353,70,423,Conversation log +19479810.0,bd73fbc3-25f5-4e8a-8efa-a63bfbdd7cf3,3,ChatGPT,2573,159,2732,Conversation log +19479937.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,2,ChatGPT,462,29,491,Conversation log +19479963.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,0,ChatGPT,394,33,427,Conversation log +19479982.0,3dda1072-611f-4497-8afa-0cd4e30553fb,7,ChatGPT,366,385,751,Conversation log +19480016.0,bba5d06c-a6ce-4b6e-94db-a2022253e428,2,ChatGPT,264,41,305,Conversation log +19480116.0,8942a261-5744-4644-b6c3-b7cf3684bb89,4,ChatGPT,130,171,301,Conversation log +19480120.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,2,ChatGPT,363,49,412,Conversation log +19480183.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,1,ChatGPT,412,42,454,Conversation log +19480201.0,014d885c-28f4-46ac-853f-2dd376471a01,21,GPT-4,95,301,396,Conversation log +19480211.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,1,ChatGPT,456,45,501,Conversation log +19480244.0,75456ff6-769e-4f64-b987-4427f1acb11c,1,ChatGPT,25,15,40,Conversation log +19480245.0,f08fcfa0-365c-487d-a022-3ddbac088f33,3,ChatGPT,36,14,50,Conversation log +19480361.0,b9811a13-a9ab-4dcb-bd09-71561c50c901,5,ChatGPT,22,275,297,Conversation log +19480369.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,2,ChatGPT,498,56,554,Conversation log +19480389.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,2,ChatGPT,586,56,642,Conversation log +19480409.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,2,ChatGPT,530,42,572,Conversation log +19480480.0,75456ff6-769e-4f64-b987-4427f1acb11c,2,ChatGPT,81,29,110,Conversation log +19480521.0,053d1e8d-3601-4060-814d-44c056aa1df6,1,ChatGPT,81,77,158,Conversation log +19480534.0,053d1e8d-3601-4060-814d-44c056aa1df6,8,GPT-4,240,79,319,Conversation log +19480555.0,90c02ac1-d3fb-46b6-a7e3-cc4bc0f6cfe3,2,ChatGPT,62,106,168,Conversation log +19480590.0,27746ab4-f311-4cd3-8ef1-839cc0b7e8c4,13,ChatGPT,19,913,932,Conversation log +19480688.0,90c02ac1-d3fb-46b6-a7e3-cc4bc0f6cfe3,5,ChatGPT,200,326,526,Conversation log +19480702.0,90c02ac1-d3fb-46b6-a7e3-cc4bc0f6cfe3,3,ChatGPT,539,197,736,Conversation log +19480717.0,90c02ac1-d3fb-46b6-a7e3-cc4bc0f6cfe3,3,ChatGPT,750,203,953,Conversation log +19480744.0,a0a1380b-0e3e-4977-a689-711747ef8a25,24,GPT-4,694,92,786,Conversation log +19480786.0,0f284f8b-add5-493e-8fda-a57620a580ee,11,GPT-4,1064,44,1108,Conversation log +19480806.0,0f284f8b-add5-493e-8fda-a57620a580ee,12,GPT-4,825,61,886,Conversation log +19480904.0,bc2666a9-8581-4b83-9841-20a0b5a6cf7e,2,ChatGPT,104,53,157,Conversation log +19480949.0,cafa6119-e832-4ed5-834b-049c9ab362b0,2,ChatGPT,22,112,134,Conversation log +19480962.0,b9811a13-a9ab-4dcb-bd09-71561c50c901,7,ChatGPT,318,280,598,Conversation log +19480963.0,7afcfc41-af8f-49cb-8cda-86acbdbb4ab6,3,ChatGPT,950,81,1031,Conversation log +19480973.0,021a5e07-4d20-4615-9855-229182369c8f,2,ChatGPT,1164,146,1310,Conversation log +19480986.0,021a5e07-4d20-4615-9855-229182369c8f,1,ChatGPT,906,115,1021,Conversation log +19480994.0,9286be0f-c848-46af-bcf1-f2ce2c1916b1,2,ChatGPT,1477,35,1512,Conversation log +19480994.0,021a5e07-4d20-4615-9855-229182369c8f,2,ChatGPT,726,56,782,Conversation log +19481010.0,5a98fdec-ce4d-4b14-9506-3a5fbbd15c8b,4,ChatGPT,20,209,229,Conversation log +19481059.0,7f0d0ece-3f46-4e9c-a8ae-db8122db4a7b,5,ChatGPT,16,147,163,Conversation log +19481064.0,00ecf1a2-1bc8-4078-9480-11cce1395f72,59,GPT-4,614,452,1066,Conversation log +19481091.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,4,ChatGPT,526,45,571,Conversation log +19481109.0,cafa6119-e832-4ed5-834b-049c9ab362b0,1,ChatGPT,165,73,238,Conversation log +19481113.0,9ca5e263-a763-47a5-992a-f939b5e73a4d,6,GPT-4,93,42,135,Conversation log +19481123.0,1b78554b-528f-4cea-b8f2-235b32b2166d,3,ChatGPT,31,216,247,Conversation log +19481147.0,8f6cae9b-7fd5-48eb-a6f5-45934878a327,3,ChatGPT,17,213,230,Conversation log +19481191.0,cafa6119-e832-4ed5-834b-049c9ab362b0,3,ChatGPT,286,168,454,Conversation log +19481204.0,9ca5e263-a763-47a5-992a-f939b5e73a4d,8,GPT-4,165,116,281,Conversation log +19481210.0,b9811a13-a9ab-4dcb-bd09-71561c50c901,2,ChatGPT,673,48,721,Conversation log +19481223.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,2,ChatGPT,472,36,508,Conversation log +19481238.0,cafa6119-e832-4ed5-834b-049c9ab362b0,4,ChatGPT,472,261,733,Conversation log +19481239.0,101b03d1-c3e5-4030-9bd6-e26cc0eebaf0,2,ChatGPT,431,46,477,Conversation log +19481289.0,021a5e07-4d20-4615-9855-229182369c8f,3,ChatGPT,658,48,706,Conversation log +19481302.0,bba5d06c-a6ce-4b6e-94db-a2022253e428,0,ChatGPT,341,19,360,Conversation log +19481328.0,9ca5e263-a763-47a5-992a-f939b5e73a4d,2,GPT-4,304,5,309,Conversation log +19481337.0,bba5d06c-a6ce-4b6e-94db-a2022253e428,1,ChatGPT,246,34,280,Conversation log +19481366.0,469e228e-3e34-4311-b5af-2568f8e954b1,60,GPT-4,20,594,614,Conversation log +19481418.0,9286be0f-c848-46af-bcf1-f2ce2c1916b1,2,ChatGPT,1505,27,1532,Conversation log +19481424.0,9286be0f-c848-46af-bcf1-f2ce2c1916b1,1,ChatGPT,1309,15,1324,Conversation log +19481433.0,9286be0f-c848-46af-bcf1-f2ce2c1916b1,1,ChatGPT,868,68,936,Conversation log +19481442.0,355868d5-7817-4035-9fb3-97102e8433ec,3,ChatGPT,49,152,201,Conversation log +19481513.0,07fba814-8468-408d-9823-60c86fb9c385,2,ChatGPT,116,176,292,Conversation log +19481540.0,9286be0f-c848-46af-bcf1-f2ce2c1916b1,1,ChatGPT,425,55,480,Conversation log +19481552.0,9286be0f-c848-46af-bcf1-f2ce2c1916b1,1,ChatGPT,458,42,500,Conversation log +19481563.0,0f284f8b-add5-493e-8fda-a57620a580ee,3,GPT-4,870,11,881,Conversation log diff --git a/benchmarks/traffic_replay/fixtures/burstgpt_dirty.csv b/benchmarks/traffic_replay/fixtures/burstgpt_dirty.csv new file mode 100644 index 0000000..bcdcd3c --- /dev/null +++ b/benchmarks/traffic_replay/fixtures/burstgpt_dirty.csv @@ -0,0 +1,10 @@ +Timestamp,Model,Request tokens,Response tokens,Total tokens,Log Type +10,ChatGPT,100,50,150,Conversation log +20,ChatGPT,0,0,0,Conversation log +30,ChatGPT,120,0,120,Conversation log +40,ChatGPT,-5,10,5,Conversation log +50,ChatGPT,abc,10,10,Conversation log +55,ChatGPT,,50,50,Conversation log +60,ChatGPT,100 +45,ChatGPT,100,50,150,Conversation log +70,GPT-4,200,100,300,API log diff --git a/benchmarks/traffic_replay/fixtures/burstgpt_slice.csv b/benchmarks/traffic_replay/fixtures/burstgpt_slice.csv new file mode 100644 index 0000000..9ead040 --- /dev/null +++ b/benchmarks/traffic_replay/fixtures/burstgpt_slice.csv @@ -0,0 +1,401 @@ +Timestamp,Model,Request tokens,Response tokens,Total tokens,Log Type +5,ChatGPT,472,18,490,Conversation log +45,ChatGPT,1087,230,1317,Conversation log +118,GPT-4,417,276,693,Conversation log +185,ChatGPT,1360,647,2007,Conversation log +214,ChatGPT,185,215,400,Conversation log +233,GPT-4,586,293,879,Conversation log +261,ChatGPT,37,1656,1693,Conversation log +267,ChatGPT,54,503,557,Conversation log +410,ChatGPT,1528,414,1942,Conversation log +535,ChatGPT,89,370,459,Conversation log +560,GPT-4,549,362,911,Conversation log +638,ChatGPT,172,69,241,Conversation log +686,GPT-4,969,206,1175,Conversation log +741,ChatGPT,97,137,234,Conversation log +771,GPT-4,1574,501,2075,Conversation log +821,ChatGPT,253,239,492,Conversation log +966,ChatGPT,23,344,367,Conversation log +974,ChatGPT,509,207,716,Conversation log +1028,ChatGPT,387,349,736,Conversation log +1072,ChatGPT,733,180,913,Conversation log +1106,ChatGPT,0,0,0,Conversation log +1129,ChatGPT,1305,477,1782,Conversation log +1278,ChatGPT,17,382,399,Conversation log +1449,ChatGPT,601,557,1158,Conversation log +1611,GPT-4,1327,239,1566,Conversation log +1780,ChatGPT,1437,181,1618,Conversation log +2069,ChatGPT,1336,245,1581,Conversation log +2173,ChatGPT,19,440,459,Conversation log +2273,GPT-4,92,95,187,Conversation log +2293,ChatGPT,507,448,955,Conversation log +2579,ChatGPT,938,263,1201,Conversation log +2584,ChatGPT,1544,608,2152,Conversation log +2608,GPT-4,1840,378,2218,Conversation log +2684,ChatGPT,991,462,1453,Conversation log +2826,ChatGPT,1109,329,1438,Conversation log +3009,ChatGPT,1476,443,1919,Conversation log +3091,ChatGPT,14,11,25,Conversation log +3232,GPT-4,1741,428,2169,Conversation log +4525,GPT-4,0,0,0,Conversation log +4574,ChatGPT,1306,267,1573,Conversation log +4608,GPT-4,0,0,0,Conversation log +4725,GPT-4,0,0,0,Conversation log +4798,GPT-4,614,323,937,Conversation log +5812,ChatGPT,1960,136,2096,Conversation log +6049,ChatGPT,1698,627,2325,Conversation log +6263,ChatGPT,1907,616,2523,Conversation log +6499,ChatGPT,137,225,362,Conversation log +6882,GPT-4,20,86,106,Conversation log +6998,GPT-4,127,268,395,Conversation log +7442,GPT-4,665,167,832,Conversation log +7982,GPT-4,855,297,1152,Conversation log +8052,GPT-4,1191,30,1221,Conversation log +8126,GPT-4,18,67,85,Conversation log +8208,GPT-4,101,148,249,Conversation log +8211,ChatGPT,112,336,448,Conversation log +8258,GPT-4,264,76,340,Conversation log +8681,ChatGPT,82,343,425,Conversation log +8782,ChatGPT,506,343,849,Conversation log +8989,ChatGPT,530,339,869,Conversation log +9015,ChatGPT,942,396,1338,Conversation log +9418,ChatGPT,1364,416,1780,Conversation log +10205,ChatGPT,122,496,618,Conversation log +10414,ChatGPT,24,63,87,Conversation log +10443,ChatGPT,698,258,956,Conversation log +10577,ChatGPT,103,76,179,Conversation log +10600,ChatGPT,1073,129,1202,Conversation log +10603,ChatGPT,200,95,295,Conversation log +10636,ChatGPT,1282,188,1470,Conversation log +10669,ChatGPT,1762,418,2180,Conversation log +10740,ChatGPT,1625,268,1893,Conversation log +10760,ChatGPT,25,326,351,Conversation log +10760,ChatGPT,1639,188,1827,Conversation log +10812,ChatGPT,42,197,239,Conversation log +10825,ChatGPT,1637,231,1868,Conversation log +10949,ChatGPT,1701,381,2082,Conversation log +10995,ChatGPT,383,319,702,Conversation log +11096,ChatGPT,266,246,512,Conversation log +11411,ChatGPT,1450,497,1947,Conversation log +11517,ChatGPT,1631,422,2053,Conversation log +11539,ChatGPT,540,285,825,Conversation log +11642,ChatGPT,1797,577,2374,Conversation log +11738,ChatGPT,0,0,0,Conversation log +11817,ChatGPT,20,310,330,Conversation log +11853,ChatGPT,352,298,650,Conversation log +12560,ChatGPT,726,334,1060,Conversation log +14618,ChatGPT,870,269,1139,Conversation log +19020,ChatGPT,66,342,408,Conversation log +19108,ChatGPT,478,296,774,Conversation log +19499,ChatGPT,837,292,1129,Conversation log +19601,ChatGPT,1152,237,1389,Conversation log +20288,ChatGPT,42,17,59,Conversation log +26939,ChatGPT,29,69,98,Conversation log +27037,ChatGPT,136,74,210,Conversation log +27162,ChatGPT,229,69,298,Conversation log +27307,ChatGPT,326,66,392,Conversation log +27894,ChatGPT,408,322,730,Conversation log +27948,ChatGPT,667,79,746,Conversation log +28389,ChatGPT,657,192,849,Conversation log +28436,ChatGPT,788,292,1080,Conversation log +28571,ChatGPT,1035,227,1262,Conversation log +28765,GPT-4,930,83,1013,API log +28829,GPT-4,929,80,1009,API log +29450,ChatGPT,36,14,50,Conversation log +29549,ChatGPT,103,23,126,Conversation log +29568,ChatGPT,209,242,451,Conversation log +29648,ChatGPT,505,166,671,Conversation log +29822,ChatGPT,166,34,200,Conversation log +29832,ChatGPT,19,326,345,Conversation log +29848,ChatGPT,239,30,269,Conversation log +29870,ChatGPT,357,559,916,Conversation log +29907,ChatGPT,320,28,348,Conversation log +29987,ChatGPT,333,35,368,Conversation log +29992,ChatGPT,342,11,353,Conversation log +30024,ChatGPT,353,62,415,Conversation log +30096,ChatGPT,26,205,231,Conversation log +30324,ChatGPT,950,67,1017,Conversation log +30342,ChatGPT,249,229,478,Conversation log +30348,ChatGPT,926,73,999,Conversation log +30368,ChatGPT,495,270,765,Conversation log +30371,ChatGPT,807,73,880,Conversation log +30393,ChatGPT,587,96,683,Conversation log +30402,ChatGPT,0,0,0,Conversation log +30414,ChatGPT,0,0,0,Conversation log +30418,ChatGPT,0,0,0,Conversation log +30422,ChatGPT,1539,82,1621,Conversation log +30424,ChatGPT,428,194,622,Conversation log +30476,ChatGPT,434,1018,1452,Conversation log +30528,ChatGPT,561,79,640,Conversation log +30601,ChatGPT,565,90,655,Conversation log +30612,ChatGPT,31,144,175,Conversation log +30649,ChatGPT,21,16,37,Conversation log +30654,ChatGPT,578,300,878,Conversation log +30677,ChatGPT,68,232,300,Conversation log +30795,ChatGPT,14,423,437,Conversation log +30852,ChatGPT,839,465,1304,Conversation log +30878,ChatGPT,447,381,828,Conversation log +30899,ChatGPT,838,436,1274,Conversation log +31163,ChatGPT,1107,288,1395,Conversation log +31244,ChatGPT,1306,245,1551,Conversation log +31293,ChatGPT,1458,366,1824,Conversation log +31390,ChatGPT,1524,427,1951,Conversation log +31458,ChatGPT,1435,462,1897,Conversation log +31790,ChatGPT,1618,161,1779,Conversation log +31810,ChatGPT,1536,208,1744,Conversation log +31826,ChatGPT,1385,150,1535,Conversation log +31841,ChatGPT,1113,197,1310,Conversation log +31906,ChatGPT,109,1413,1522,Conversation log +31985,ChatGPT,45,972,1017,Conversation log +31994,ChatGPT,36,307,343,Conversation log +32004,GPT-4,380,210,590,Conversation log +32034,ChatGPT,28,62,90,Conversation log +32119,ChatGPT,42,954,996,Conversation log +32185,ChatGPT,246,130,376,Conversation log +32247,ChatGPT,395,787,1182,Conversation log +32311,ChatGPT,1254,312,1566,Conversation log +32387,ChatGPT,840,246,1086,Conversation log +32488,ChatGPT,1169,845,2014,Conversation log +32561,ChatGPT,1304,343,1647,Conversation log +32582,ChatGPT,19,56,75,Conversation log +32599,ChatGPT,92,252,344,Conversation log +32601,ChatGPT,69,381,450,Conversation log +32625,ChatGPT,10,318,328,Conversation log +32656,ChatGPT,531,40,571,Conversation log +32663,ChatGPT,23,426,449,Conversation log +32674,ChatGPT,49,11,60,Conversation log +32678,ChatGPT,40,541,581,Conversation log +32680,GPT-4,99,470,569,Conversation log +32698,ChatGPT,134,55,189,Conversation log +32820,ChatGPT,42,1362,1404,Conversation log +32915,ChatGPT,1877,455,2332,Conversation log +32946,ChatGPT,12,391,403,Conversation log +33017,ChatGPT,302,119,421,Conversation log +33035,ChatGPT,447,639,1086,Conversation log +33042,ChatGPT,10,3,13,Conversation log +33068,ChatGPT,26,67,93,Conversation log +33072,ChatGPT,103,16,119,Conversation log +33099,ChatGPT,0,0,0,Conversation log +33111,ChatGPT,1737,300,2037,Conversation log +33206,ChatGPT,22,416,438,Conversation log +33312,ChatGPT,111,1158,1269,Conversation log +33313,ChatGPT,100,104,204,Conversation log +33324,ChatGPT,1416,586,2002,Conversation log +33371,ChatGPT,39,1238,1277,Conversation log +33407,ChatGPT,22,287,309,Conversation log +33411,ChatGPT,13,4,17,Conversation log +33413,ChatGPT,405,256,661,Conversation log +33437,ChatGPT,38,19,57,Conversation log +33454,ChatGPT,708,44,752,Conversation log +33481,ChatGPT,963,109,1072,Conversation log +33509,ChatGPT,1276,266,1542,Conversation log +33510,ChatGPT,1065,524,1589,Conversation log +33527,ChatGPT,271,54,325,Conversation log +33581,ChatGPT,1314,1903,3217,Conversation log +33602,ChatGPT,337,356,693,Conversation log +33632,ChatGPT,123,261,384,Conversation log +33632,ChatGPT,387,58,445,Conversation log +33637,ChatGPT,102,36,138,Conversation log +33696,ChatGPT,854,350,1204,Conversation log +33747,ChatGPT,522,65,587,Conversation log +33784,ChatGPT,916,484,1400,Conversation log +33810,ChatGPT,603,6,609,Conversation log +33850,ChatGPT,628,495,1123,Conversation log +33866,ChatGPT,31,26,57,Conversation log +34015,ChatGPT,16,421,437,Conversation log +34016,ChatGPT,1640,664,2304,Conversation log +34035,ChatGPT,1232,438,1670,Conversation log +34050,ChatGPT,489,92,581,Conversation log +34068,ChatGPT,40,1092,1132,Conversation log +34069,ChatGPT,461,496,957,Conversation log +34076,ChatGPT,362,155,517,Conversation log +34087,ChatGPT,149,3,152,Conversation log +34100,ChatGPT,1540,35,1575,Conversation log +34102,GPT-4,974,513,1487,Conversation log +34105,ChatGPT,541,201,742,Conversation log +34117,ChatGPT,954,51,1005,Conversation log +34126,ChatGPT,930,92,1022,Conversation log +34129,ChatGPT,34,15,49,Conversation log +34139,ChatGPT,1169,2019,3188,Conversation log +34146,GPT-4,1512,368,1880,Conversation log +34173,ChatGPT,1994,570,2564,Conversation log +34173,ChatGPT,489,338,827,Conversation log +34193,ChatGPT,72,19,91,Conversation log +34246,ChatGPT,1692,401,2093,Conversation log +34257,ChatGPT,849,5,854,Conversation log +34271,ChatGPT,490,27,517,Conversation log +34273,ChatGPT,205,144,349,Conversation log +34278,ChatGPT,1390,260,1650,Conversation log +34291,ChatGPT,547,302,849,Conversation log +34294,GPT-4,32,262,294,Conversation log +34320,ChatGPT,1021,335,1356,Conversation log +34323,ChatGPT,122,376,498,Conversation log +34347,ChatGPT,1844,233,2077,Conversation log +34352,ChatGPT,98,33,131,Conversation log +34384,ChatGPT,436,31,467,Conversation log +34405,ChatGPT,30,294,324,Conversation log +34420,ChatGPT,55,34,89,Conversation log +34444,ChatGPT,16,199,215,Conversation log +34525,ChatGPT,704,331,1035,Conversation log +34534,ChatGPT,71,731,802,Conversation log +34542,ChatGPT,47,32,79,Conversation log +34560,ChatGPT,1298,630,1928,Conversation log +34601,GPT-4,15,115,130,Conversation log +34615,GPT-4,145,314,459,Conversation log +34640,ChatGPT,0,0,0,Conversation log +34680,GPT-4,477,297,774,Conversation log +34746,GPT-4,189,253,442,Conversation log +34758,ChatGPT,418,95,513,Conversation log +34770,GPT-4,585,26,611,Conversation log +34802,GPT-4,757,211,968,Conversation log +34818,ChatGPT,705,299,1004,Conversation log +34824,GPT-4,203,200,403,Conversation log +34832,ChatGPT,17,114,131,Conversation log +34868,ChatGPT,22,29,51,Conversation log +34876,ChatGPT,210,416,626,Conversation log +34890,GPT-4,978,197,1175,Conversation log +34896,GPT-4,188,181,369,Conversation log +34911,GPT-4,226,362,588,Conversation log +34919,ChatGPT,16,18,34,Conversation log +34928,ChatGPT,45,14,59,Conversation log +34943,GPT-4,163,19,182,Conversation log +34976,ChatGPT,0,0,0,Conversation log +34986,ChatGPT,1258,384,1642,Conversation log +34988,ChatGPT,1956,353,2309,Conversation log +34995,GPT-4,389,221,610,Conversation log +35016,ChatGPT,1046,395,1441,Conversation log +35056,ChatGPT,867,30,897,Conversation log +35087,ChatGPT,145,618,763,Conversation log +35096,GPT-4,603,339,942,Conversation log +35132,GPT-4,19,380,399,Conversation log +35134,ChatGPT,870,80,950,Conversation log +35138,ChatGPT,72,13,85,Conversation log +35142,ChatGPT,963,90,1053,Conversation log +35149,GPT-4,33,82,115,Conversation log +35156,GPT-4,342,231,573,Conversation log +35161,GPT-4,161,22,183,Conversation log +35168,GPT-4,419,385,804,Conversation log +35178,ChatGPT,244,149,393,Conversation log +35184,ChatGPT,66,36,102,Conversation log +35220,ChatGPT,162,14,176,Conversation log +35227,GPT-4,210,17,227,Conversation log +35235,ChatGPT,862,214,1076,Conversation log +35244,ChatGPT,263,17,280,Conversation log +35246,ChatGPT,168,35,203,Conversation log +35258,ChatGPT,1122,59,1181,Conversation log +35260,ChatGPT,1036,13,1049,Conversation log +35266,ChatGPT,353,15,368,Conversation log +35293,ChatGPT,354,342,696,Conversation log +35319,ChatGPT,302,887,1189,Conversation log +35369,ChatGPT,723,541,1264,Conversation log +35370,ChatGPT,1202,157,1359,Conversation log +35374,GPT-4,466,60,526,Conversation log +35403,ChatGPT,217,241,458,Conversation log +35410,ChatGPT,25,57,82,Conversation log +35415,ChatGPT,1360,702,2062,Conversation log +35426,ChatGPT,70,82,152,Conversation log +35429,ChatGPT,10,111,121,Conversation log +35453,ChatGPT,140,98,238,Conversation log +35459,ChatGPT,20,556,576,Conversation log +35520,ChatGPT,590,93,683,Conversation log +35541,ChatGPT,308,151,459,Conversation log +35576,GPT-4,58,26,84,Conversation log +35611,ChatGPT,35,12,47,Conversation log +35647,ChatGPT,74,19,93,Conversation log +35746,ChatGPT,15,38,53,Conversation log +35761,ChatGPT,0,0,0,Conversation log +35776,ChatGPT,396,333,729,Conversation log +35788,ChatGPT,247,506,753,Conversation log +35796,ChatGPT,757,356,1113,Conversation log +35816,GPT-4,45,32,77,Conversation log +35837,GPT-4,273,25,298,Conversation log +35861,ChatGPT,274,442,716,Conversation log +35876,ChatGPT,1130,812,1942,Conversation log +35894,GPT-4,820,26,846,Conversation log +35898,GPT-4,788,188,976,Conversation log +35921,ChatGPT,18,412,430,Conversation log +35954,ChatGPT,179,329,508,Conversation log +35975,ChatGPT,19,38,57,Conversation log +36021,ChatGPT,23,27,50,Conversation log +36025,ChatGPT,1286,16,1302,Conversation log +36043,ChatGPT,535,297,832,Conversation log +36055,ChatGPT,17,7,24,Conversation log +36069,ChatGPT,36,24,60,Conversation log +36074,ChatGPT,992,15,1007,Conversation log +36097,GPT-4,695,16,711,Conversation log +36114,ChatGPT,36,238,274,Conversation log +36116,GPT-4,146,299,445,Conversation log +36121,GPT-4,149,256,405,Conversation log +36165,ChatGPT,73,273,346,Conversation log +36173,GPT-4,985,74,1059,Conversation log +36195,ChatGPT,18,457,475,Conversation log +36246,GPT-4,1958,737,2695,Conversation log +36282,GPT-4,36,505,541,Conversation log +36286,ChatGPT,66,17,83,Conversation log +36308,ChatGPT,88,300,388,Conversation log +36309,ChatGPT,203,285,488,Conversation log +36343,ChatGPT,232,514,746,Conversation log +36352,GPT-4,74,404,478,Conversation log +36362,ChatGPT,537,249,786,Conversation log +36371,ChatGPT,0,0,0,Conversation log +36385,GPT-4,567,416,983,Conversation log +36435,ChatGPT,14,203,217,Conversation log +36447,GPT-4,995,454,1449,Conversation log +36471,ChatGPT,836,701,1537,Conversation log +36479,ChatGPT,81,999,1080,Conversation log +36516,ChatGPT,477,163,640,Conversation log +36591,ChatGPT,1098,721,1819,Conversation log +36607,GPT-4,29,2,31,Conversation log +36617,ChatGPT,24,14,38,Conversation log +36626,GPT-4,1475,485,1960,Conversation log +36638,GPT-4,1025,405,1430,Conversation log +36646,GPT-4,391,25,416,Conversation log +36647,ChatGPT,48,11,59,Conversation log +36660,ChatGPT,69,14,83,Conversation log +36665,ChatGPT,36,370,406,Conversation log +36673,GPT-4,347,147,494,Conversation log +36700,ChatGPT,1847,1362,3209,Conversation log +36701,GPT-4,510,599,1109,Conversation log +36716,ChatGPT,20,606,626,Conversation log +36738,ChatGPT,20,171,191,Conversation log +36754,ChatGPT,0,0,0,Conversation log +36758,ChatGPT,0,0,0,Conversation log +36789,ChatGPT,26,23,49,Conversation log +36791,ChatGPT,299,20,319,Conversation log +36804,GPT-4,1995,663,2658,Conversation log +36805,ChatGPT,59,31,90,Conversation log +36836,ChatGPT,1365,214,1579,Conversation log +36850,ChatGPT,1595,354,1949,Conversation log +36868,GPT-4,500,238,738,Conversation log +36883,ChatGPT,51,1442,1493,Conversation log +36915,GPT-4,2126,379,2505,Conversation log +36922,ChatGPT,33,81,114,Conversation log +36968,ChatGPT,79,59,138,Conversation log +36989,GPT-4,145,45,190,Conversation log +36990,ChatGPT,118,24,142,Conversation log +36993,ChatGPT,222,43,265,Conversation log +36997,ChatGPT,21,387,408,Conversation log +37000,GPT-4,752,280,1032,Conversation log +37006,ChatGPT,158,104,262,Conversation log +37008,ChatGPT,170,14,184,Conversation log +37053,ChatGPT,290,90,380,Conversation log +37081,ChatGPT,601,327,928,Conversation log +37106,GPT-4,29,213,242,Conversation log +37117,GPT-4,112,110,222,Conversation log +37140,ChatGPT,50,962,1012,Conversation log +37148,ChatGPT,733,311,1044,Conversation log +37178,ChatGPT,9,9,18,Conversation log +37182,ChatGPT,40,11,51,Conversation log +37183,GPT-4,41,552,593,Conversation log +37189,ChatGPT,1544,1421,2965,Conversation log +37198,GPT-4,28,231,259,Conversation log +37201,ChatGPT,24,443,467,Conversation log +37218,ChatGPT,657,1011,1668,Conversation log +37228,ChatGPT,60,46,106,Conversation log +37235,GPT-4,275,225,500,Conversation log +37236,ChatGPT,0,0,0,Conversation log +37243,ChatGPT,0,0,0,Conversation log +37255,ChatGPT,50,1143,1193,Conversation log +37272,GPT-4,251,23,274,Conversation log +37274,GPT-4,18,70,88,Conversation log +37274,ChatGPT,67,58,125,Conversation log diff --git a/benchmarks/traffic_replay/fixtures/mooncake_dirty.jsonl b/benchmarks/traffic_replay/fixtures/mooncake_dirty.jsonl new file mode 100644 index 0000000..c90273b --- /dev/null +++ b/benchmarks/traffic_replay/fixtures/mooncake_dirty.jsonl @@ -0,0 +1,9 @@ +{"timestamp": 0, "input_length": 1024, "output_length": 64, "hash_ids": [1, 2]} +{"timestamp": 1000, "input_length": 0, "output_length": 64, "hash_ids": []} +{"timestamp": 2000, "input_length": 512, "output_length": 0, "hash_ids": [3]} +{"timestamp": 3000, "input_length": -5, "output_length": 8, "hash_ids": [4]} +{"timestamp": 4000, "input_length": "x", "output_length": 8, "hash_ids": [5]} +{"timestamp": 5000, "output_length": 8, "hash_ids": [6]} +this line is not json at all +{"timestamp": 4500, "input_length": 512, "output_length": 8, "hash_ids": [7]} +{"timestamp": 6000, "input_length": 1536, "output_length": 32, "hash_ids": [1, 2, 8]} diff --git a/benchmarks/traffic_replay/fixtures/mooncake_slice.jsonl b/benchmarks/traffic_replay/fixtures/mooncake_slice.jsonl new file mode 100644 index 0000000..20ebead --- /dev/null +++ b/benchmarks/traffic_replay/fixtures/mooncake_slice.jsonl @@ -0,0 +1,400 @@ +{"timestamp": 0, "input_length": 6758, "output_length": 500, "hash_ids": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]} +{"timestamp": 0, "input_length": 7322, "output_length": 490, "hash_ids": [0, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]} +{"timestamp": 0, "input_length": 7236, "output_length": 794, "hash_ids": [0, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41]} +{"timestamp": 0, "input_length": 2290, "output_length": 316, "hash_ids": [0, 42, 43, 44, 45]} +{"timestamp": 0, "input_length": 6760, "output_length": 3, "hash_ids": [0, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58]} +{"timestamp": 0, "input_length": 4834, "output_length": 173, "hash_ids": [0, 59, 60, 61, 62, 63, 64, 65, 66, 67]} +{"timestamp": 0, "input_length": 23141, "output_length": 453, "hash_ids": [0, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112]} +{"timestamp": 0, "input_length": 26888, "output_length": 458, "hash_ids": [0, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164]} +{"timestamp": 0, "input_length": 10498, "output_length": 402, "hash_ids": [0, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184]} +{"timestamp": 0, "input_length": 17450, "output_length": 610, "hash_ids": [0, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218]} +{"timestamp": 3000, "input_length": 13544, "output_length": 71, "hash_ids": [0, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244]} +{"timestamp": 3000, "input_length": 87169, "output_length": 402, "hash_ids": [0, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414]} +{"timestamp": 3000, "input_length": 6324, "output_length": 548, "hash_ids": [0, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426]} +{"timestamp": 3000, "input_length": 2012, "output_length": 354, "hash_ids": [0, 427, 428, 429]} +{"timestamp": 3000, "input_length": 7324, "output_length": 14, "hash_ids": [0, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443]} +{"timestamp": 3000, "input_length": 9418, "output_length": 145, "hash_ids": [0, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, 456, 457, 458, 459, 460, 461]} +{"timestamp": 3000, "input_length": 915, "output_length": 355, "hash_ids": [0, 462]} +{"timestamp": 3000, "input_length": 12846, "output_length": 466, "hash_ids": [0, 463, 464, 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, 487]} +{"timestamp": 3000, "input_length": 20506, "output_length": 929, "hash_ids": [0, 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527]} +{"timestamp": 3000, "input_length": 16609, "output_length": 349, "hash_ids": [0, 528, 529, 530, 531, 532, 533, 534, 535, 536, 537, 538, 539, 540, 541, 542, 543, 544, 545, 546, 547, 548, 549, 550, 551, 552, 553, 554, 555, 556, 557, 558, 559]} +{"timestamp": 3000, "input_length": 26353, "output_length": 370, "hash_ids": [0, 560, 561, 562, 563, 564, 565, 566, 567, 568, 569, 570, 571, 572, 573, 574, 575, 576, 577, 578, 579, 580, 581, 582, 583, 584, 585, 586, 587, 588, 589, 590, 591, 592, 593, 594, 595, 596, 597, 598, 599, 600, 601, 602, 603, 604, 605, 606, 607, 608, 609, 610]} +{"timestamp": 3000, "input_length": 6059, "output_length": 475, "hash_ids": [0, 611, 612, 613, 614, 615, 616, 617, 618, 619, 620, 621]} +{"timestamp": 3000, "input_length": 5954, "output_length": 420, "hash_ids": [0, 622, 623, 624, 625, 626, 627, 628, 629, 630, 631, 632]} +{"timestamp": 3000, "input_length": 11339, "output_length": 848, "hash_ids": [0, 633, 634, 635, 636, 637, 638, 639, 640, 641, 642, 643, 644, 645, 646, 647, 648, 649, 650, 651, 652, 653, 654]} +{"timestamp": 3000, "input_length": 15172, "output_length": 80, "hash_ids": [0, 655, 656, 657, 658, 659, 660, 661, 662, 663, 664, 665, 666, 667, 668, 669, 670, 671, 672, 673, 674, 675, 676, 677, 678, 679, 680, 681, 682, 683]} +{"timestamp": 3000, "input_length": 45922, "output_length": 265, "hash_ids": [0, 684, 685, 686, 687, 688, 689, 690, 691, 692, 693, 694, 695, 696, 697, 698, 699, 700, 701, 702, 703, 704, 705, 706, 707, 708, 709, 710, 711, 712, 713, 714, 715, 716, 717, 718, 719, 720, 721, 722, 723, 724, 725, 726, 727, 728, 729, 730, 731, 732, 733, 734, 735, 736, 737, 738, 739, 740, 741, 742, 743, 744, 745, 746, 747, 748, 749, 750, 751, 752, 753, 754, 755, 756, 757, 758, 759, 760, 761, 762, 763, 764, 765, 766, 767, 768, 769, 770, 771, 772]} +{"timestamp": 5999, "input_length": 1053, "output_length": 26, "hash_ids": [0, 773, 774]} +{"timestamp": 5999, "input_length": 5710, "output_length": 745, "hash_ids": [0, 775, 776, 777, 778, 779, 780, 781, 782, 783, 784, 785]} +{"timestamp": 5999, "input_length": 7238, "output_length": 11, "hash_ids": [0, 786, 787, 788, 789, 790, 791, 792, 793, 794, 795, 796, 797, 798, 799]} +{"timestamp": 9000, "input_length": 10355, "output_length": 584, "hash_ids": [0, 800, 801, 802, 803, 804, 805, 806, 807, 808, 809, 810, 811, 812, 813, 814, 815, 816, 817, 818, 819]} +{"timestamp": 9000, "input_length": 1477, "output_length": 615, "hash_ids": [0, 820, 821]} +{"timestamp": 9000, "input_length": 15366, "output_length": 344, "hash_ids": [0, 822, 823, 824, 825, 826, 827, 828, 829, 830, 831, 832, 833, 834, 835, 836, 837, 838, 839, 840, 841, 842, 843, 844, 845, 846, 847, 848, 849, 850, 851]} +{"timestamp": 9000, "input_length": 3806, "output_length": 309, "hash_ids": [0, 852, 853, 854, 855, 856, 857, 858]} +{"timestamp": 9000, "input_length": 8772, "output_length": 1, "hash_ids": [0, 859, 860, 861, 862, 863, 864, 865, 866, 867, 868, 869, 870, 871, 872, 873, 874, 875]} +{"timestamp": 9000, "input_length": 26156, "output_length": 238, "hash_ids": [0, 876, 877, 878, 879, 880, 881, 882, 883, 884, 885, 886, 887, 888, 889, 890, 891, 892, 893, 894, 895, 896, 897, 898, 899, 900, 901, 902, 903, 904, 905, 906, 907, 908, 909, 910, 911, 912, 913, 914, 915, 916, 917, 918, 919, 920, 921, 922, 923, 924, 925, 926]} +{"timestamp": 9000, "input_length": 16635, "output_length": 449, "hash_ids": [0, 927, 928, 929, 930, 931, 932, 933, 934, 935, 936, 937, 938, 939, 940, 941, 942, 943, 944, 945, 946, 947, 948, 949, 950, 951, 952, 953, 954, 955, 956, 957, 958]} +{"timestamp": 9000, "input_length": 2293, "output_length": 31, "hash_ids": [0, 959, 960, 961, 962]} +{"timestamp": 9000, "input_length": 1110, "output_length": 240, "hash_ids": [0, 963, 964]} +{"timestamp": 12000, "input_length": 3628, "output_length": 555, "hash_ids": [0, 965, 966, 967, 968, 969, 970, 971]} +{"timestamp": 12000, "input_length": 2038, "output_length": 524, "hash_ids": [0, 972, 973, 974]} +{"timestamp": 12000, "input_length": 1902, "output_length": 587, "hash_ids": [0, 975, 976, 977]} +{"timestamp": 12000, "input_length": 14041, "output_length": 227, "hash_ids": [0, 978, 979, 980, 981, 982, 983, 984, 985, 986, 987, 988, 989, 990, 991, 992, 993, 994, 995, 996, 997, 998, 999, 1000, 1001, 1002, 1003, 1004]} +{"timestamp": 12000, "input_length": 12095, "output_length": 239, "hash_ids": [0, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1023, 1024, 1025, 1026, 1027]} +{"timestamp": 12000, "input_length": 1066, "output_length": 324, "hash_ids": [0, 1028, 1029]} +{"timestamp": 12000, "input_length": 9615, "output_length": 203, "hash_ids": [0, 1030, 1031, 1032, 1033, 1034, 1035, 1036, 1037, 1038, 1039, 1040, 1041, 1042, 1043, 1044, 1045, 1046, 1047]} +{"timestamp": 12000, "input_length": 19976, "output_length": 105, "hash_ids": [0, 1048, 1049, 1050, 1051, 1052, 1053, 1054, 1055, 1056, 1057, 1058, 1059, 1060, 1061, 1062, 1063, 1064, 1065, 1066, 1067, 1068, 1069, 1070, 1071, 1072, 1073, 1074, 1075, 1076, 1077, 1078, 1079, 1080, 1081, 1082, 1083, 1084, 1085, 1086]} +{"timestamp": 15000, "input_length": 6525, "output_length": 481, "hash_ids": [0, 1087, 1088, 1089, 1090, 1091, 1092, 1093, 1094, 1095, 1096, 1097, 1098]} +{"timestamp": 15000, "input_length": 898, "output_length": 324, "hash_ids": [0, 1099]} +{"timestamp": 15000, "input_length": 5391, "output_length": 361, "hash_ids": [0, 1100, 1101, 1102, 1103, 1104, 1105, 1106, 1107, 1108, 1109]} +{"timestamp": 15000, "input_length": 23631, "output_length": 362, "hash_ids": [0, 1110, 1111, 1112, 1113, 1114, 1115, 1116, 1117, 1118, 1119, 1120, 1121, 1122, 1123, 1124, 1125, 1126, 1127, 1128, 1129, 1130, 1131, 1132, 1133, 1134, 1135, 1136, 1137, 1138, 1139, 1140, 1141, 1142, 1143, 1144, 1145, 1146, 1147, 1148, 1149, 1150, 1151, 1152, 1153, 1154, 1155]} +{"timestamp": 15000, "input_length": 38383, "output_length": 589, "hash_ids": [0, 1156, 1157, 1158, 1159, 1160, 1161, 1162, 1163, 1164, 1165, 1166, 1167, 1168, 1169, 1170, 1171, 1172, 1173, 1174, 1175, 1176, 1177, 1178, 1179, 1180, 1181, 1182, 1183, 1184, 1185, 1186, 1187, 1188, 1189, 1190, 1191, 1192, 1193, 1194, 1195, 1196, 1197, 1198, 1199, 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211, 1212, 1213, 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227, 1228, 1229]} +{"timestamp": 15000, "input_length": 7543, "output_length": 415, "hash_ids": [0, 1230, 1231, 1232, 1233, 1234, 1235, 1236, 1237, 1238, 1239, 1240, 1241, 1242, 1243]} +{"timestamp": 15000, "input_length": 8974, "output_length": 219, "hash_ids": [0, 1244, 1245, 1246, 1247, 1248, 1249, 1250, 1251, 1252, 1253, 1254, 1255, 1256, 1257, 1258, 1259, 1260]} +{"timestamp": 15000, "input_length": 24246, "output_length": 587, "hash_ids": [0, 1261, 1262, 1263, 1264, 1265, 1266, 1267, 1268, 1269, 1270, 1271, 1272, 1273, 1274, 1275, 1276, 1277, 1278, 1279, 1280, 1281, 1282, 1283, 1284, 1285, 1286, 1287, 1288, 1289, 1290, 1291, 1292, 1293, 1294, 1295, 1296, 1297, 1298, 1299, 1300, 1301, 1302, 1303, 1304, 1305, 1306, 1307]} +{"timestamp": 15000, "input_length": 15761, "output_length": 417, "hash_ids": [0, 1308, 1309, 1310, 1311, 1312, 1313, 1314, 1315, 1316, 1317, 1318, 1319, 1320, 1321, 1322, 1323, 1324, 1325, 1326, 1327, 1328, 1329, 1330, 1331, 1332, 1333, 1334, 1335, 1336, 1337]} +{"timestamp": 15000, "input_length": 27701, "output_length": 459, "hash_ids": [0, 1338, 1339, 1340, 1341, 1342, 1343, 1344, 1345, 1346, 1347, 1348, 1349, 1350, 1351, 1352, 1353, 1354, 1355, 1356, 1357, 1358, 1359, 1360, 1361, 1362, 1363, 1364, 1365, 1366, 1367, 1368, 1369, 1370, 1371, 1372, 1373, 1374, 1375, 1376, 1377, 1378, 1379, 1380, 1381, 1382, 1383, 1384, 1385, 1386, 1387, 1388, 1389, 1390, 1391]} +{"timestamp": 15000, "input_length": 11041, "output_length": 98, "hash_ids": [0, 1392, 1393, 1394, 1395, 1396, 1397, 1398, 1399, 1400, 1401, 1402, 1403, 1404, 1405, 1406, 1407, 1408, 1409, 1410, 1411, 1412]} +{"timestamp": 15000, "input_length": 4836, "output_length": 24, "hash_ids": [0, 1413, 1414, 1415, 1416, 1417, 1418, 1419, 1420, 1421]} +{"timestamp": 15000, "input_length": 2350, "output_length": 355, "hash_ids": [0, 1422, 1423, 1424, 1425]} +{"timestamp": 15000, "input_length": 934, "output_length": 421, "hash_ids": [0, 1426]} +{"timestamp": 15000, "input_length": 14081, "output_length": 522, "hash_ids": [0, 1427, 1428, 1429, 1430, 1431, 1432, 1433, 1434, 1435, 1436, 1437, 1438, 1439, 1440, 1441, 1442, 1443, 1444, 1445, 1446, 1447, 1448, 1449, 1450, 1451, 1452, 1453]} +{"timestamp": 18000, "input_length": 7935, "output_length": 375, "hash_ids": [0, 1454, 1455, 1456, 1457, 1458, 1459, 1460, 1461, 1462, 1463, 1464, 1465, 1466, 1467, 1468]} +{"timestamp": 18000, "input_length": 13886, "output_length": 534, "hash_ids": [0, 1469, 1470, 1471, 1472, 1473, 1474, 1475, 1476, 1477, 1478, 1479, 1480, 1481, 1482, 1483, 1484, 1485, 1486, 1487, 1488, 1489, 1490, 1491, 1492, 1493, 1494, 1495]} +{"timestamp": 18000, "input_length": 898, "output_length": 57, "hash_ids": [0, 1496]} +{"timestamp": 18000, "input_length": 19694, "output_length": 178, "hash_ids": [0, 1497, 1498, 1499, 1500, 1501, 1502, 1503, 1504, 1505, 1506, 1507, 1508, 1509, 1510, 1511, 1512, 1513, 1514, 1515, 1516, 1517, 1518, 1519, 1520, 1521, 1522, 1523, 1524, 1525, 1526, 1527, 1528, 1529, 1530, 1531, 1532, 1533, 1534]} +{"timestamp": 21000, "input_length": 3184, "output_length": 195, "hash_ids": [0, 1535, 1536, 1537, 1538, 1539, 1540]} +{"timestamp": 21000, "input_length": 2651, "output_length": 366, "hash_ids": [0, 1541, 1542, 1543, 1544, 1545]} +{"timestamp": 21000, "input_length": 23143, "output_length": 26, "hash_ids": [0, 1546, 1547, 1548, 1549, 1550, 1551, 1552, 1553, 1554, 1555, 1556, 1557, 1558, 1559, 1560, 1561, 1562, 1563, 1564, 1565, 1566, 1567, 1568, 1569, 1570, 1571, 1572, 1573, 1574, 1575, 1576, 1577, 1578, 1579, 1580, 1581, 1582, 1583, 1584, 1585, 1586, 1587, 1588, 1589, 1590]} +{"timestamp": 21000, "input_length": 5922, "output_length": 542, "hash_ids": [0, 1591, 1592, 1593, 1594, 1595, 1596, 1597, 1598, 1599, 1600, 1601]} +{"timestamp": 21000, "input_length": 8429, "output_length": 439, "hash_ids": [0, 1602, 1603, 1604, 1605, 1606, 1607, 1608, 1609, 1610, 1611, 1612, 1613, 1614, 1615, 1616, 1617]} +{"timestamp": 21000, "input_length": 8352, "output_length": 387, "hash_ids": [0, 1618, 1619, 1620, 1621, 1622, 1623, 1624, 1625, 1626, 1627, 1628, 1629, 1630, 1631, 1632, 1633]} +{"timestamp": 24000, "input_length": 12159, "output_length": 237, "hash_ids": [0, 1634, 1635, 1636, 1637, 1638, 1639, 1640, 1641, 1642, 1643, 1644, 1645, 1646, 1647, 1648, 1649, 1650, 1651, 1652, 1653, 1654, 1655, 1656]} +{"timestamp": 24000, "input_length": 28214, "output_length": 603, "hash_ids": [0, 1657, 1658, 1659, 1660, 1661, 1662, 1663, 1664, 1665, 1666, 1667, 1668, 1669, 1670, 1671, 1672, 1673, 1674, 1675, 1676, 1677, 1678, 1679, 1680, 1681, 1682, 1683, 1684, 1685, 1686, 1687, 1688, 1689, 1690, 1691, 1692, 1693, 1694, 1695, 1696, 1697, 1698, 1699, 1700, 1701, 1702, 1703, 1704, 1705, 1706, 1707, 1708, 1709, 1710, 1711]} +{"timestamp": 24000, "input_length": 26891, "output_length": 3, "hash_ids": [0, 1712, 1713, 1714, 1715, 1716, 1717, 1718, 1719, 1720, 1721, 1722, 1723, 1724, 1725, 1726, 1727, 1728, 1729, 1730, 1731, 1732, 1733, 1734, 1735, 1736, 1737, 1738, 1739, 1740, 1741, 1742, 1743, 1744, 1745, 1746, 1747, 1748, 1749, 1750, 1751, 1752, 1753, 1754, 1755, 1756, 1757, 1758, 1759, 1760, 1761, 1762, 1763]} +{"timestamp": 24000, "input_length": 2133, "output_length": 347, "hash_ids": [0, 1764, 1765, 1766, 1767]} +{"timestamp": 24000, "input_length": 36422, "output_length": 255, "hash_ids": [0, 1768, 1769, 1770, 1771, 1772, 1773, 1774, 1775, 1776, 1777, 1778, 1779, 1780, 1781, 1782, 1783, 1784, 1785, 1786, 1787, 1788, 1789, 1790, 1791, 1792, 1793, 1794, 1795, 1796, 1797, 1798, 1799, 1800, 1801, 1802, 1803, 1804, 1805, 1806, 1807, 1808, 1809, 1810, 1811, 1812, 1813, 1814, 1815, 1816, 1817, 1818, 1819, 1820, 1821, 1822, 1823, 1824, 1825, 1826, 1827, 1828, 1829, 1830, 1831, 1832, 1833, 1834, 1835, 1836, 1837, 1838]} +{"timestamp": 27000, "input_length": 1424, "output_length": 290, "hash_ids": [0, 1839, 1840]} +{"timestamp": 27000, "input_length": 18092, "output_length": 387, "hash_ids": [0, 1841, 1842, 1843, 1844, 1845, 1846, 1847, 1848, 1849, 1850, 1851, 1852, 1853, 1854, 1855, 1856, 1857, 1858, 1859, 1860, 1861, 1862, 1863, 1864, 1865, 1866, 1867, 1868, 1869, 1870, 1871, 1872, 1873, 1874, 1875]} +{"timestamp": 27000, "input_length": 35126, "output_length": 538, "hash_ids": [0, 1876, 1877, 1878, 1879, 1880, 1881, 1882, 1883, 1884, 1885, 1886, 1887, 1888, 1889, 1890, 1891, 1892, 1893, 1894, 1895, 1896, 1897, 1898, 1899, 1900, 1901, 1902, 1903, 1904, 1905, 1906, 1907, 1908, 1909, 1910, 1911, 1912, 1913, 1914, 1915, 1916, 1917, 1918, 1919, 1920, 1921, 1922, 1923, 1924, 1925, 1926, 1927, 1928, 1929, 1930, 1931, 1932, 1933, 1934, 1935, 1936, 1937, 1938, 1939, 1940, 1941, 1942, 1943]} +{"timestamp": 27000, "input_length": 954, "output_length": 48, "hash_ids": [0, 1944]} +{"timestamp": 27000, "input_length": 15852, "output_length": 629, "hash_ids": [0, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952, 1953, 1954, 1955, 1956, 1957, 1958, 1959, 1960, 1961, 1962, 1963, 1964, 1965, 1966, 1967, 1968, 1969, 1970, 1971, 1972, 1973, 1974]} +{"timestamp": 27000, "input_length": 9242, "output_length": 632, "hash_ids": [0, 1975, 1976, 1977, 1978, 1979, 1980, 1981, 1982, 1983, 1984, 1985, 1986, 1987, 1988, 1989, 1990, 1991, 1992]} +{"timestamp": 27000, "input_length": 12927, "output_length": 460, "hash_ids": [0, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017]} +{"timestamp": 27000, "input_length": 17799, "output_length": 345, "hash_ids": [0, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025, 2026, 2027, 2028, 2029, 2030, 2031, 2032, 2033, 2034, 2035, 2036, 2037, 2038, 2039, 2040, 2041, 2042, 2043, 2044, 2045, 2046, 2047, 2048, 2049, 2050, 2051]} +{"timestamp": 27000, "input_length": 8034, "output_length": 1, "hash_ids": [0, 2052, 2053, 2054, 2055, 2056, 2057, 2058, 2059, 2060, 2061, 2062, 2063, 2064, 2065, 2066]} +{"timestamp": 27000, "input_length": 10753, "output_length": 596, "hash_ids": [0, 2067, 2068, 2069, 2070, 2071, 2072, 2073, 2074, 2075, 2076, 2077, 2078, 2079, 2080, 2081, 2082, 2083, 2084, 2085, 2086, 2087]} +{"timestamp": 27000, "input_length": 4541, "output_length": 362, "hash_ids": [0, 2088, 2089, 2090, 2091, 2092, 2093, 2094, 2095]} +{"timestamp": 30000, "input_length": 42722, "output_length": 464, "hash_ids": [0, 2096, 2097, 2098, 2099, 2100, 2101, 2102, 2103, 2104, 2105, 2106, 2107, 2108, 2109, 2110, 2111, 2112, 2113, 2114, 2115, 2116, 2117, 2118, 2119, 2120, 2121, 2122, 2123, 2124, 2125, 2126, 2127, 2128, 2129, 2130, 2131, 2132, 2133, 2134, 2135, 2136, 2137, 2138, 2139, 2140, 2141, 2142, 2143, 2144, 2145, 2146, 2147, 2148, 2149, 2150, 2151, 2152, 2153, 2154, 2155, 2156, 2157, 2158, 2159, 2160, 2161, 2162, 2163, 2164, 2165, 2166, 2167, 2168, 2169, 2170, 2171, 2172, 2173, 2174, 2175, 2176, 2177, 2178]} +{"timestamp": 30000, "input_length": 10500, "output_length": 34, "hash_ids": [0, 2179, 2180, 2181, 2182, 2183, 2184, 2185, 2186, 2187, 2188, 2189, 2190, 2191, 2192, 2193, 2194, 2195, 2196, 2197, 2198]} +{"timestamp": 30000, "input_length": 13616, "output_length": 492, "hash_ids": [0, 2199, 2200, 2201, 2202, 2203, 2204, 2205, 2206, 2207, 2208, 2209, 2210, 2211, 2212, 2213, 2214, 2215, 2216, 2217, 2218, 2219, 2220, 2221, 2222, 2223, 2224]} +{"timestamp": 30000, "input_length": 49902, "output_length": 549, "hash_ids": [0, 2225, 2226, 2227, 2228, 2229, 2230, 2231, 2232, 2233, 2234, 2235, 2236, 2237, 2238, 2239, 2240, 2241, 2242, 2243, 2244, 2245, 2246, 2247, 2248, 2249, 2250, 2251, 2252, 2253, 2254, 2255, 2256, 2257, 2258, 2259, 2260, 2261, 2262, 2263, 2264, 2265, 2266, 2267, 2268, 2269, 2270, 2271, 2272, 2273, 2274, 2275, 2276, 2277, 2278, 2279, 2280, 2281, 2282, 2283, 2284, 2285, 2286, 2287, 2288, 2289, 2290, 2291, 2292, 2293, 2294, 2295, 2296, 2297, 2298, 2299, 2300, 2301, 2302, 2303, 2304, 2305, 2306, 2307, 2308, 2309, 2310, 2311, 2312, 2313, 2314, 2315, 2316, 2317, 2318, 2319, 2320, 2321]} +{"timestamp": 30000, "input_length": 17071, "output_length": 440, "hash_ids": [0, 2322, 2323, 2324, 2325, 2326, 2327, 2328, 2329, 2330, 2331, 2332, 2333, 2334, 2335, 2336, 2337, 2338, 2339, 2340, 2341, 2342, 2343, 2344, 2345, 2346, 2347, 2348, 2349, 2350, 2351, 2352, 2353, 2354]} +{"timestamp": 33000, "input_length": 21094, "output_length": 563, "hash_ids": [0, 2355, 2356, 2357, 2358, 2359, 2360, 2361, 2362, 2363, 2364, 2365, 2366, 2367, 2368, 2369, 2370, 2371, 2372, 2373, 2374, 2375, 2376, 2377, 2378, 2379, 2380, 2381, 2382, 2383, 2384, 2385, 2386, 2387, 2388, 2389, 2390, 2391, 2392, 2393, 2394, 2395]} +{"timestamp": 33000, "input_length": 29265, "output_length": 448, "hash_ids": [0, 2396, 2397, 2398, 2399, 2400, 2401, 2402, 2403, 2404, 2405, 2406, 2407, 2408, 2409, 2410, 2411, 2412, 2413, 2414, 2415, 2416, 2417, 2418, 2419, 2420, 2421, 2422, 2423, 2424, 2425, 2426, 2427, 2428, 2429, 2430, 2431, 2432, 2433, 2434, 2435, 2436, 2437, 2438, 2439, 2440, 2441, 2442, 2443, 2444, 2445, 2446, 2447, 2448, 2449, 2450, 2451, 2452]} +{"timestamp": 33000, "input_length": 17453, "output_length": 35, "hash_ids": [0, 2453, 2454, 2455, 2456, 2457, 2458, 2459, 2460, 2461, 2462, 2463, 2464, 2465, 2466, 2467, 2468, 2469, 2470, 2471, 2472, 2473, 2474, 2475, 2476, 2477, 2478, 2479, 2480, 2481, 2482, 2483, 2484, 2485, 2486]} +{"timestamp": 33000, "input_length": 82276, "output_length": 683, "hash_ids": [0, 2487, 2488, 2489, 2490, 2491, 2492, 2493, 2494, 2495, 2496, 2497, 2498, 2499, 2500, 2501, 2502, 2503, 2504, 2505, 2506, 2507, 2508, 2509, 2510, 2511, 2512, 2513, 2514, 2515, 2516, 2517, 2518, 2519, 2520, 2521, 2522, 2523, 2524, 2525, 2526, 2527, 2528, 2529, 2530, 2531, 2532, 2533, 2534, 2535, 2536, 2537, 2538, 2539, 2540, 2541, 2542, 2543, 2544, 2545, 2546, 2547, 2548, 2549, 2550, 2551, 2552, 2553, 2554, 2555, 2556, 2557, 2558, 2559, 2560, 2561, 2562, 2563, 2564, 2565, 2566, 2567, 2568, 2569, 2570, 2571, 2572, 2573, 2574, 2575, 2576, 2577, 2578, 2579, 2580, 2581, 2582, 2583, 2584, 2585, 2586, 2587, 2588, 2589, 2590, 2591, 2592, 2593, 2594, 2595, 2596, 2597, 2598, 2599, 2600, 2601, 2602, 2603, 2604, 2605, 2606, 2607, 2608, 2609, 2610, 2611, 2612, 2613, 2614, 2615, 2616, 2617, 2618, 2619, 2620, 2621, 2622, 2623, 2624, 2625, 2626, 2627, 2628, 2629, 2630, 2631, 2632, 2633, 2634, 2635, 2636, 2637, 2638, 2639, 2640, 2641, 2642, 2643, 2644, 2645, 2646]} +{"timestamp": 33000, "input_length": 26045, "output_length": 578, "hash_ids": [0, 2647, 2648, 2649, 2650, 2651, 2652, 2653, 2654, 2655, 2656, 2657, 2658, 2659, 2660, 2661, 2662, 2663, 2664, 2665, 2666, 2667, 2668, 2669, 2670, 2671, 2672, 2673, 2674, 2675, 2676, 2677, 2678, 2679, 2680, 2681, 2682, 2683, 2684, 2685, 2686, 2687, 2688, 2689, 2690, 2691, 2692, 2693, 2694, 2695, 2696]} +{"timestamp": 33000, "input_length": 120633, "output_length": 580, "hash_ids": [0, 2697, 2698, 2699, 2700, 2701, 2702, 2703, 2704, 2705, 2706, 2707, 2708, 2709, 2710, 2711, 2712, 2713, 2714, 2715, 2716, 2717, 2718, 2719, 2720, 2721, 2722, 2723, 2724, 2725, 2726, 2727, 2728, 2729, 2730, 2731, 2732, 2733, 2734, 2735, 2736, 2737, 2738, 2739, 2740, 2741, 2742, 2743, 2744, 2745, 2746, 2747, 2748, 2749, 2750, 2751, 2752, 2753, 2754, 2755, 2756, 2757, 2758, 2759, 2760, 2761, 2762, 2763, 2764, 2765, 2766, 2767, 2768, 2769, 2770, 2771, 2772, 2773, 2774, 2775, 2776, 2777, 2778, 2779, 2780, 2781, 2782, 2783, 2784, 2785, 2786, 2787, 2788, 2789, 2790, 2791, 2792, 2793, 2794, 2795, 2796, 2797, 2798, 2799, 2800, 2801, 2802, 2803, 2804, 2805, 2806, 2807, 2808, 2809, 2810, 2811, 2812, 2813, 2814, 2815, 2816, 2817, 2818, 2819, 2820, 2821, 2822, 2823, 2824, 2825, 2826, 2827, 2828, 2829, 2830, 2831, 2832, 2833, 2834, 2835, 2836, 2837, 2838, 2839, 2840, 2841, 2842, 2843, 2844, 2845, 2846, 2847, 2848, 2849, 2850, 2851, 2852, 2853, 2854, 2855, 2856, 2857, 2858, 2859, 2860, 2861, 2862, 2863, 2864, 2865, 2866, 2867, 2868, 2869, 2870, 2871, 2872, 2873, 2874, 2875, 2876, 2877, 2878, 2879, 2880, 2881, 2882, 2883, 2884, 2885, 2886, 2887, 2888, 2889, 2890, 2891, 2892, 2893, 2894, 2895, 2896, 2897, 2898, 2899, 2900, 2901, 2902, 2903, 2904, 2905, 2906, 2907, 2908, 2909, 2910, 2911, 2912, 2913, 2914, 2915, 2916, 2917, 2918, 2919, 2920, 2921, 2922, 2923, 2924, 2925, 2926, 2927, 2928, 2929, 2930, 2931]} +{"timestamp": 33000, "input_length": 1322, "output_length": 514, "hash_ids": [0, 2932, 2933]} +{"timestamp": 33000, "input_length": 916, "output_length": 265, "hash_ids": [0, 2934]} +{"timestamp": 36000, "input_length": 2885, "output_length": 613, "hash_ids": [0, 2935, 2936, 2937, 2938, 2939]} +{"timestamp": 36000, "input_length": 8619, "output_length": 593, "hash_ids": [0, 2940, 2941, 2942, 2943, 2944, 2945, 2946, 2947, 2948, 2949, 2950, 2951, 2952, 2953, 2954, 2955]} +{"timestamp": 36000, "input_length": 1729, "output_length": 31, "hash_ids": [0, 2956, 2957, 2958]} +{"timestamp": 36000, "input_length": 2428, "output_length": 508, "hash_ids": [0, 2959, 2960, 2961, 2962]} +{"timestamp": 36000, "input_length": 11527, "output_length": 373, "hash_ids": [0, 2963, 2964, 2965, 2966, 2967, 2968, 2969, 2970, 2971, 2972, 2973, 2974, 2975, 2976, 2977, 2978, 2979, 2980, 2981, 2982, 2983, 2984]} +{"timestamp": 36000, "input_length": 1087, "output_length": 505, "hash_ids": [0, 2985, 2986]} +{"timestamp": 36000, "input_length": 2392, "output_length": 621, "hash_ids": [0, 2987, 2988, 2989, 2990]} +{"timestamp": 36000, "input_length": 12000, "output_length": 529, "hash_ids": [0, 2991, 2992, 2993, 2994, 2995, 2996, 2997, 2998, 2999, 3000, 3001, 3002, 3003, 3004, 3005, 3006, 3007, 3008, 3009, 3010, 3011, 3012, 3013]} +{"timestamp": 39000, "input_length": 13546, "output_length": 17, "hash_ids": [0, 3014, 3015, 3016, 3017, 3018, 3019, 3020, 3021, 3022, 3023, 3024, 3025, 3026, 3027, 3028, 3029, 3030, 3031, 3032, 3033, 3034, 3035, 3036, 3037, 3038, 3039]} +{"timestamp": 39000, "input_length": 13689, "output_length": 358, "hash_ids": [0, 3040, 3041, 3042, 3043, 3044, 3045, 3046, 3047, 3048, 3049, 3050, 3051, 3052, 3053, 3054, 3055, 3056, 3057, 3058, 3059, 3060, 3061, 3062, 3063, 3064, 3065]} +{"timestamp": 39000, "input_length": 16464, "output_length": 182, "hash_ids": [0, 3066, 3067, 3068, 3069, 3070, 3071, 3072, 3073, 3074, 3075, 3076, 3077, 3078, 3079, 3080, 3081, 3082, 3083, 3084, 3085, 3086, 3087, 3088, 3089, 3090, 3091, 3092, 3093, 3094, 3095, 3096, 3097]} +{"timestamp": 39000, "input_length": 22808, "output_length": 463, "hash_ids": [0, 3098, 3099, 3100, 3101, 3102, 3103, 3104, 3105, 3106, 3107, 3108, 3109, 3110, 3111, 3112, 3113, 3114, 3115, 3116, 3117, 3118, 3119, 3120, 3121, 3122, 3123, 3124, 3125, 3126, 3127, 3128, 3129, 3130, 3131, 3132, 3133, 3134, 3135, 3136, 3137, 3138, 3139, 3140, 3141]} +{"timestamp": 39000, "input_length": 8487, "output_length": 306, "hash_ids": [0, 3142, 3143, 3144, 3145, 3146, 3147, 3148, 3149, 3150, 3151, 3152, 3153, 3154, 3155, 3156, 3157]} +{"timestamp": 39000, "input_length": 9227, "output_length": 160, "hash_ids": [0, 3158, 3159, 3160, 3161, 3162, 3163, 3164, 3165, 3166, 3167, 3168, 3169, 3170, 3171, 3172, 3173, 3174, 3175]} +{"timestamp": 39000, "input_length": 3535, "output_length": 228, "hash_ids": [0, 3176, 3177, 3178, 3179, 3180, 3181]} +{"timestamp": 39000, "input_length": 1206, "output_length": 223, "hash_ids": [0, 3182, 3183]} +{"timestamp": 39000, "input_length": 8520, "output_length": 232, "hash_ids": [0, 3184, 3185, 3186, 3187, 3188, 3189, 3190, 3191, 3192, 3193, 3194, 3195, 3196, 3197, 3198, 3199]} +{"timestamp": 39000, "input_length": 897, "output_length": 574, "hash_ids": [0, 3200]} +{"timestamp": 39000, "input_length": 23379, "output_length": 616, "hash_ids": [0, 3201, 3202, 3203, 3204, 3205, 3206, 3207, 3208, 3209, 3210, 3211, 3212, 3213, 3214, 3215, 3216, 3217, 3218, 3219, 3220, 3221, 3222, 3223, 3224, 3225, 3226, 3227, 3228, 3229, 3230, 3231, 3232, 3233, 3234, 3235, 3236, 3237, 3238, 3239, 3240, 3241, 3242, 3243, 3244, 3245]} +{"timestamp": 41999, "input_length": 87172, "output_length": 7, "hash_ids": [0, 3246, 3247, 3248, 3249, 3250, 3251, 3252, 3253, 3254, 3255, 3256, 3257, 3258, 3259, 3260, 3261, 3262, 3263, 3264, 3265, 3266, 3267, 3268, 3269, 3270, 3271, 3272, 3273, 3274, 3275, 3276, 3277, 3278, 3279, 3280, 3281, 3282, 3283, 3284, 3285, 3286, 3287, 3288, 3289, 3290, 3291, 3292, 3293, 3294, 3295, 3296, 3297, 3298, 3299, 3300, 3301, 3302, 3303, 3304, 3305, 3306, 3307, 3308, 3309, 3310, 3311, 3312, 3313, 3314, 3315, 3316, 3317, 3318, 3319, 3320, 3321, 3322, 3323, 3324, 3325, 3326, 3327, 3328, 3329, 3330, 3331, 3332, 3333, 3334, 3335, 3336, 3337, 3338, 3339, 3340, 3341, 3342, 3343, 3344, 3345, 3346, 3347, 3348, 3349, 3350, 3351, 3352, 3353, 3354, 3355, 3356, 3357, 3358, 3359, 3360, 3361, 3362, 3363, 3364, 3365, 3366, 3367, 3368, 3369, 3370, 3371, 3372, 3373, 3374, 3375, 3376, 3377, 3378, 3379, 3380, 3381, 3382, 3383, 3384, 3385, 3386, 3387, 3388, 3389, 3390, 3391, 3392, 3393, 3394, 3395, 3396, 3397, 3398, 3399, 3400, 3401, 3402, 3403, 3404, 3405, 3406, 3407, 3408, 3409, 3410, 3411, 3412, 3413, 3414, 3415]} +{"timestamp": 41999, "input_length": 5164, "output_length": 494, "hash_ids": [0, 3416, 3417, 3418, 3419, 3420, 3421, 3422, 3423, 3424, 3425]} +{"timestamp": 41999, "input_length": 6053, "output_length": 89, "hash_ids": [0, 3426, 3427, 3428, 3429, 3430, 3431, 3432, 3433, 3434, 3435, 3436]} +{"timestamp": 41999, "input_length": 15454, "output_length": 338, "hash_ids": [0, 3437, 3438, 3439, 3440, 3441, 3442, 3443, 3444, 3445, 3446, 3447, 3448, 3449, 3450, 3451, 3452, 3453, 3454, 3455, 3456, 3457, 3458, 3459, 3460, 3461, 3462, 3463, 3464, 3465, 3466]} +{"timestamp": 41999, "input_length": 69645, "output_length": 400, "hash_ids": [0, 3467, 3468, 3469, 3470, 3471, 3472, 3473, 3474, 3475, 3476, 3477, 3478, 3479, 3480, 3481, 3482, 3483, 3484, 3485, 3486, 3487, 3488, 3489, 3490, 3491, 3492, 3493, 3494, 3495, 3496, 3497, 3498, 3499, 3500, 3501, 3502, 3503, 3504, 3505, 3506, 3507, 3508, 3509, 3510, 3511, 3512, 3513, 3514, 3515, 3516, 3517, 3518, 3519, 3520, 3521, 3522, 3523, 3524, 3525, 3526, 3527, 3528, 3529, 3530, 3531, 3532, 3533, 3534, 3535, 3536, 3537, 3538, 3539, 3540, 3541, 3542, 3543, 3544, 3545, 3546, 3547, 3548, 3549, 3550, 3551, 3552, 3553, 3554, 3555, 3556, 3557, 3558, 3559, 3560, 3561, 3562, 3563, 3564, 3565, 3566, 3567, 3568, 3569, 3570, 3571, 3572, 3573, 3574, 3575, 3576, 3577, 3578, 3579, 3580, 3581, 3582, 3583, 3584, 3585, 3586, 3587, 3588, 3589, 3590, 3591, 3592, 3593, 3594, 3595, 3596, 3597, 3598, 3599, 3600, 3601, 3602]} +{"timestamp": 41999, "input_length": 13183, "output_length": 566, "hash_ids": [0, 3603, 3604, 3605, 3606, 3607, 3608, 3609, 3610, 3611, 3612, 3613, 3614, 3615, 3616, 3617, 3618, 3619, 3620, 3621, 3622, 3623, 3624, 3625, 3626, 3627]} +{"timestamp": 41999, "input_length": 1895, "output_length": 456, "hash_ids": [0, 3628, 3629, 3630]} +{"timestamp": 45000, "input_length": 907, "output_length": 396, "hash_ids": [0, 3631]} +{"timestamp": 45000, "input_length": 896, "output_length": 439, "hash_ids": [0, 3632]} +{"timestamp": 45000, "input_length": 27152, "output_length": 216, "hash_ids": [0, 3633, 3634, 3635, 3636, 3637, 3638, 3639, 3640, 3641, 3642, 3643, 3644, 3645, 3646, 3647, 3648, 3649, 3650, 3651, 3652, 3653, 3654, 3655, 3656, 3657, 3658, 3659, 3660, 3661, 3662, 3663, 3664, 3665, 3666, 3667, 3668, 3669, 3670, 3671, 3672, 3673, 3674, 3675, 3676, 3677, 3678, 3679, 3680, 3681, 3682, 3683, 3684, 3685]} +{"timestamp": 45000, "input_length": 6326, "output_length": 28, "hash_ids": [0, 3686, 3687, 3688, 3689, 3690, 3691, 3692, 3693, 3694, 3695, 3696, 3697]} +{"timestamp": 45000, "input_length": 2158, "output_length": 604, "hash_ids": [0, 3698, 3699, 3700, 3701]} +{"timestamp": 45000, "input_length": 11244, "output_length": 453, "hash_ids": [0, 3702, 3703, 3704, 3705, 3706, 3707, 3708, 3709, 3710, 3711, 3712, 3713, 3714, 3715, 3716, 3717, 3718, 3719, 3720, 3721, 3722]} +{"timestamp": 48000, "input_length": 15418, "output_length": 534, "hash_ids": [0, 3723, 3724, 3725, 3726, 3727, 3728, 3729, 3730, 3731, 3732, 3733, 3734, 3735, 3736, 3737, 3738, 3739, 3740, 3741, 3742, 3743, 3744, 3745, 3746, 3747, 3748, 3749, 3750, 3751, 3752]} +{"timestamp": 48000, "input_length": 3024, "output_length": 111, "hash_ids": [0, 1541, 1542, 1543, 1544, 3753]} +{"timestamp": 48000, "input_length": 49948, "output_length": 173, "hash_ids": [0, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 3754, 3755, 3756, 3757, 3758, 3759, 3760, 3761, 3762, 3763, 3764, 3765, 3766, 3767, 3768, 3769, 3770, 3771, 3772, 3773, 3774, 3775, 3776, 3777, 3778, 3779, 3780, 3781, 3782, 3783, 3784, 3785, 3786, 3787, 3788, 3789, 3790, 3791, 3792, 3793, 3794, 3795, 3796, 3797, 3798, 3799, 3800, 3801, 3802, 3803, 3804, 3805, 3806, 3807, 3808, 3809, 3810, 3811, 3812, 3813, 3814, 3815, 3816, 3817, 3818, 3819, 3820, 3821, 3822, 3823, 3824, 3825]} +{"timestamp": 48000, "input_length": 914, "output_length": 387, "hash_ids": [0, 3826]} +{"timestamp": 48000, "input_length": 21279, "output_length": 111, "hash_ids": [0, 3827, 3828, 3829, 3830, 3831, 3832, 3833, 3834, 3835, 3836, 3837, 3838, 3839, 3840, 3841, 3842, 3843, 3844, 3845, 3846, 3847, 3848, 3849, 3850, 3851, 3852, 3853, 3854, 3855, 3856, 3857, 3858, 3859, 3860, 3861, 3862, 3863, 3864, 3865, 3866, 3867]} +{"timestamp": 48000, "input_length": 7833, "output_length": 374, "hash_ids": [0, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 3868, 3869]} +{"timestamp": 48000, "input_length": 2014, "output_length": 10, "hash_ids": [0, 3870, 3871, 3872]} +{"timestamp": 48000, "input_length": 945, "output_length": 536, "hash_ids": [0, 3873]} +{"timestamp": 48000, "input_length": 2986, "output_length": 436, "hash_ids": [0, 3874, 3875, 3876, 3877, 3878]} +{"timestamp": 51000, "input_length": 1083, "output_length": 481, "hash_ids": [0, 3879, 3880]} +{"timestamp": 51000, "input_length": 13242, "output_length": 434, "hash_ids": [0, 3881, 3882, 3883, 3884, 3885, 3886, 3887, 3888, 3889, 3890, 3891, 3892, 3893, 3894, 3895, 3896, 3897, 3898, 3899, 3900, 3901, 3902, 3903, 3904, 3905]} +{"timestamp": 51000, "input_length": 41053, "output_length": 575, "hash_ids": [0, 3906, 3907, 3908, 3909, 3910, 3911, 3912, 3913, 3914, 3915, 3916, 3917, 3918, 3919, 3920, 3921, 3922, 3923, 3924, 3925, 3926, 3927, 3928, 3929, 3930, 3931, 3932, 3933, 3934, 3935, 3936, 3937, 3938, 3939, 3940, 3941, 3942, 3943, 3944, 3945, 3946, 3947, 3948, 3949, 3950, 3951, 3952, 3953, 3954, 3955, 3956, 3957, 3958, 3959, 3960, 3961, 3962, 3963, 3964, 3965, 3966, 3967, 3968, 3969, 3970, 3971, 3972, 3973, 3974, 3975, 3976, 3977, 3978, 3979, 3980, 3981, 3982, 3983, 3984, 3985]} +{"timestamp": 51000, "input_length": 24621, "output_length": 362, "hash_ids": [0, 3986, 3987, 3988, 3989, 3990, 3991, 3992, 3993, 3994, 3995, 3996, 3997, 3998, 3999, 4000, 4001, 4002, 4003, 4004, 4005, 4006, 4007, 4008, 4009, 4010, 4011, 4012, 4013, 4014, 4015, 4016, 4017, 4018, 4019, 4020, 4021, 4022, 4023, 4024, 4025, 4026, 4027, 4028, 4029, 4030, 4031, 4032, 4033]} +{"timestamp": 51000, "input_length": 898, "output_length": 906, "hash_ids": [0, 4034]} +{"timestamp": 51000, "input_length": 1085, "output_length": 75, "hash_ids": [0, 4035, 4036]} +{"timestamp": 51000, "input_length": 5654, "output_length": 442, "hash_ids": [0, 4037, 4038, 4039, 4040, 4041, 4042, 4043, 4044, 4045, 4046, 4047]} +{"timestamp": 51000, "input_length": 5939, "output_length": 15, "hash_ids": [0, 4048, 4049, 4050, 4051, 4052, 4053, 4054, 4055, 4056, 4057, 4058]} +{"timestamp": 54000, "input_length": 5100, "output_length": 413, "hash_ids": [0, 4059, 4060, 4061, 4062, 4063, 4064, 4065, 4066, 4067]} +{"timestamp": 54000, "input_length": 9421, "output_length": 17, "hash_ids": [0, 4068, 4069, 4070, 4071, 4072, 4073, 4074, 4075, 4076, 4077, 4078, 4079, 4080, 4081, 4082, 4083, 4084, 4085]} +{"timestamp": 54000, "input_length": 2187, "output_length": 186, "hash_ids": [0, 4086, 4087, 4088, 4089]} +{"timestamp": 54000, "input_length": 10704, "output_length": 346, "hash_ids": [0, 4090, 4091, 4092, 4093, 4094, 4095, 4096, 4097, 4098, 4099, 4100, 4101, 4102, 4103, 4104, 4105, 4106, 4107, 4108, 4109]} +{"timestamp": 54000, "input_length": 9359, "output_length": 339, "hash_ids": [0, 4110, 4111, 4112, 4113, 4114, 4115, 4116, 4117, 4118, 4119, 4120, 4121, 4122, 4123, 4124, 4125, 4126, 4127]} +{"timestamp": 54000, "input_length": 976, "output_length": 237, "hash_ids": [0, 4128]} +{"timestamp": 54000, "input_length": 18720, "output_length": 109, "hash_ids": [0, 4129, 4130, 4131, 4132, 4133, 4134, 4135, 4136, 4137, 4138, 4139, 4140, 4141, 4142, 4143, 4144, 4145, 4146, 4147, 4148, 4149, 4150, 4151, 4152, 4153, 4154, 4155, 4156, 4157, 4158, 4159, 4160, 4161, 4162, 4163, 4164]} +{"timestamp": 57000, "input_length": 2016, "output_length": 302, "hash_ids": [0, 4165, 4166, 4167]} +{"timestamp": 57000, "input_length": 6554, "output_length": 254, "hash_ids": [0, 4168, 4169, 4170, 4171, 4172, 4173, 4174, 4175, 4176, 4177, 4178, 4179]} +{"timestamp": 57000, "input_length": 917, "output_length": 21, "hash_ids": [0, 4180]} +{"timestamp": 57000, "input_length": 5682, "output_length": 35, "hash_ids": [0, 4181, 4182, 4183, 4184, 4185, 4186, 4187, 4188, 4189, 4190, 4191]} +{"timestamp": 57000, "input_length": 1902, "output_length": 720, "hash_ids": [0, 4192, 4193, 4194]} +{"timestamp": 57000, "input_length": 1383, "output_length": 722, "hash_ids": [0, 4195, 4196]} +{"timestamp": 60000, "input_length": 893, "output_length": 449, "hash_ids": [0, 4197]} +{"timestamp": 60000, "input_length": 11383, "output_length": 337, "hash_ids": [0, 4198, 4199, 4200, 4201, 4202, 4203, 4204, 4205, 4206, 4207, 4208, 4209, 4210, 4211, 4212, 4213, 4214, 4215, 4216, 4217, 4218, 4219]} +{"timestamp": 60000, "input_length": 2717, "output_length": 359, "hash_ids": [0, 4220, 4221, 4222, 4223, 4224]} +{"timestamp": 60000, "input_length": 12849, "output_length": 3, "hash_ids": [0, 4225, 4226, 4227, 4228, 4229, 4230, 4231, 4232, 4233, 4234, 4235, 4236, 4237, 4238, 4239, 4240, 4241, 4242, 4243, 4244, 4245, 4246, 4247, 4248, 4249]} +{"timestamp": 63000, "input_length": 19878, "output_length": 161, "hash_ids": [0, 1497, 1498, 1499, 1500, 1501, 1502, 1503, 1504, 1505, 1506, 1507, 1508, 1509, 1510, 1511, 1512, 1513, 1514, 1515, 1516, 1517, 1518, 1519, 1520, 1521, 1522, 1523, 1524, 1525, 1526, 1527, 1528, 1529, 1530, 1531, 1532, 1533, 4250]} +{"timestamp": 63000, "input_length": 895, "output_length": 133, "hash_ids": [0, 4251]} +{"timestamp": 63000, "input_length": 18306, "output_length": 575, "hash_ids": [0, 4252, 4253, 4254, 4255, 4256, 4257, 4258, 4259, 4260, 4261, 4262, 4263, 4264, 4265, 4266, 4267, 4268, 4269, 4270, 4271, 4272, 4273, 4274, 4275, 4276, 4277, 4278, 4279, 4280, 4281, 4282, 4283, 4284, 4285, 4286]} +{"timestamp": 63000, "input_length": 21641, "output_length": 467, "hash_ids": [0, 4287, 4288, 4289, 4290, 4291, 4292, 4293, 4294, 4295, 4296, 4297, 4298, 4299, 4300, 4301, 4302, 4303, 4304, 4305, 4306, 4307, 4308, 4309, 4310, 4311, 4312, 4313, 4314, 4315, 4316, 4317, 4318, 4319, 4320, 4321, 4322, 4323, 4324, 4325, 4326, 4327, 4328]} +{"timestamp": 63000, "input_length": 7103, "output_length": 48, "hash_ids": [0, 4329, 4330, 4331, 4332, 4333, 4334, 4335, 4336, 4337, 4338, 4339, 4340, 4341]} +{"timestamp": 65999, "input_length": 1529, "output_length": 513, "hash_ids": [0, 4342, 4343]} +{"timestamp": 65999, "input_length": 10938, "output_length": 247, "hash_ids": [0, 4344, 4345, 4346, 4347, 4348, 4349, 4350, 4351, 4352, 4353, 4354, 4355, 4356, 4357, 4358, 4359, 4360, 4361, 4362, 4363, 4364]} +{"timestamp": 65999, "input_length": 11954, "output_length": 463, "hash_ids": [0, 4365, 4366, 4367, 4368, 4369, 4370, 4371, 4372, 4373, 4374, 4375, 4376, 4377, 4378, 4379, 4380, 4381, 4382, 4383, 4384, 4385, 4386, 4387]} +{"timestamp": 65999, "input_length": 979, "output_length": 427, "hash_ids": [0, 4388]} +{"timestamp": 65999, "input_length": 7446, "output_length": 319, "hash_ids": [0, 4389, 4390, 4391, 4392, 4393, 4394, 4395, 4396, 4397, 4398, 4399, 4400, 4401, 4402]} +{"timestamp": 65999, "input_length": 1073, "output_length": 481, "hash_ids": [0, 4403, 4404]} +{"timestamp": 65999, "input_length": 14315, "output_length": 364, "hash_ids": [0, 1030, 1031, 1032, 1033, 1034, 1035, 1036, 1037, 1038, 1039, 1040, 1041, 1042, 1043, 1044, 1045, 1046, 4405, 4406, 4407, 4408, 4409, 4410, 4411, 4412, 4413, 4414]} +{"timestamp": 65999, "input_length": 102869, "output_length": 401, "hash_ids": [0, 4415, 4416, 4417, 4418, 4419, 4420, 4421, 4422, 4423, 4424, 4425, 4426, 4427, 4428, 4429, 4430, 4431, 4432, 4433, 4434, 4435, 4436, 4437, 4438, 4439, 4440, 4441, 4442, 4443, 4444, 4445, 4446, 4447, 4448, 4449, 4450, 4451, 4452, 4453, 4454, 4455, 4456, 4457, 4458, 4459, 4460, 4461, 4462, 4463, 4464, 4465, 4466, 4467, 4468, 4469, 4470, 4471, 4472, 4473, 4474, 4475, 4476, 4477, 4478, 4479, 4480, 4481, 4482, 4483, 4484, 4485, 4486, 4487, 4488, 4489, 4490, 4491, 4492, 4493, 4494, 4495, 4496, 4497, 4498, 4499, 4500, 4501, 4502, 4503, 4504, 4505, 4506, 4507, 4508, 4509, 4510, 4511, 4512, 4513, 4514, 4515, 4516, 4517, 4518, 4519, 4520, 4521, 4522, 4523, 4524, 4525, 4526, 4527, 4528, 4529, 4530, 4531, 4532, 4533, 4534, 4535, 4536, 4537, 4538, 4539, 4540, 4541, 4542, 4543, 4544, 4545, 4546, 4547, 4548, 4549, 4550, 4551, 4552, 4553, 4554, 4555, 4556, 4557, 4558, 4559, 4560, 4561, 4562, 4563, 4564, 4565, 4566, 4567, 4568, 4569, 4570, 4571, 4572, 4573, 4574, 4575, 4576, 4577, 4578, 4579, 4580, 4581, 4582, 4583, 4584, 4585, 4586, 4587, 4588, 4589, 4590, 4591, 4592, 4593, 4594, 4595, 4596, 4597, 4598, 4599, 4600, 4601, 4602, 4603, 4604, 4605, 4606, 4607, 4608, 4609, 4610, 4611, 4612, 4613, 4614]} +{"timestamp": 65999, "input_length": 75501, "output_length": 531, "hash_ids": [0, 4615, 4616, 4617, 4618, 4619, 4620, 4621, 4622, 4623, 4624, 4625, 4626, 4627, 4628, 4629, 4630, 4631, 4632, 4633, 4634, 4635, 4636, 4637, 4638, 4639, 4640, 4641, 4642, 4643, 4644, 4645, 4646, 4647, 4648, 4649, 4650, 4651, 4652, 4653, 4654, 4655, 4656, 4657, 4658, 4659, 4660, 4661, 4662, 4663, 4664, 4665, 4666, 4667, 4668, 4669, 4670, 4671, 4672, 4673, 4674, 4675, 4676, 4677, 4678, 4679, 4680, 4681, 4682, 4683, 4684, 4685, 4686, 4687, 4688, 4689, 4690, 4691, 4692, 4693, 4694, 4695, 4696, 4697, 4698, 4699, 4700, 4701, 4702, 4703, 4704, 4705, 4706, 4707, 4708, 4709, 4710, 4711, 4712, 4713, 4714, 4715, 4716, 4717, 4718, 4719, 4720, 4721, 4722, 4723, 4724, 4725, 4726, 4727, 4728, 4729, 4730, 4731, 4732, 4733, 4734, 4735, 4736, 4737, 4738, 4739, 4740, 4741, 4742, 4743, 4744, 4745, 4746, 4747, 4748, 4749, 4750, 4751, 4752, 4753, 4754, 4755, 4756, 4757, 4758, 4759, 4760, 4761]} +{"timestamp": 65999, "input_length": 15233, "output_length": 300, "hash_ids": [0, 978, 979, 980, 981, 982, 983, 984, 985, 986, 987, 988, 989, 990, 991, 992, 993, 994, 995, 996, 997, 998, 999, 1000, 1001, 1002, 1003, 4762, 4763, 4764]} +{"timestamp": 69000, "input_length": 10116, "output_length": 509, "hash_ids": [0, 4765, 4766, 4767, 4768, 4769, 4770, 4771, 4772, 4773, 4774, 4775, 4776, 4777, 4778, 4779, 4780, 4781, 4782, 4783]} +{"timestamp": 69000, "input_length": 4885, "output_length": 353, "hash_ids": [0, 4784, 4785, 4786, 4787, 4788, 4789, 4790, 4791, 4792]} +{"timestamp": 69000, "input_length": 4577, "output_length": 225, "hash_ids": [0, 4793, 4794, 4795, 4796, 4797, 4798, 4799, 4800]} +{"timestamp": 69000, "input_length": 893, "output_length": 334, "hash_ids": [0, 4801]} +{"timestamp": 69000, "input_length": 20509, "output_length": 5, "hash_ids": [0, 4802, 4803, 4804, 4805, 4806, 4807, 4808, 4809, 4810, 4811, 4812, 4813, 4814, 4815, 4816, 4817, 4818, 4819, 4820, 4821, 4822, 4823, 4824, 4825, 4826, 4827, 4828, 4829, 4830, 4831, 4832, 4833, 4834, 4835, 4836, 4837, 4838, 4839, 4840, 4841]} +{"timestamp": 69000, "input_length": 9921, "output_length": 356, "hash_ids": [0, 4842, 4843, 4844, 4845, 4846, 4847, 4848, 4849, 4850, 4851, 4852, 4853, 4854, 4855, 4856, 4857, 4858, 4859, 4860]} +{"timestamp": 69000, "input_length": 1176, "output_length": 120, "hash_ids": [0, 4861, 4862]} +{"timestamp": 69000, "input_length": 9271, "output_length": 306, "hash_ids": [0, 4863, 4864, 4865, 4866, 4867, 4868, 4869, 4870, 4871, 4872, 4873, 4874, 4875, 4876, 4877, 4878, 4879, 4880]} +{"timestamp": 69000, "input_length": 71938, "output_length": 433, "hash_ids": [0, 4881, 4882, 4883, 4884, 4885, 4886, 4887, 4888, 4889, 4890, 4891, 4892, 4893, 4894, 4895, 4896, 4897, 4898, 4899, 4900, 4901, 4902, 4903, 4904, 4905, 4906, 4907, 4908, 4909, 4910, 4911, 4912, 4913, 4914, 4915, 4916, 4917, 4918, 4919, 4920, 4921, 4922, 4923, 4924, 4925, 4926, 4927, 4928, 4929, 4930, 4931, 4932, 4933, 4934, 4935, 4936, 4937, 4938, 4939, 4940, 4941, 4942, 4943, 4944, 4945, 4946, 4947, 4948, 4949, 4950, 4951, 4952, 4953, 4954, 4955, 4956, 4957, 4958, 4959, 4960, 4961, 4962, 4963, 4964, 4965, 4966, 4967, 4968, 4969, 4970, 4971, 4972, 4973, 4974, 4975, 4976, 4977, 4978, 4979, 4980, 4981, 4982, 4983, 4984, 4985, 4986, 4987, 4988, 4989, 4990, 4991, 4992, 4993, 4994, 4995, 4996, 4997, 4998, 4999, 5000, 5001, 5002, 5003, 5004, 5005, 5006, 5007, 5008, 5009, 5010, 5011, 5012, 5013, 5014, 5015, 5016, 5017, 5018, 5019, 5020]} +{"timestamp": 69000, "input_length": 1341, "output_length": 128, "hash_ids": [0, 5021, 5022]} +{"timestamp": 69000, "input_length": 2103, "output_length": 508, "hash_ids": [0, 820, 5023, 5024, 5025]} +{"timestamp": 69000, "input_length": 16554, "output_length": 725, "hash_ids": [0, 5026, 5027, 5028, 5029, 5030, 5031, 5032, 5033, 5034, 5035, 5036, 5037, 5038, 5039, 5040, 5041, 5042, 5043, 5044, 5045, 5046, 5047, 5048, 5049, 5050, 5051, 5052, 5053, 5054, 5055, 5056, 5057]} +{"timestamp": 72000, "input_length": 17027, "output_length": 904, "hash_ids": [0, 5058, 5059, 5060, 5061, 5062, 5063, 5064, 5065, 5066, 5067, 5068, 5069, 5070, 5071, 5072, 5073, 5074, 5075, 5076, 5077, 5078, 5079, 5080, 5081, 5082, 5083, 5084, 5085, 5086, 5087, 5088, 5089, 5090]} +{"timestamp": 72000, "input_length": 1625, "output_length": 578, "hash_ids": [0, 5091, 5092, 5093]} +{"timestamp": 72000, "input_length": 1535, "output_length": 356, "hash_ids": [0, 5094, 5095]} +{"timestamp": 72000, "input_length": 8450, "output_length": 19, "hash_ids": [0, 5096, 5097, 5098, 5099, 5100, 5101, 5102, 5103, 5104, 5105, 5106, 5107, 5108, 5109, 5110, 5111]} +{"timestamp": 72000, "input_length": 16894, "output_length": 431, "hash_ids": [0, 5112, 5113, 5114, 5115, 5116, 5117, 5118, 5119, 5120, 5121, 5122, 5123, 5124, 5125, 5126, 5127, 5128, 5129, 5130, 5131, 5132, 5133, 5134, 5135, 5136, 5137, 5138, 5139, 5140, 5141, 5142, 5143]} +{"timestamp": 72000, "input_length": 16611, "output_length": 24, "hash_ids": [0, 5144, 5145, 5146, 5147, 5148, 5149, 5150, 5151, 5152, 5153, 5154, 5155, 5156, 5157, 5158, 5159, 5160, 5161, 5162, 5163, 5164, 5165, 5166, 5167, 5168, 5169, 5170, 5171, 5172, 5173, 5174, 5175]} +{"timestamp": 72000, "input_length": 19978, "output_length": 448, "hash_ids": [0, 5176, 5177, 5178, 5179, 5180, 5181, 5182, 5183, 5184, 5185, 5186, 5187, 5188, 5189, 5190, 5191, 5192, 5193, 5194, 5195, 5196, 5197, 5198, 5199, 5200, 5201, 5202, 5203, 5204, 5205, 5206, 5207, 5208, 5209, 5210, 5211, 5212, 5213, 5214]} +{"timestamp": 72000, "input_length": 1577, "output_length": 579, "hash_ids": [0, 5215, 5216, 5217]} +{"timestamp": 72000, "input_length": 9550, "output_length": 713, "hash_ids": [0, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, 456, 457, 458, 459, 460, 5218]} +{"timestamp": 72000, "input_length": 6517, "output_length": 565, "hash_ids": [0, 5219, 5220, 5221, 5222, 5223, 5224, 5225, 5226, 5227, 5228, 5229, 5230]} +{"timestamp": 74999, "input_length": 13538, "output_length": 480, "hash_ids": [0, 5231, 5232, 5233, 5234, 5235, 5236, 5237, 5238, 5239, 5240, 5241, 5242, 5243, 5244, 5245, 5246, 5247, 5248, 5249, 5250, 5251, 5252, 5253, 5254, 5255, 5256]} +{"timestamp": 74999, "input_length": 898, "output_length": 286, "hash_ids": [0, 5257]} +{"timestamp": 74999, "input_length": 6299, "output_length": 413, "hash_ids": [0, 5258, 5259, 5260, 5261, 5262, 5263, 5264, 5265, 5266, 5267, 5268, 5269]} +{"timestamp": 74999, "input_length": 8770, "output_length": 121, "hash_ids": [0, 5270, 5271, 5272, 5273, 5274, 5275, 5276, 5277, 5278, 5279, 5280, 5281, 5282, 5283, 5284, 5285, 5286]} +{"timestamp": 74999, "input_length": 38149, "output_length": 316, "hash_ids": [0, 5287, 5288, 5289, 5290, 5291, 5292, 5293, 5294, 5295, 5296, 5297, 5298, 5299, 5300, 5301, 5302, 5303, 5304, 5305, 5306, 5307, 5308, 5309, 5310, 5311, 5312, 5313, 5314, 5315, 5316, 5317, 5318, 5319, 5320, 5321, 5322, 5323, 5324, 5325, 5326, 5327, 5328, 5329, 5330, 5331, 5332, 5333, 5334, 5335, 5336, 5337, 5338, 5339, 5340, 5341, 5342, 5343, 5344, 5345, 5346, 5347, 5348, 5349, 5350, 5351, 5352, 5353, 5354, 5355, 5356, 5357, 5358, 5359, 5360]} +{"timestamp": 74999, "input_length": 5095, "output_length": 476, "hash_ids": [0, 5361, 5362, 5363, 5364, 5365, 5366, 5367, 5368, 5369]} +{"timestamp": 74999, "input_length": 6994, "output_length": 461, "hash_ids": [0, 5370, 5371, 5372, 5373, 5374, 5375, 5376, 5377, 5378, 5379, 5380, 5381, 5382]} +{"timestamp": 74999, "input_length": 37879, "output_length": 532, "hash_ids": [0, 5383, 5384, 5385, 5386, 5387, 5388, 5389, 5390, 5391, 5392, 5393, 5394, 5395, 5396, 5397, 5398, 5399, 5400, 5401, 5402, 5403, 5404, 5405, 5406, 5407, 5408, 5409, 5410, 5411, 5412, 5413, 5414, 5415, 5416, 5417, 5418, 5419, 5420, 5421, 5422, 5423, 5424, 5425, 5426, 5427, 5428, 5429, 5430, 5431, 5432, 5433, 5434, 5435, 5436, 5437, 5438, 5439, 5440, 5441, 5442, 5443, 5444, 5445, 5446, 5447, 5448, 5449, 5450, 5451, 5452, 5453, 5454, 5455]} +{"timestamp": 74999, "input_length": 17583, "output_length": 464, "hash_ids": [0, 5456, 5457, 5458, 5459, 5460, 5461, 5462, 5463, 5464, 5465, 5466, 5467, 5468, 5469, 5470, 5471, 5472, 5473, 5474, 5475, 5476, 5477, 5478, 5479, 5480, 5481, 5482, 5483, 5484, 5485, 5486, 5487, 5488, 5489]} +{"timestamp": 74999, "input_length": 17779, "output_length": 576, "hash_ids": [0, 5490, 5491, 5492, 5493, 5494, 5495, 5496, 5497, 5498, 5499, 5500, 5501, 5502, 5503, 5504, 5505, 5506, 5507, 5508, 5509, 5510, 5511, 5512, 5513, 5514, 5515, 5516, 5517, 5518, 5519, 5520, 5521, 5522, 5523]} +{"timestamp": 74999, "input_length": 14349, "output_length": 501, "hash_ids": [0, 5524, 5525, 5526, 5527, 5528, 5529, 5530, 5531, 5532, 5533, 5534, 5535, 5536, 5537, 5538, 5539, 5540, 5541, 5542, 5543, 5544, 5545, 5546, 5547, 5548, 5549, 5550, 5551]} +{"timestamp": 74999, "input_length": 8639, "output_length": 835, "hash_ids": [0, 5552, 5553, 5554, 5555, 5556, 5557, 5558, 5559, 5560, 5561, 5562, 5563, 5564, 5565, 5566, 5567]} +{"timestamp": 74999, "input_length": 921, "output_length": 452, "hash_ids": [0, 5568]} +{"timestamp": 74999, "input_length": 965, "output_length": 637, "hash_ids": [0, 5569]} +{"timestamp": 74999, "input_length": 13996, "output_length": 315, "hash_ids": [0, 5570, 5571, 5572, 5573, 5574, 5575, 5576, 5577, 5578, 5579, 5580, 5581, 5582, 5583, 5584, 5585, 5586, 5587, 5588, 5589, 5590, 5591, 5592, 5593, 5594, 5595, 5596]} +{"timestamp": 74999, "input_length": 6549, "output_length": 442, "hash_ids": [0, 611, 612, 613, 614, 615, 616, 617, 618, 619, 620, 5597, 5598]} +{"timestamp": 78000, "input_length": 51186, "output_length": 620, "hash_ids": [0, 5599, 5600, 5601, 5602, 5603, 5604, 5605, 5606, 5607, 5608, 5609, 5610, 5611, 5612, 5613, 5614, 5615, 5616, 5617, 5618, 5619, 5620, 5621, 5622, 5623, 5624, 5625, 5626, 5627, 5628, 5629, 5630, 5631, 5632, 5633, 5634, 5635, 5636, 5637, 5638, 5639, 5640, 5641, 5642, 5643, 5644, 5645, 5646, 5647, 5648, 5649, 5650, 5651, 5652, 5653, 5654, 5655, 5656, 5657, 5658, 5659, 5660, 5661, 5662, 5663, 5664, 5665, 5666, 5667, 5668, 5669, 5670, 5671, 5672, 5673, 5674, 5675, 5676, 5677, 5678, 5679, 5680, 5681, 5682, 5683, 5684, 5685, 5686, 5687, 5688, 5689, 5690, 5691, 5692, 5693, 5694, 5695, 5696, 5697]} +{"timestamp": 78000, "input_length": 16982, "output_length": 526, "hash_ids": [0, 655, 656, 657, 658, 659, 660, 661, 662, 663, 664, 665, 666, 667, 668, 669, 670, 671, 672, 673, 674, 675, 676, 677, 678, 679, 680, 681, 682, 5698, 5699, 5700, 5701, 5702]} +{"timestamp": 78000, "input_length": 26356, "output_length": 25, "hash_ids": [0, 5703, 5704, 5705, 5706, 5707, 5708, 5709, 5710, 5711, 5712, 5713, 5714, 5715, 5716, 5717, 5718, 5719, 5720, 5721, 5722, 5723, 5724, 5725, 5726, 5727, 5728, 5729, 5730, 5731, 5732, 5733, 5734, 5735, 5736, 5737, 5738, 5739, 5740, 5741, 5742, 5743, 5744, 5745, 5746, 5747, 5748, 5749, 5750, 5751, 5752, 5753]} +{"timestamp": 78000, "input_length": 45135, "output_length": 109, "hash_ids": [0, 5754, 5755, 5756, 5757, 5758, 5759, 5760, 5761, 5762, 5763, 5764, 5765, 5766, 5767, 5768, 5769, 5770, 5771, 5772, 5773, 5774, 5775, 5776, 5777, 5778, 5779, 5780, 5781, 5782, 5783, 5784, 5785, 5786, 5787, 5788, 5789, 5790, 5791, 5792, 5793, 5794, 5795, 5796, 5797, 5798, 5799, 5800, 5801, 5802, 5803, 5804, 5805, 5806, 5807, 5808, 5809, 5810, 5811, 5812, 5813, 5814, 5815, 5816, 5817, 5818, 5819, 5820, 5821, 5822, 5823, 5824, 5825, 5826, 5827, 5828, 5829, 5830, 5831, 5832, 5833, 5834, 5835, 5836, 5837, 5838, 5839, 5840, 5841]} +{"timestamp": 78000, "input_length": 5395, "output_length": 545, "hash_ids": [0, 5842, 5843, 5844, 5845, 5846, 5847, 5848, 5849, 5850, 5851]} +{"timestamp": 81000, "input_length": 1362, "output_length": 480, "hash_ids": [0, 5852, 5853]} +{"timestamp": 81000, "input_length": 22075, "output_length": 367, "hash_ids": [0, 5854, 5855, 5856, 5857, 5858, 5859, 5860, 5861, 5862, 5863, 5864, 5865, 5866, 5867, 5868, 5869, 5870, 5871, 5872, 5873, 5874, 5875, 5876, 5877, 5878, 5879, 5880, 5881, 5882, 5883, 5884, 5885, 5886, 5887, 5888, 5889, 5890, 5891, 5892, 5893, 5894, 5895, 5896]} +{"timestamp": 81000, "input_length": 6062, "output_length": 4, "hash_ids": [0, 5897, 5898, 5899, 5900, 5901, 5902, 5903, 5904, 5905, 5906, 5907]} +{"timestamp": 81000, "input_length": 1641, "output_length": 567, "hash_ids": [0, 5908, 5909, 5910]} +{"timestamp": 81000, "input_length": 13122, "output_length": 172, "hash_ids": [0, 3158, 3159, 3160, 3161, 5911, 5912, 5913, 5914, 5915, 5916, 5917, 5918, 5919, 5920, 5921, 5922, 5923, 5924, 5925, 5926, 5927, 5928, 5929, 5930, 5931]} +{"timestamp": 81000, "input_length": 4054, "output_length": 547, "hash_ids": [0, 5932, 5933, 5934, 5935, 5936, 5937, 5938]} +{"timestamp": 81000, "input_length": 906, "output_length": 28, "hash_ids": [0, 5939]} +{"timestamp": 81000, "input_length": 3709, "output_length": 300, "hash_ids": [0, 5940, 5941, 5942, 5943, 5944, 5945, 5946]} +{"timestamp": 81000, "input_length": 23998, "output_length": 645, "hash_ids": [0, 5947, 5948, 5949, 5950, 5951, 5952, 5953, 5954, 5955, 5956, 5957, 5958, 5959, 5960, 5961, 5962, 5963, 5964, 5965, 5966, 5967, 5968, 5969, 5970, 5971, 5972, 5973, 5974, 5975, 5976, 5977, 5978, 5979, 5980, 5981, 5982, 5983, 5984, 5985, 5986, 5987, 5988, 5989, 5990, 5991, 5992]} +{"timestamp": 81000, "input_length": 4535, "output_length": 521, "hash_ids": [0, 5993, 5994, 5995, 5996, 5997, 5998, 5999, 6000]} +{"timestamp": 81000, "input_length": 5956, "output_length": 17, "hash_ids": [0, 6001, 6002, 6003, 6004, 6005, 6006, 6007, 6008, 6009, 6010, 6011]} +{"timestamp": 81000, "input_length": 25174, "output_length": 284, "hash_ids": [0, 6012, 6013, 6014, 6015, 6016, 6017, 6018, 6019, 6020, 6021, 6022, 6023, 6024, 6025, 6026, 6027, 6028, 6029, 6030, 6031, 6032, 6033, 6034, 6035, 6036, 6037, 6038, 6039, 6040, 6041, 6042, 6043, 6044, 6045, 6046, 6047, 6048, 6049, 6050, 6051, 6052, 6053, 6054, 6055, 6056, 6057, 6058, 6059, 6060]} +{"timestamp": 84000, "input_length": 7911, "output_length": 510, "hash_ids": [0, 6061, 6062, 6063, 6064, 6065, 6066, 6067, 6068, 6069, 6070, 6071, 6072, 6073, 6074, 6075]} +{"timestamp": 84000, "input_length": 89867, "output_length": 556, "hash_ids": [0, 6076, 6077, 6078, 6079, 6080, 6081, 6082, 6083, 6084, 6085, 6086, 6087, 6088, 6089, 6090, 6091, 6092, 6093, 6094, 6095, 6096, 6097, 6098, 6099, 6100, 6101, 6102, 6103, 6104, 6105, 6106, 6107, 6108, 6109, 6110, 6111, 6112, 6113, 6114, 6115, 6116, 6117, 6118, 6119, 6120, 6121, 6122, 6123, 6124, 6125, 6126, 6127, 6128, 6129, 6130, 6131, 6132, 6133, 6134, 6135, 6136, 6137, 6138, 6139, 6140, 6141, 6142, 6143, 6144, 6145, 6146, 6147, 6148, 6149, 6150, 6151, 6152, 6153, 6154, 6155, 6156, 6157, 6158, 6159, 6160, 6161, 6162, 6163, 6164, 6165, 6166, 6167, 6168, 6169, 6170, 6171, 6172, 6173, 6174, 6175, 6176, 6177, 6178, 6179, 6180, 6181, 6182, 6183, 6184, 6185, 6186, 6187, 6188, 6189, 6190, 6191, 6192, 6193, 6194, 6195, 6196, 6197, 6198, 6199, 6200, 6201, 6202, 6203, 6204, 6205, 6206, 6207, 6208, 6209, 6210, 6211, 6212, 6213, 6214, 6215, 6216, 6217, 6218, 6219, 6220, 6221, 6222, 6223, 6224, 6225, 6226, 6227, 6228, 6229, 6230, 6231, 6232, 6233, 6234, 6235, 6236, 6237, 6238, 6239, 6240, 6241, 6242, 6243, 6244, 6245, 6246, 6247, 6248, 6249, 6250]} +{"timestamp": 84000, "input_length": 6299, "output_length": 165, "hash_ids": [0, 6251, 6252, 6253, 6254, 6255, 6256, 6257, 6258, 6259, 6260, 6261, 6262]} +{"timestamp": 84000, "input_length": 6809, "output_length": 196, "hash_ids": [0, 6263, 6264, 6265, 6266, 6267, 6268, 6269, 6270, 6271, 6272, 6273, 6274, 6275]} +{"timestamp": 84000, "input_length": 7190, "output_length": 26, "hash_ids": [0, 4329, 4330, 4331, 4332, 4333, 4334, 4335, 4336, 4337, 4338, 4339, 4340, 6276, 6277]} +{"timestamp": 84000, "input_length": 20529, "output_length": 362, "hash_ids": [0, 6278, 6279, 6280, 6281, 6282, 6283, 6284, 6285, 6286, 6287, 6288, 6289, 6290, 6291, 6292, 6293, 6294, 6295, 6296, 6297, 6298, 6299, 6300, 6301, 6302, 6303, 6304, 6305, 6306, 6307, 6308, 6309, 6310, 6311, 6312, 6313, 6314, 6315, 6316, 6317]} +{"timestamp": 84000, "input_length": 1372, "output_length": 427, "hash_ids": [0, 6318, 6319]} +{"timestamp": 84000, "input_length": 10089, "output_length": 421, "hash_ids": [0, 6320, 6321, 6322, 6323, 6324, 6325, 6326, 6327, 6328, 6329, 6330, 6331, 6332, 6333, 6334, 6335, 6336, 6337, 6338]} +{"timestamp": 84000, "input_length": 18081, "output_length": 428, "hash_ids": [0, 6339, 6340, 6341, 6342, 6343, 6344, 6345, 6346, 6347, 6348, 6349, 6350, 6351, 6352, 6353, 6354, 6355, 6356, 6357, 6358, 6359, 6360, 6361, 6362, 6363, 6364, 6365, 6366, 6367, 6368, 6369, 6370, 6371, 6372, 6373]} +{"timestamp": 84000, "input_length": 75162, "output_length": 456, "hash_ids": [0, 6374, 6375, 6376, 6377, 6378, 6379, 6380, 6381, 6382, 6383, 6384, 6385, 6386, 6387, 6388, 6389, 6390, 6391, 6392, 6393, 6394, 6395, 6396, 6397, 6398, 6399, 6400, 6401, 6402, 6403, 6404, 6405, 6406, 6407, 6408, 6409, 6410, 6411, 6412, 6413, 6414, 6415, 6416, 6417, 6418, 6419, 6420, 6421, 6422, 6423, 6424, 6425, 6426, 6427, 6428, 6429, 6430, 6431, 6432, 6433, 6434, 6435, 6436, 6437, 6438, 6439, 6440, 6441, 6442, 6443, 6444, 6445, 6446, 6447, 6448, 6449, 6450, 6451, 6452, 6453, 6454, 6455, 6456, 6457, 6458, 6459, 6460, 6461, 6462, 6463, 6464, 6465, 6466, 6467, 6468, 6469, 6470, 6471, 6472, 6473, 6474, 6475, 6476, 6477, 6478, 6479, 6480, 6481, 6482, 6483, 6484, 6485, 6486, 6487, 6488, 6489, 6490, 6491, 6492, 6493, 6494, 6495, 6496, 6497, 6498, 6499, 6500, 6501, 6502, 6503, 6504, 6505, 6506, 6507, 6508, 6509, 6510, 6511, 6512, 6513, 6514, 6515, 6516, 6517, 6518, 6519]} +{"timestamp": 86999, "input_length": 892, "output_length": 78, "hash_ids": [0, 6520]} +{"timestamp": 86999, "input_length": 11404, "output_length": 445, "hash_ids": [0, 2322, 6521, 6522, 6523, 6524, 6525, 6526, 6527, 6528, 6529, 6530, 6531, 6532, 6533, 6534, 6535, 6536, 6537, 6538, 6539, 6540, 6541]} +{"timestamp": 86999, "input_length": 7764, "output_length": 328, "hash_ids": [0, 6542, 6543, 6544, 6545, 6546, 6547, 6548, 6549, 6550, 6551, 6552, 6553, 6554, 6555, 6556]} +{"timestamp": 86999, "input_length": 14794, "output_length": 409, "hash_ids": [0, 6557, 6558, 6559, 6560, 6561, 6562, 6563, 6564, 6565, 6566, 6567, 6568, 6569, 6570, 6571, 6572, 6573, 6574, 6575, 6576, 6577, 6578, 6579, 6580, 6581, 6582, 6583, 6584]} +{"timestamp": 86999, "input_length": 1035, "output_length": 183, "hash_ids": [0, 6585, 6586]} +{"timestamp": 86999, "input_length": 19716, "output_length": 445, "hash_ids": [0, 6587, 6588, 6589, 6590, 6591, 6592, 6593, 6594, 6595, 6596, 6597, 6598, 6599, 6600, 6601, 6602, 6603, 6604, 6605, 6606, 6607, 6608, 6609, 6610, 6611, 6612, 6613, 6614, 6615, 6616, 6617, 6618, 6619, 6620, 6621, 6622, 6623, 6624]} +{"timestamp": 86999, "input_length": 1309, "output_length": 111, "hash_ids": [0, 6625, 6626]} +{"timestamp": 86999, "input_length": 2520, "output_length": 549, "hash_ids": [0, 6627, 6628, 6629, 6630]} +{"timestamp": 86999, "input_length": 16706, "output_length": 445, "hash_ids": [0, 6631, 6632, 6633, 6634, 6635, 6636, 6637, 6638, 6639, 6640, 6641, 6642, 6643, 6644, 6645, 6646, 6647, 6648, 6649, 6650, 6651, 6652, 6653, 6654, 6655, 6656, 6657, 6658, 6659, 6660, 6661, 6662]} +{"timestamp": 86999, "input_length": 1757, "output_length": 431, "hash_ids": [0, 6663, 6664, 6665]} +{"timestamp": 86999, "input_length": 11341, "output_length": 16, "hash_ids": [0, 6666, 6667, 6668, 6669, 6670, 6671, 6672, 6673, 6674, 6675, 6676, 6677, 6678, 6679, 6680, 6681, 6682, 6683, 6684, 6685, 6686, 6687]} +{"timestamp": 86999, "input_length": 906, "output_length": 28, "hash_ids": [0, 6688]} +{"timestamp": 86999, "input_length": 1887, "output_length": 312, "hash_ids": [0, 6689, 6690, 6691]} +{"timestamp": 86999, "input_length": 17869, "output_length": 881, "hash_ids": [0, 6692, 6693, 6694, 6695, 6696, 6697, 6698, 6699, 6700, 6701, 6702, 6703, 6704, 6705, 6706, 6707, 6708, 6709, 6710, 6711, 6712, 6713, 6714, 6715, 6716, 6717, 6718, 6719, 6720, 6721, 6722, 6723, 6724, 6725]} +{"timestamp": 90000, "input_length": 3954, "output_length": 297, "hash_ids": [0, 3874, 3875, 3876, 3877, 6726, 6727, 6728]} +{"timestamp": 90000, "input_length": 1902, "output_length": 654, "hash_ids": [0, 975, 976, 977]} +{"timestamp": 90000, "input_length": 27294, "output_length": 268, "hash_ids": [0, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1023, 1024, 1025, 1026, 6729, 6730, 6731, 6732, 6733, 6734, 6735, 6736, 6737, 6738, 6739, 6740, 6741, 6742, 6743, 6744, 6745, 6746, 6747, 6748, 6749, 6750, 6751, 6752, 6753, 6754, 6755, 6756, 6757, 6758, 6759]} +{"timestamp": 90000, "input_length": 9412, "output_length": 555, "hash_ids": [0, 6760, 6761, 6762, 6763, 6764, 6765, 6766, 6767, 6768, 6769, 6770, 6771, 6772, 6773, 6774, 6775, 6776, 6777]} +{"timestamp": 90000, "input_length": 12436, "output_length": 1644, "hash_ids": [0, 6778, 6779, 6780, 6781, 6782, 6783, 6784, 6785, 6786, 6787, 6788, 6789, 6790, 6791, 6792, 6793, 6794, 6795, 6796, 6797, 6798, 6799, 6800, 6801]} +{"timestamp": 90000, "input_length": 5240, "output_length": 534, "hash_ids": [0, 6802, 6803, 6804, 6805, 6806, 6807, 6808, 6809, 6810, 6811]} +{"timestamp": 90000, "input_length": 55009, "output_length": 644, "hash_ids": [0, 6812, 6813, 6814, 6815, 6816, 6817, 6818, 6819, 6820, 6821, 6822, 6823, 6824, 6825, 6826, 6827, 6828, 6829, 6830, 6831, 6832, 6833, 6834, 6835, 6836, 6837, 6838, 6839, 6840, 6841, 6842, 6843, 6844, 6845, 6846, 6847, 6848, 6849, 6850, 6851, 6852, 6853, 6854, 6855, 6856, 6857, 6858, 6859, 6860, 6861, 6862, 6863, 6864, 6865, 6866, 6867, 6868, 6869, 6870, 6871, 6872, 6873, 6874, 6875, 6876, 6877, 6878, 6879, 6880, 6881, 6882, 6883, 6884, 6885, 6886, 6887, 6888, 6889, 6890, 6891, 6892, 6893, 6894, 6895, 6896, 6897, 6898, 6899, 6900, 6901, 6902, 6903, 6904, 6905, 6906, 6907, 6908, 6909, 6910, 6911, 6912, 6913, 6914, 6915, 6916, 6917, 6918]} +{"timestamp": 90000, "input_length": 15174, "output_length": 13, "hash_ids": [0, 6919, 6920, 6921, 6922, 6923, 6924, 6925, 6926, 6927, 6928, 6929, 6930, 6931, 6932, 6933, 6934, 6935, 6936, 6937, 6938, 6939, 6940, 6941, 6942, 6943, 6944, 6945, 6946, 6947]} +{"timestamp": 90000, "input_length": 39419, "output_length": 243, "hash_ids": [0, 6948, 6949, 6950, 6951, 6952, 6953, 6954, 6955, 6956, 6957, 6958, 6959, 6960, 6961, 6962, 6963, 6964, 6965, 6966, 6967, 6968, 6969, 6970, 6971, 6972, 6973, 6974, 6975, 6976, 6977, 6978, 6979, 6980, 6981, 6982, 6983, 6984, 6985, 6986, 6987, 6988, 6989, 6990, 6991, 6992, 6993, 6994, 6995, 6996, 6997, 6998, 6999, 7000, 7001, 7002, 7003, 7004, 7005, 7006, 7007, 7008, 7009, 7010, 7011, 7012, 7013, 7014, 7015, 7016, 7017, 7018, 7019, 7020, 7021, 7022, 7023]} +{"timestamp": 93000, "input_length": 903, "output_length": 171, "hash_ids": [0, 7024]} +{"timestamp": 93000, "input_length": 2787, "output_length": 415, "hash_ids": [0, 7025, 7026, 7027, 7028, 7029]} +{"timestamp": 93000, "input_length": 9177, "output_length": 531, "hash_ids": [0, 7030, 7031, 7032, 7033, 7034, 7035, 7036, 7037, 7038, 7039, 7040, 7041, 7042, 7043, 7044, 7045, 7046]} +{"timestamp": 93000, "input_length": 36053, "output_length": 632, "hash_ids": [0, 7047, 7048, 7049, 7050, 7051, 7052, 7053, 7054, 7055, 7056, 7057, 7058, 7059, 7060, 7061, 7062, 7063, 7064, 7065, 7066, 7067, 7068, 7069, 7070, 7071, 7072, 7073, 7074, 7075, 7076, 7077, 7078, 7079, 7080, 7081, 7082, 7083, 7084, 7085, 7086, 7087, 7088, 7089, 7090, 7091, 7092, 7093, 7094, 7095, 7096, 7097, 7098, 7099, 7100, 7101, 7102, 7103, 7104, 7105, 7106, 7107, 7108, 7109, 7110, 7111, 7112, 7113, 7114, 7115, 7116]} +{"timestamp": 95999, "input_length": 13883, "output_length": 775, "hash_ids": [0, 7117, 7118, 7119, 7120, 7121, 7122, 7123, 7124, 7125, 7126, 7127, 7128, 7129, 7130, 7131, 7132, 7133, 7134, 7135, 7136, 7137, 7138, 7139, 7140, 7141, 7142, 7143]} +{"timestamp": 95999, "input_length": 8767, "output_length": 449, "hash_ids": [0, 7144, 7145, 7146, 7147, 7148, 7149, 7150, 7151, 7152, 7153, 7154, 7155, 7156, 7157, 7158, 7159, 7160]} +{"timestamp": 95999, "input_length": 7736, "output_length": 347, "hash_ids": [0, 7161, 7162, 7163, 7164, 7165, 7166, 7167, 7168, 7169, 7170, 7171, 7172, 7173, 7174, 7175]} +{"timestamp": 95999, "input_length": 13742, "output_length": 304, "hash_ids": [0, 7176, 7177, 7178, 7179, 7180, 7181, 7182, 7183, 7184, 7185, 7186, 7187, 7188, 7189, 7190, 7191, 7192, 7193, 7194, 7195, 7196, 7197, 7198, 7199, 7200, 7201]} +{"timestamp": 95999, "input_length": 897, "output_length": 94, "hash_ids": [0, 7202]} +{"timestamp": 95999, "input_length": 45924, "output_length": 24, "hash_ids": [0, 7203, 7204, 7205, 7206, 7207, 7208, 7209, 7210, 7211, 7212, 7213, 7214, 7215, 7216, 7217, 7218, 7219, 7220, 7221, 7222, 7223, 7224, 7225, 7226, 7227, 7228, 7229, 7230, 7231, 7232, 7233, 7234, 7235, 7236, 7237, 7238, 7239, 7240, 7241, 7242, 7243, 7244, 7245, 7246, 7247, 7248, 7249, 7250, 7251, 7252, 7253, 7254, 7255, 7256, 7257, 7258, 7259, 7260, 7261, 7262, 7263, 7264, 7265, 7266, 7267, 7268, 7269, 7270, 7271, 7272, 7273, 7274, 7275, 7276, 7277, 7278, 7279, 7280, 7281, 7282, 7283, 7284, 7285, 7286, 7287, 7288, 7289, 7290, 7291]} +{"timestamp": 95999, "input_length": 7617, "output_length": 435, "hash_ids": [0, 7292, 7293, 7294, 7295, 7296, 7297, 7298, 7299, 7300, 7301, 7302, 7303, 7304, 7305]} +{"timestamp": 95999, "input_length": 3142, "output_length": 398, "hash_ids": [0, 1541, 1542, 1543, 1544, 7306, 7307]} +{"timestamp": 95999, "input_length": 24132, "output_length": 2000, "hash_ids": [0, 7308, 7309, 7310, 7311, 7312, 7313, 7314, 7315, 7316, 7317, 7318, 7319, 7320, 7321, 7322, 7323, 7324, 7325, 7326, 7327, 7328, 7329, 7330, 7331, 7332, 7333, 7334, 7335, 7336, 7337, 7338, 7339, 7340, 7341, 7342, 7343, 7344, 7345, 7346, 7347, 7348, 7349, 7350, 7351, 7352, 7353, 7354]} +{"timestamp": 99000, "input_length": 4701, "output_length": 446, "hash_ids": [0, 7355, 7356, 7357, 7358, 7359, 7360, 7361, 7362, 7363]} +{"timestamp": 99000, "input_length": 19085, "output_length": 247, "hash_ids": [0, 7364, 7365, 7366, 7367, 7368, 7369, 7370, 7371, 7372, 7373, 7374, 7375, 7376, 7377, 7378, 7379, 7380, 7381, 7382, 7383, 7384, 7385, 7386, 7387, 7388, 7389, 7390, 7391, 7392, 7393, 7394, 7395, 7396, 7397, 7398, 7399, 7400]} +{"timestamp": 99000, "input_length": 899, "output_length": 418, "hash_ids": [0, 7401]} +{"timestamp": 99000, "input_length": 6603, "output_length": 20, "hash_ids": [0, 7402, 7403, 7404, 7405, 7406, 7407, 7408, 7409, 7410, 7411, 7412, 7413]} +{"timestamp": 99000, "input_length": 50465, "output_length": 621, "hash_ids": [0, 2225, 2226, 2227, 2228, 2229, 2230, 2231, 2232, 2233, 2234, 2235, 2236, 2237, 2238, 2239, 2240, 2241, 2242, 2243, 2244, 2245, 2246, 2247, 2248, 2249, 2250, 2251, 2252, 2253, 2254, 2255, 2256, 2257, 2258, 2259, 2260, 2261, 2262, 2263, 2264, 2265, 2266, 2267, 2268, 2269, 2270, 2271, 2272, 2273, 2274, 2275, 2276, 2277, 2278, 2279, 2280, 2281, 2282, 2283, 2284, 2285, 2286, 2287, 2288, 2289, 2290, 2291, 2292, 2293, 2294, 2295, 2296, 2297, 2298, 2299, 2300, 2301, 2302, 2303, 2304, 2305, 2306, 2307, 2308, 2309, 2310, 2311, 2312, 2313, 2314, 2315, 2316, 2317, 2318, 2319, 2320, 7414, 7415]} +{"timestamp": 99000, "input_length": 3025, "output_length": 673, "hash_ids": [0, 7416, 7417, 7418, 7419, 7420]} +{"timestamp": 99000, "input_length": 914, "output_length": 275, "hash_ids": [0, 7421]} +{"timestamp": 99000, "input_length": 14434, "output_length": 385, "hash_ids": [0, 1469, 1470, 1471, 1472, 1473, 1474, 1475, 1476, 1477, 1478, 1479, 1480, 1481, 1482, 1483, 1484, 1485, 1486, 1487, 1488, 1489, 1490, 1491, 1492, 1493, 1494, 7422, 7423]} +{"timestamp": 99000, "input_length": 17188, "output_length": 595, "hash_ids": [0, 7424, 7425, 7426, 7427, 7428, 7429, 7430, 7431, 7432, 7433, 7434, 7435, 7436, 7437, 7438, 7439, 7440, 7441, 7442, 7443, 7444, 7445, 7446, 7447, 7448, 7449, 7450, 7451, 7452, 7453, 7454, 7455, 7456]} +{"timestamp": 102000, "input_length": 67220, "output_length": 358, "hash_ids": [0, 7457, 7458, 7459, 7460, 7461, 7462, 7463, 7464, 7465, 7466, 7467, 7468, 7469, 7470, 7471, 7472, 7473, 7474, 7475, 7476, 7477, 7478, 7479, 7480, 7481, 7482, 7483, 7484, 7485, 7486, 7487, 7488, 7489, 7490, 7491, 7492, 7493, 7494, 7495, 7496, 7497, 7498, 7499, 7500, 7501, 7502, 7503, 7504, 7505, 7506, 7507, 7508, 7509, 7510, 7511, 7512, 7513, 7514, 7515, 7516, 7517, 7518, 7519, 7520, 7521, 7522, 7523, 7524, 7525, 7526, 7527, 7528, 7529, 7530, 7531, 7532, 7533, 7534, 7535, 7536, 7537, 7538, 7539, 7540, 7541, 7542, 7543, 7544, 7545, 7546, 7547, 7548, 7549, 7550, 7551, 7552, 7553, 7554, 7555, 7556, 7557, 7558, 7559, 7560, 7561, 7562, 7563, 7564, 7565, 7566, 7567, 7568, 7569, 7570, 7571, 7572, 7573, 7574, 7575, 7576, 7577, 7578, 7579, 7580, 7581, 7582, 7583, 7584, 7585, 7586, 7587]} +{"timestamp": 102000, "input_length": 1056, "output_length": 3, "hash_ids": [0, 7588, 7589]} +{"timestamp": 102000, "input_length": 10275, "output_length": 269, "hash_ids": [0, 7590, 7591, 7592, 7593, 7594, 7595, 7596, 7597, 7598, 7599, 7600, 7601, 7602, 7603, 7604, 7605, 7606, 7607, 7608, 7609]} +{"timestamp": 102000, "input_length": 13630, "output_length": 510, "hash_ids": [0, 1634, 1635, 1636, 1637, 1638, 1639, 1640, 1641, 1642, 1643, 1644, 1645, 1646, 1647, 1648, 1649, 1650, 1651, 1652, 1653, 1654, 1655, 7610, 7611, 7612, 7613]} +{"timestamp": 102000, "input_length": 8659, "output_length": 831, "hash_ids": [0, 7614, 7615, 7616, 7617, 7618, 7619, 7620, 7621, 7622, 7623, 7624, 7625, 7626, 7627, 7628, 7629]} +{"timestamp": 102000, "input_length": 24495, "output_length": 306, "hash_ids": [0, 7630, 7631, 7632, 7633, 7634, 7635, 7636, 7637, 7638, 7639, 7640, 7641, 7642, 7643, 7644, 7645, 7646, 7647, 7648, 7649, 7650, 7651, 7652, 7653, 7654, 7655, 7656, 7657, 7658, 7659, 7660, 7661, 7662, 7663, 7664, 7665, 7666, 7667, 7668, 7669, 7670, 7671, 7672, 7673, 7674, 7675, 7676]} +{"timestamp": 102000, "input_length": 69282, "output_length": 430, "hash_ids": [0, 7677, 7678, 7679, 7680, 7681, 7682, 7683, 7684, 7685, 7686, 7687, 7688, 7689, 7690, 7691, 7692, 7693, 7694, 7695, 7696, 7697, 7698, 7699, 7700, 7701, 7702, 7703, 7704, 7705, 7706, 7707, 7708, 7709, 7710, 7711, 7712, 7713, 7714, 7715, 7716, 7717, 7718, 7719, 7720, 7721, 7722, 7723, 7724, 7725, 7726, 7727, 7728, 7729, 7730, 7731, 7732, 7733, 7734, 7735, 7736, 7737, 7738, 7739, 7740, 7741, 7742, 7743, 7744, 7745, 7746, 7747, 7748, 7749, 7750, 7751, 7752, 7753, 7754, 7755, 7756, 7757, 7758, 7759, 7760, 7761, 7762, 7763, 7764, 7765, 7766, 7767, 7768, 7769, 7770, 7771, 7772, 7773, 7774, 7775, 7776, 7777, 7778, 7779, 7780, 7781, 7782, 7783, 7784, 7785, 7786, 7787, 7788, 7789, 7790, 7791, 7792, 7793, 7794, 7795, 7796, 7797, 7798, 7799, 7800, 7801, 7802, 7803, 7804, 7805, 7806, 7807, 7808, 7809, 7810, 7811]} +{"timestamp": 102000, "input_length": 3378, "output_length": 617, "hash_ids": [0, 1535, 1536, 1537, 1538, 1539, 7812]} +{"timestamp": 102000, "input_length": 979, "output_length": 140, "hash_ids": [0, 7813]} +{"timestamp": 102000, "input_length": 4743, "output_length": 541, "hash_ids": [0, 7814, 7815, 7816, 7817, 7818, 7819, 7820, 7821, 7822]} +{"timestamp": 102000, "input_length": 5714, "output_length": 5, "hash_ids": [0, 7823, 7824, 7825, 7826, 7827, 7828, 7829, 7830, 7831, 7832, 7833]} +{"timestamp": 105000, "input_length": 28831, "output_length": 634, "hash_ids": [0, 1657, 1658, 1659, 1660, 1661, 1662, 1663, 1664, 1665, 1666, 1667, 1668, 1669, 1670, 1671, 1672, 1673, 1674, 1675, 1676, 1677, 1678, 1679, 1680, 1681, 1682, 1683, 1684, 1685, 1686, 1687, 1688, 1689, 1690, 1691, 1692, 1693, 1694, 1695, 1696, 1697, 1698, 1699, 1700, 1701, 1702, 1703, 1704, 1705, 1706, 1707, 1708, 1709, 1710, 7834, 7835]} +{"timestamp": 105000, "input_length": 11850, "output_length": 382, "hash_ids": [0, 4110, 4111, 4112, 4113, 4114, 4115, 4116, 4117, 4118, 4119, 4120, 4121, 4122, 4123, 4124, 4125, 4126, 7836, 7837, 7838, 7839, 7840, 7841]} +{"timestamp": 105000, "input_length": 10357, "output_length": 35, "hash_ids": [0, 7842, 7843, 7844, 7845, 7846, 7847, 7848, 7849, 7850, 7851, 7852, 7853, 7854, 7855, 7856, 7857, 7858, 7859, 7860, 7861]} +{"timestamp": 105000, "input_length": 21398, "output_length": 384, "hash_ids": [0, 3827, 3828, 3829, 3830, 3831, 3832, 3833, 3834, 3835, 3836, 3837, 3838, 3839, 3840, 3841, 3842, 3843, 3844, 3845, 3846, 3847, 3848, 3849, 3850, 3851, 3852, 3853, 3854, 3855, 3856, 3857, 3858, 3859, 3860, 3861, 3862, 3863, 3864, 3865, 3866, 7862]} +{"timestamp": 105000, "input_length": 3944, "output_length": 515, "hash_ids": [0, 7863, 7864, 7865, 7866, 7867, 7868, 7869]} +{"timestamp": 108000, "input_length": 8532, "output_length": 38, "hash_ids": [0, 5096, 5097, 5098, 5099, 5100, 5101, 5102, 5103, 5104, 5105, 5106, 5107, 5108, 5109, 5110, 7870]} +{"timestamp": 108000, "input_length": 26728, "output_length": 370, "hash_ids": [0, 560, 561, 562, 563, 564, 565, 566, 567, 568, 569, 570, 571, 572, 573, 574, 575, 576, 577, 578, 579, 580, 581, 582, 583, 584, 585, 586, 587, 588, 589, 590, 591, 592, 593, 594, 595, 596, 597, 598, 599, 600, 601, 602, 603, 604, 605, 606, 607, 608, 609, 7871, 7872]} +{"timestamp": 108000, "input_length": 12588, "output_length": 387, "hash_ids": [0, 7873, 7874, 7875, 7876, 7877, 7878, 7879, 7880, 7881, 7882, 7883, 7884, 7885, 7886, 7887, 7888, 7889, 7890, 7891, 7892, 7893, 7894, 7895, 7896]} +{"timestamp": 108000, "input_length": 1661, "output_length": 93, "hash_ids": [0, 5021, 7897, 7898]} +{"timestamp": 108000, "input_length": 3510, "output_length": 760, "hash_ids": [0, 2935, 2936, 2937, 2938, 7899, 7900]} +{"timestamp": 108000, "input_length": 20187, "output_length": 124, "hash_ids": [0, 7901, 7902, 7903, 7904, 7905, 7906, 7907, 7908, 7909, 7910, 7911, 7912, 7913, 7914, 7915, 7916, 7917, 7918, 7919, 7920, 7921, 7922, 7923, 7924, 7925, 7926, 7927, 7928, 7929, 7930, 7931, 7932, 7933, 7934, 7935, 7936, 7937, 7938, 7939]} +{"timestamp": 108000, "input_length": 14980, "output_length": 32, "hash_ids": [0, 7940, 7941, 7942, 7943, 7944, 7945, 7946, 7947, 7948, 7949, 7950, 7951, 7952, 7953, 7954, 7955, 7956, 7957, 7958, 7959, 7960, 7961, 7962, 7963, 7964, 7965, 7966, 7967, 7968]} +{"timestamp": 108000, "input_length": 13695, "output_length": 527, "hash_ids": [0, 7969, 7970, 7971, 7972, 7973, 7974, 7975, 7976, 7977, 7978, 7979, 7980, 7981, 7982, 7983, 7984, 7985, 7986, 7987, 7988, 7989, 7990, 7991, 7992, 7993, 7994]} +{"timestamp": 108000, "input_length": 8058, "output_length": 457, "hash_ids": [0, 2052, 2053, 2054, 2055, 2056, 2057, 2058, 2059, 2060, 2061, 2062, 2063, 2064, 2065, 7995]} +{"timestamp": 111000, "input_length": 9325, "output_length": 644, "hash_ids": [0, 7996, 7997, 7998, 7999, 8000, 8001, 8002, 8003, 8004, 8005, 8006, 8007, 8008, 8009, 8010, 8011, 8012, 8013]} +{"timestamp": 111000, "input_length": 1479, "output_length": 41, "hash_ids": [0, 8014, 8015]} +{"timestamp": 111000, "input_length": 14314, "output_length": 57, "hash_ids": [0, 8016, 8017, 8018, 8019, 8020, 8021, 8022, 8023, 8024, 8025, 8026, 8027, 8028, 8029, 8030, 8031, 8032, 8033, 8034, 8035, 8036, 8037, 8038, 8039, 8040, 8041, 8042]} +{"timestamp": 111000, "input_length": 1060, "output_length": 60, "hash_ids": [0, 8043, 8044]} +{"timestamp": 111000, "input_length": 9535, "output_length": 529, "hash_ids": [0, 8045, 8046, 8047, 8048, 8049, 8050, 8051, 8052, 8053, 8054, 8055, 8056, 8057, 8058, 8059, 8060, 8061, 8062]} +{"timestamp": 111000, "input_length": 33990, "output_length": 521, "hash_ids": [0, 3098, 3099, 3100, 3101, 3102, 3103, 3104, 3105, 3106, 3107, 3108, 3109, 3110, 3111, 3112, 3113, 3114, 3115, 3116, 3117, 3118, 3119, 3120, 3121, 3122, 3123, 3124, 3125, 3126, 3127, 3128, 3129, 3130, 3131, 3132, 3133, 3134, 3135, 3136, 3137, 3138, 3139, 3140, 8063, 8064, 8065, 8066, 8067, 8068, 8069, 8070, 8071, 8072, 8073, 8074, 8075, 8076, 8077, 8078, 8079, 8080, 8081, 8082, 8083, 8084, 8085]} +{"timestamp": 111000, "input_length": 5240, "output_length": 111, "hash_ids": [0, 6802, 6803, 6804, 6805, 6806, 6807, 6808, 6809, 6810, 6811]} +{"timestamp": 111000, "input_length": 23983, "output_length": 553, "hash_ids": [0, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 8086, 8087, 8088, 8089, 8090, 8091, 8092, 8093, 8094, 8095, 8096, 8097, 8098, 8099, 8100, 8101, 8102]} +{"timestamp": 111000, "input_length": 6886, "output_length": 366, "hash_ids": [0, 8103, 8104, 8105, 8106, 8107, 8108, 8109, 8110, 8111, 8112, 8113, 8114, 8115]} +{"timestamp": 113999, "input_length": 10602, "output_length": 123, "hash_ids": [0, 8116, 8117, 8118, 8119, 8120, 8121, 8122, 8123, 8124, 8125, 8126, 8127, 8128, 8129, 8130, 8131, 8132, 8133, 8134, 8135]} +{"timestamp": 113999, "input_length": 115635, "output_length": 271, "hash_ids": [0, 8136, 8137, 8138, 8139, 8140, 8141, 8142, 8143, 8144, 8145, 8146, 8147, 8148, 8149, 8150, 8151, 8152, 8153, 8154, 8155, 8156, 8157, 8158, 8159, 8160, 8161, 8162, 8163, 8164, 8165, 8166, 8167, 8168, 8169, 8170, 8171, 8172, 8173, 8174, 8175, 8176, 8177, 8178, 8179, 8180, 8181, 8182, 8183, 8184, 8185, 8186, 8187, 8188, 8189, 8190, 8191, 8192, 8193, 8194, 8195, 8196, 8197, 8198, 8199, 8200, 8201, 8202, 8203, 8204, 8205, 8206, 8207, 8208, 8209, 8210, 8211, 8212, 8213, 8214, 8215, 8216, 8217, 8218, 8219, 8220, 8221, 8222, 8223, 8224, 8225, 8226, 8227, 8228, 8229, 8230, 8231, 8232, 8233, 8234, 8235, 8236, 8237, 8238, 8239, 8240, 8241, 8242, 8243, 8244, 8245, 8246, 8247, 8248, 8249, 8250, 8251, 8252, 8253, 8254, 8255, 8256, 8257, 8258, 8259, 8260, 8261, 8262, 8263, 8264, 8265, 8266, 8267, 8268, 8269, 8270, 8271, 8272, 8273, 8274, 8275, 8276, 8277, 8278, 8279, 8280, 8281, 8282, 8283, 8284, 8285, 8286, 8287, 8288, 8289, 8290, 8291, 8292, 8293, 8294, 8295, 8296, 8297, 8298, 8299, 8300, 8301, 8302, 8303, 8304, 8305, 8306, 8307, 8308, 8309, 8310, 8311, 8312, 8313, 8314, 8315, 8316, 8317, 8318, 8319, 8320, 8321, 8322, 8323, 8324, 8325, 8326, 8327, 8328, 8329, 8330, 8331, 8332, 8333, 8334, 8335, 8336, 8337, 8338, 8339, 8340, 8341, 8342, 8343, 8344, 8345, 8346, 8347, 8348, 8349, 8350, 8351, 8352, 8353, 8354, 8355, 8356, 8357, 8358, 8359, 8360]} +{"timestamp": 113999, "input_length": 24829, "output_length": 614, "hash_ids": [0, 1261, 1262, 1263, 1264, 1265, 1266, 1267, 1268, 1269, 1270, 1271, 1272, 1273, 1274, 1275, 1276, 1277, 1278, 1279, 1280, 1281, 1282, 1283, 1284, 1285, 1286, 1287, 1288, 1289, 1290, 1291, 1292, 1293, 1294, 1295, 1296, 1297, 1298, 1299, 1300, 1301, 1302, 1303, 1304, 8361, 8362, 8363, 8364]} +{"timestamp": 113999, "input_length": 40536, "output_length": 356, "hash_ids": [0, 8365, 8366, 8367, 8368, 8369, 8370, 8371, 8372, 8373, 8374, 8375, 8376, 8377, 8378, 8379, 8380, 8381, 8382, 8383, 8384, 8385, 8386, 8387, 8388, 8389, 8390, 8391, 8392, 8393, 8394, 8395, 8396, 8397, 8398, 8399, 8400, 8401, 8402, 8403, 8404, 8405, 8406, 8407, 8408, 8409, 8410, 8411, 8412, 8413, 8414, 8415, 8416, 8417, 8418, 8419, 8420, 8421, 8422, 8423, 8424, 8425, 8426, 8427, 8428, 8429, 8430, 8431, 8432, 8433, 8434, 8435, 8436, 8437, 8438, 8439, 8440, 8441, 8442, 8443]} +{"timestamp": 113999, "input_length": 13712, "output_length": 394, "hash_ids": [0, 8444, 8445, 8446, 8447, 8448, 8449, 8450, 8451, 8452, 8453, 8454, 8455, 8456, 8457, 8458, 8459, 8460, 8461, 8462, 8463, 8464, 8465, 8466, 8467, 8468, 8469]} +{"timestamp": 113999, "input_length": 20151, "output_length": 662, "hash_ids": [0, 8470, 8471, 8472, 8473, 8474, 8475, 8476, 8477, 8478, 8479, 8480, 8481, 8482, 8483, 8484, 8485, 8486, 8487, 8488, 8489, 8490, 8491, 8492, 8493, 8494, 8495, 8496, 8497, 8498, 8499, 8500, 8501, 8502, 8503, 8504, 8505, 8506, 8507, 8508]} +{"timestamp": 113999, "input_length": 6303, "output_length": 104, "hash_ids": [0, 8509, 8510, 8511, 8512, 8513, 8514, 8515, 8516, 8517, 8518, 8519, 8520]} +{"timestamp": 113999, "input_length": 15368, "output_length": 23, "hash_ids": [0, 8521, 8522, 8523, 8524, 8525, 8526, 8527, 8528, 8529, 8530, 8531, 8532, 8533, 8534, 8535, 8536, 8537, 8538, 8539, 8540, 8541, 8542, 8543, 8544, 8545, 8546, 8547, 8548, 8549, 8550]} +{"timestamp": 113999, "input_length": 6077, "output_length": 128, "hash_ids": [0, 4048, 4049, 4050, 4051, 4052, 4053, 4054, 4055, 4056, 4057, 8551]} +{"timestamp": 117000, "input_length": 900, "output_length": 200, "hash_ids": [0, 8552]} +{"timestamp": 117000, "input_length": 1216, "output_length": 384, "hash_ids": [0, 8553, 8554]} +{"timestamp": 117000, "input_length": 28631, "output_length": 372, "hash_ids": [0, 8555, 8556, 8557, 8558, 8559, 8560, 8561, 8562, 8563, 8564, 8565, 8566, 8567, 8568, 8569, 8570, 8571, 8572, 8573, 8574, 8575, 8576, 8577, 8578, 8579, 8580, 8581, 8582, 8583, 8584, 8585, 8586, 8587, 8588, 8589, 8590, 8591, 8592, 8593, 8594, 8595, 8596, 8597, 8598, 8599, 8600, 8601, 8602, 8603, 8604, 8605, 8606, 8607, 8608, 8609]} +{"timestamp": 117000, "input_length": 1888, "output_length": 407, "hash_ids": [0, 8610, 8611, 8612]} +{"timestamp": 117000, "input_length": 1434, "output_length": 90, "hash_ids": [0, 6625, 8613]} +{"timestamp": 120000, "input_length": 3808, "output_length": 16, "hash_ids": [0, 8614, 8615, 8616, 8617, 8618, 8619, 8620]} +{"timestamp": 120000, "input_length": 894, "output_length": 349, "hash_ids": [0, 8621]} +{"timestamp": 120000, "input_length": 35126, "output_length": 618, "hash_ids": [0, 1876, 1877, 1878, 1879, 1880, 1881, 1882, 1883, 1884, 1885, 1886, 1887, 1888, 1889, 1890, 1891, 1892, 1893, 1894, 1895, 1896, 1897, 1898, 1899, 1900, 1901, 1902, 1903, 1904, 1905, 1906, 1907, 1908, 1909, 1910, 1911, 1912, 1913, 1914, 1915, 1916, 1917, 1918, 1919, 1920, 1921, 1922, 1923, 1924, 1925, 1926, 1927, 1928, 1929, 1930, 1931, 1932, 1933, 1934, 1935, 1936, 1937, 1938, 1939, 1940, 1941, 1942, 1943]} +{"timestamp": 120000, "input_length": 18907, "output_length": 327, "hash_ids": [0, 4129, 4130, 4131, 4132, 4133, 4134, 4135, 4136, 4137, 4138, 4139, 4140, 4141, 4142, 4143, 4144, 4145, 4146, 4147, 4148, 4149, 4150, 4151, 4152, 4153, 4154, 4155, 4156, 4157, 4158, 4159, 4160, 4161, 4162, 4163, 8622]} +{"timestamp": 120000, "input_length": 6929, "output_length": 303, "hash_ids": [0, 8623, 8624, 8625, 8626, 8627, 8628, 8629, 8630, 8631, 8632, 8633, 8634, 8635]} +{"timestamp": 120000, "input_length": 896, "output_length": 354, "hash_ids": [0, 8636]} +{"timestamp": 120000, "input_length": 895, "output_length": 301, "hash_ids": [0, 8637]} +{"timestamp": 123000, "input_length": 10242, "output_length": 504, "hash_ids": [0, 8638, 8639, 8640, 8641, 8642, 8643, 8644, 8645, 8646, 8647, 8648, 8649, 8650, 8651, 8652, 8653, 8654, 8655, 8656, 8657]} +{"timestamp": 123000, "input_length": 1176, "output_length": 401, "hash_ids": [0, 8658, 8659]} +{"timestamp": 123000, "input_length": 1412, "output_length": 369, "hash_ids": [0, 8660, 8661]} +{"timestamp": 123000, "input_length": 19453, "output_length": 510, "hash_ids": [0, 8662, 8663, 8664, 8665, 8666, 8667, 8668, 8669, 8670, 8671, 8672, 8673, 8674, 8675, 8676, 8677, 8678, 8679, 8680, 8681, 8682, 8683, 8684, 8685, 8686, 8687, 8688, 8689, 8690, 8691, 8692, 8693, 8694, 8695, 8696, 8697, 8698]} +{"timestamp": 125999, "input_length": 10048, "output_length": 172, "hash_ids": [0, 8699, 8700, 8701, 8702, 8703, 8704, 8705, 8706, 8707, 8708, 8709, 8710, 8711, 8712, 8713, 8714, 8715, 8716, 8717]} +{"timestamp": 125999, "input_length": 8848, "output_length": 225, "hash_ids": [0, 8718, 8719, 8720, 8721, 8722, 8723, 8724, 8725, 8726, 8727, 8728, 8729, 8730, 8731, 8732, 8733, 8734]} +{"timestamp": 125999, "input_length": 24663, "output_length": 110, "hash_ids": [0, 8735, 8736, 8737, 8738, 8739, 8740, 8741, 8742, 8743, 8744, 8745, 8746, 8747, 8748, 8749, 8750, 8751, 8752, 8753, 8754, 8755, 8756, 8757, 8758, 8759, 8760, 8761, 8762, 8763, 8764, 8765, 8766, 8767, 8768, 8769, 8770, 8771, 8772, 8773, 8774, 8775, 8776, 8777, 8778, 8779, 8780, 8781, 8782]} +{"timestamp": 125999, "input_length": 6630, "output_length": 465, "hash_ids": [0, 8783, 8784, 8785, 8786, 8787, 8788, 8789, 8790, 8791, 8792, 8793, 8794]} +{"timestamp": 125999, "input_length": 12745, "output_length": 347, "hash_ids": [0, 8795, 8796, 8797, 8798, 8799, 8800, 8801, 8802, 8803, 8804, 8805, 8806, 8807, 8808, 8809, 8810, 8811, 8812, 8813, 8814, 8815, 8816, 8817, 8818]} +{"timestamp": 125999, "input_length": 67013, "output_length": 360, "hash_ids": [0, 8819, 8820, 8821, 8822, 8823, 8824, 8825, 8826, 8827, 8828, 8829, 8830, 8831, 8832, 8833, 8834, 8835, 8836, 8837, 8838, 8839, 8840, 8841, 8842, 8843, 8844, 8845, 8846, 8847, 8848, 8849, 8850, 8851, 8852, 8853, 8854, 8855, 8856, 8857, 8858, 8859, 8860, 8861, 8862, 8863, 8864, 8865, 8866, 8867, 8868, 8869, 8870, 8871, 8872, 8873, 8874, 8875, 8876, 8877, 8878, 8879, 8880, 8881, 8882, 8883, 8884, 8885, 8886, 8887, 8888, 8889, 8890, 8891, 8892, 8893, 8894, 8895, 8896, 8897, 8898, 8899, 8900, 8901, 8902, 8903, 8904, 8905, 8906, 8907, 8908, 8909, 8910, 8911, 8912, 8913, 8914, 8915, 8916, 8917, 8918, 8919, 8920, 8921, 8922, 8923, 8924, 8925, 8926, 8927, 8928, 8929, 8930, 8931, 8932, 8933, 8934, 8935, 8936, 8937, 8938, 8939, 8940, 8941, 8942, 8943, 8944, 8945, 8946, 8947, 8948]} +{"timestamp": 125999, "input_length": 4316, "output_length": 221, "hash_ids": [0, 8949, 8950, 8951, 8952, 8953, 8954, 8955, 8956]} +{"timestamp": 125999, "input_length": 8774, "output_length": 8, "hash_ids": [0, 8957, 8958, 8959, 8960, 8961, 8962, 8963, 8964, 8965, 8966, 8967, 8968, 8969, 8970, 8971, 8972, 8973]} +{"timestamp": 125999, "input_length": 1008, "output_length": 39, "hash_ids": [0, 8974]} +{"timestamp": 125999, "input_length": 14131, "output_length": 267, "hash_ids": [0, 8975, 8976, 8977, 8978, 8979, 8980, 8981, 8982, 8983, 8984, 8985, 8986, 8987, 8988, 8989, 8990, 8991, 8992, 8993, 8994, 8995, 8996, 8997, 8998, 8999, 9000, 9001]} +{"timestamp": 125999, "input_length": 23911, "output_length": 383, "hash_ids": [0, 3986, 3987, 3988, 3989, 3990, 3991, 3992, 3993, 3994, 3995, 3996, 3997, 3998, 3999, 4000, 4001, 4002, 4003, 4004, 4005, 4006, 4007, 4008, 4009, 4010, 4011, 4012, 4013, 4014, 4015, 4016, 4017, 4018, 4019, 4020, 4021, 9002, 9003, 9004, 9005, 9006, 9007, 9008, 9009, 9010, 9011]} +{"timestamp": 129000, "input_length": 9582, "output_length": 300, "hash_ids": [0, 9012, 9013, 9014, 9015, 9016, 9017, 9018, 9019, 9020, 9021, 9022, 9023, 9024, 9025, 9026, 9027, 9028, 9029]} +{"timestamp": 129000, "input_length": 14376, "output_length": 241, "hash_ids": [0, 9030, 9031, 9032, 9033, 9034, 9035, 9036, 9037, 9038, 9039, 9040, 9041, 9042, 9043, 9044, 9045, 9046, 9047, 9048, 9049, 9050, 9051, 9052, 9053, 9054, 9055, 9056, 9057]} +{"timestamp": 129000, "input_length": 16577, "output_length": 2000, "hash_ids": [0, 9058, 9059, 9060, 9061, 9062, 9063, 9064, 9065, 9066, 9067, 9068, 9069, 9070, 9071, 9072, 9073, 9074, 9075, 9076, 9077, 9078, 9079, 9080, 9081, 9082, 9083, 9084, 9085, 9086, 9087, 9088, 9089]} +{"timestamp": 129000, "input_length": 4999, "output_length": 230, "hash_ids": [0, 5993, 5994, 5995, 5996, 5997, 5998, 5999, 9090, 9091]} +{"timestamp": 129000, "input_length": 26158, "output_length": 3, "hash_ids": [0, 9092, 9093, 9094, 9095, 9096, 9097, 9098, 9099, 9100, 9101, 9102, 9103, 9104, 9105, 9106, 9107, 9108, 9109, 9110, 9111, 9112, 9113, 9114, 9115, 9116, 9117, 9118, 9119, 9120, 9121, 9122, 9123, 9124, 9125, 9126, 9127, 9128, 9129, 9130, 9131, 9132, 9133, 9134, 9135, 9136, 9137, 9138, 9139, 9140, 9141, 9142]} +{"timestamp": 131999, "input_length": 1318, "output_length": 314, "hash_ids": [0, 9143, 9144]} +{"timestamp": 131999, "input_length": 11113, "output_length": 467, "hash_ids": [0, 9145, 9146, 9147, 9148, 9149, 9150, 9151, 9152, 9153, 9154, 9155, 9156, 9157, 9158, 9159, 9160, 9161, 9162, 9163, 9164, 9165]} +{"timestamp": 131999, "input_length": 9586, "output_length": 270, "hash_ids": [0, 4863, 4864, 4865, 4866, 4867, 4868, 4869, 4870, 4871, 4872, 4873, 4874, 4875, 4876, 4877, 4878, 4879, 9166]} +{"timestamp": 131999, "input_length": 8341, "output_length": 455, "hash_ids": [0, 9167, 9168, 9169, 9170, 9171, 9172, 9173, 9174, 9175, 9176, 9177, 9178, 9179, 9180, 9181, 9182]} +{"timestamp": 131999, "input_length": 11667, "output_length": 324, "hash_ids": [0, 9183, 9184, 9185, 9186, 9187, 9188, 9189, 9190, 9191, 9192, 9193, 9194, 9195, 9196, 9197, 9198, 9199, 9200, 9201, 9202, 9203, 9204]} +{"timestamp": 131999, "input_length": 1000, "output_length": 125, "hash_ids": [0, 9205]} +{"timestamp": 131999, "input_length": 5780, "output_length": 405, "hash_ids": [0, 9206, 9207, 9208, 9209, 9210, 9211, 9212, 9213, 9214, 9215, 9216]} +{"timestamp": 131999, "input_length": 2533, "output_length": 241, "hash_ids": [0, 9217, 9218, 9219, 9220]} +{"timestamp": 131999, "input_length": 16637, "output_length": 3, "hash_ids": [0, 9221, 9222, 9223, 9224, 9225, 9226, 9227, 9228, 9229, 9230, 9231, 9232, 9233, 9234, 9235, 9236, 9237, 9238, 9239, 9240, 9241, 9242, 9243, 9244, 9245, 9246, 9247, 9248, 9249, 9250, 9251, 9252]} +{"timestamp": 131999, "input_length": 20422, "output_length": 459, "hash_ids": [0, 4287, 4288, 4289, 4290, 4291, 4292, 4293, 4294, 4295, 4296, 4297, 4298, 4299, 9253, 9254, 9255, 9256, 9257, 9258, 9259, 9260, 9261, 9262, 9263, 9264, 9265, 9266, 9267, 9268, 9269, 9270, 9271, 9272, 9273, 9274, 9275, 9276, 9277, 9278]} +{"timestamp": 131999, "input_length": 915, "output_length": 506, "hash_ids": [0, 9279]} +{"timestamp": 135000, "input_length": 11945, "output_length": 413, "hash_ids": [0, 9280, 9281, 9282, 9283, 9284, 9285, 9286, 9287, 9288, 9289, 9290, 9291, 9292, 9293, 9294, 9295, 9296, 9297, 9298, 9299, 9300, 9301, 9302]} +{"timestamp": 135000, "input_length": 9553, "output_length": 61, "hash_ids": [0, 9303, 9304, 9305, 9306, 9307, 9308, 9309, 9310, 9311, 9312, 9313, 9314, 9315, 9316, 9317, 9318, 9319, 9320]} +{"timestamp": 135000, "input_length": 10299, "output_length": 395, "hash_ids": [0, 4037, 4038, 4039, 4040, 4041, 4042, 4043, 4044, 4045, 4046, 9321, 9322, 9323, 9324, 9325, 9326, 9327, 9328, 9329, 9330]} +{"timestamp": 135000, "input_length": 26570, "output_length": 655, "hash_ids": [0, 9331, 9332, 9333, 9334, 9335, 9336, 9337, 9338, 9339, 9340, 9341, 9342, 9343, 9344, 9345, 9346, 9347, 9348, 9349, 9350, 9351, 9352, 9353, 9354, 9355, 9356, 9357, 9358, 9359, 9360, 9361, 9362, 9363, 9364, 9365, 9366, 9367, 9368, 9369, 9370, 9371, 9372, 9373, 9374, 9375, 9376, 9377, 9378, 9379, 9380, 9381]} +{"timestamp": 135000, "input_length": 81445, "output_length": 365, "hash_ids": [0, 9382, 9383, 9384, 9385, 9386, 9387, 9388, 9389, 9390, 9391, 9392, 9393, 9394, 9395, 9396, 9397, 9398, 9399, 9400, 9401, 9402, 9403, 9404, 9405, 9406, 9407, 9408, 9409, 9410, 9411, 9412, 9413, 9414, 9415, 9416, 9417, 9418, 9419, 9420, 9421, 9422, 9423, 9424, 9425, 9426, 9427, 9428, 9429, 9430, 9431, 9432, 9433, 9434, 9435, 9436, 9437, 9438, 9439, 9440, 9441, 9442, 9443, 9444, 9445, 9446, 9447, 9448, 9449, 9450, 9451, 9452, 9453, 9454, 9455, 9456, 9457, 9458, 9459, 9460, 9461, 9462, 9463, 9464, 9465, 9466, 9467, 9468, 9469, 9470, 9471, 9472, 9473, 9474, 9475, 9476, 9477, 9478, 9479, 9480, 9481, 9482, 9483, 9484, 9485, 9486, 9487, 9488, 9489, 9490, 9491, 9492, 9493, 9494, 9495, 9496, 9497, 9498, 9499, 9500, 9501, 9502, 9503, 9504, 9505, 9506, 9507, 9508, 9509, 9510, 9511, 9512, 9513, 9514, 9515, 9516, 9517, 9518, 9519, 9520, 9521, 9522, 9523, 9524, 9525, 9526, 9527, 9528, 9529, 9530, 9531, 9532, 9533, 9534, 9535, 9536, 9537, 9538, 9539, 9540]} +{"timestamp": 135000, "input_length": 5472, "output_length": 472, "hash_ids": [0, 9541, 9542, 9543, 9544, 9545, 9546, 9547, 9548, 9549, 9550]} +{"timestamp": 135000, "input_length": 24080, "output_length": 422, "hash_ids": [0, 9551, 9552, 9553, 9554, 9555, 9556, 9557, 9558, 9559, 9560, 9561, 9562, 9563, 9564, 9565, 9566, 9567, 9568, 9569, 9570, 9571, 9572, 9573, 9574, 9575, 9576, 9577, 9578, 9579, 9580, 9581, 9582, 9583, 9584, 9585, 9586, 9587, 9588, 9589, 9590, 9591, 9592, 9593, 9594, 9595, 9596, 9597]} +{"timestamp": 135000, "input_length": 4961, "output_length": 294, "hash_ids": [0, 9598, 9599, 9600, 9601, 9602, 9603, 9604, 9605, 9606]} +{"timestamp": 135000, "input_length": 2991, "output_length": 296, "hash_ids": [0, 9607, 9608, 9609, 9610, 9611]} +{"timestamp": 138000, "input_length": 1112, "output_length": 25, "hash_ids": [0, 9612, 9613]} +{"timestamp": 138000, "input_length": 3313, "output_length": 52, "hash_ids": [0, 9614, 9615, 9616, 9617, 9618, 9619]} +{"timestamp": 138000, "input_length": 8904, "output_length": 499, "hash_ids": [0, 9620, 9621, 9622, 9623, 9624, 9625, 9626, 9627, 9628, 9629, 9630, 9631, 9632, 9633, 9634, 9635, 9636]} +{"timestamp": 138000, "input_length": 9870, "output_length": 321, "hash_ids": [0, 9637, 9638, 9639, 9640, 9641, 9642, 9643, 9644, 9645, 9646, 9647, 9648, 9649, 9650, 9651, 9652, 9653, 9654, 9655]} +{"timestamp": 138000, "input_length": 6097, "output_length": 364, "hash_ids": [0, 9656, 9657, 9658, 9659, 9660, 9661, 9662, 9663, 9664, 9665, 9666]} +{"timestamp": 138000, "input_length": 11983, "output_length": 450, "hash_ids": [0, 9667, 9668, 9669, 9670, 9671, 9672, 9673, 9674, 9675, 9676, 9677, 9678, 9679, 9680, 9681, 9682, 9683, 9684, 9685, 9686, 9687, 9688, 9689]} +{"timestamp": 138000, "input_length": 7138, "output_length": 126, "hash_ids": [0, 9690, 9691, 9692, 9693, 9694, 9695, 9696, 9697, 9698, 9699, 9700, 9701, 9702]} +{"timestamp": 138000, "input_length": 908, "output_length": 341, "hash_ids": [0, 9703]} +{"timestamp": 138000, "input_length": 121298, "output_length": 614, "hash_ids": [0, 2697, 2698, 2699, 2700, 2701, 2702, 2703, 2704, 2705, 2706, 2707, 2708, 2709, 2710, 2711, 2712, 2713, 2714, 2715, 2716, 2717, 2718, 2719, 2720, 2721, 2722, 2723, 2724, 2725, 2726, 2727, 2728, 2729, 2730, 2731, 2732, 2733, 2734, 2735, 2736, 2737, 2738, 2739, 2740, 2741, 2742, 2743, 2744, 2745, 2746, 2747, 2748, 2749, 2750, 2751, 2752, 2753, 2754, 2755, 2756, 2757, 2758, 2759, 2760, 2761, 2762, 2763, 2764, 2765, 2766, 2767, 2768, 2769, 2770, 2771, 2772, 2773, 2774, 2775, 2776, 2777, 2778, 2779, 2780, 2781, 2782, 2783, 2784, 2785, 2786, 2787, 2788, 2789, 2790, 2791, 2792, 2793, 2794, 2795, 2796, 2797, 2798, 2799, 2800, 2801, 2802, 2803, 2804, 2805, 2806, 2807, 2808, 2809, 2810, 2811, 2812, 2813, 2814, 2815, 2816, 2817, 2818, 2819, 2820, 2821, 2822, 2823, 2824, 2825, 2826, 2827, 2828, 2829, 2830, 2831, 2832, 2833, 2834, 2835, 2836, 2837, 2838, 2839, 2840, 2841, 2842, 2843, 2844, 2845, 2846, 2847, 2848, 2849, 2850, 2851, 2852, 2853, 2854, 2855, 2856, 2857, 2858, 2859, 2860, 2861, 2862, 2863, 2864, 2865, 2866, 2867, 2868, 2869, 2870, 2871, 2872, 2873, 2874, 2875, 2876, 2877, 2878, 2879, 2880, 2881, 2882, 2883, 2884, 2885, 2886, 2887, 2888, 2889, 2890, 2891, 2892, 2893, 2894, 2895, 2896, 2897, 2898, 2899, 2900, 2901, 2902, 2903, 2904, 2905, 2906, 2907, 2908, 2909, 2910, 2911, 2912, 2913, 2914, 2915, 2916, 2917, 2918, 2919, 2920, 2921, 2922, 2923, 2924, 2925, 2926, 2927, 2928, 2929, 2930, 9704, 9705]} +{"timestamp": 138000, "input_length": 11296, "output_length": 48, "hash_ids": [0, 9706, 9707, 9708, 9709, 9710, 9711, 9712, 9713, 9714, 9715, 9716, 9717, 9718, 9719, 9720, 9721, 9722, 9723, 9724, 9725, 9726, 9727]} +{"timestamp": 141000, "input_length": 1809, "output_length": 634, "hash_ids": [0, 9728, 9729, 9730]} +{"timestamp": 141000, "input_length": 24982, "output_length": 480, "hash_ids": [0, 9731, 9732, 9733, 9734, 9735, 9736, 9737, 9738, 9739, 9740, 9741, 9742, 9743, 9744, 9745, 9746, 9747, 9748, 9749, 9750, 9751, 9752, 9753, 9754, 9755, 9756, 9757, 9758, 9759, 9760, 9761, 9762, 9763, 9764, 9765, 9766, 9767, 9768, 9769, 9770, 9771, 9772, 9773, 9774, 9775, 9776, 9777, 9778]} +{"timestamp": 141000, "input_length": 3631, "output_length": 22, "hash_ids": [0, 9779, 9780, 9781, 9782, 9783, 9784, 9785]} +{"timestamp": 141000, "input_length": 18203, "output_length": 359, "hash_ids": [0, 9786, 9787, 9788, 9789, 9790, 9791, 9792, 9793, 9794, 9795, 9796, 9797, 9798, 9799, 9800, 9801, 9802, 9803, 9804, 9805, 9806, 9807, 9808, 9809, 9810, 9811, 9812, 9813, 9814, 9815, 9816, 9817, 9818, 9819, 9820]} diff --git a/benchmarks/traffic_replay/manifest.yaml b/benchmarks/traffic_replay/manifest.yaml new file mode 100644 index 0000000..eb64087 --- /dev/null +++ b/benchmarks/traffic_replay/manifest.yaml @@ -0,0 +1,28 @@ +schema: gitm.bench.manifest/v1 +benchmark: traffic_replay +dataset_root: traffic_replay/fixtures +generated_by_gitm: 0.1.13 +file_count: 7 +total_bytes: 150941 +files: +- path: benchserve_result.json + sha256: a780cd29916523c38c12ca833e92896bc75c93b6f94d92826ad10f440e2e35a4 + bytes: 1045 +- path: burstgpt3_dirty.csv + sha256: 6b78a95497587a539f0f10fd0db4eb2a56aadf1af69c3f3eb058ba1c4cee73fd + bytes: 474 +- path: burstgpt3_slice.csv + sha256: 5465fc4c38b0815b8f75d126a120c229ae93632d62e3adf1792f5e8401a73559 + bytes: 34278 +- path: burstgpt_dirty.csv + sha256: 8002d35e1b3ab5bd1169426036d3e7af93c08acd453bb879391eac752e210bc5 + bytes: 372 +- path: burstgpt_slice.csv + sha256: 7865fda5dac50938a84dfccb06c405e2f5202969ecfbd671c8399de00f4335f0 + bytes: 16840 +- path: mooncake_dirty.jsonl + sha256: 483a2e36924902f4bf7fc52ca29d73897918e04e9c30abd521b13852432621a0 + bytes: 639 +- path: mooncake_slice.jsonl + sha256: 8090d6a38401d280fd88112ba22007ff4354be59c986f4f6bcc250edd45a52dc + bytes: 97293 diff --git a/benchmarks/traffic_replay/spec.md b/benchmarks/traffic_replay/spec.md new file mode 100644 index 0000000..b661387 --- /dev/null +++ b/benchmarks/traffic_replay/spec.md @@ -0,0 +1,237 @@ +# Traffic replay v1 — what is frozen, and why + +Deliverable 1 of the validation infrastructure. The load generator for every +experiment: real production traces normalized into one format, fired through the +harness, and tagged with the workload regime every result row is keyed on. + +**Library:** `gitm/traffic/`. **CPU-only** — nothing here needs a GPU, and only +the final firing needs a server. + +```bash +python -m gitm.traffic --selftest # the check +python -m gitm.traffic --describe burstgpt # meta + regime +python -m gitm.traffic --replay mooncake --out replay.jsonl \ + --model Qwen/Qwen3.6-35B-A3B-FP8 # + validation +python -m gitm.traffic --sweep burstgpt # the grid +``` + +## 1. Sources + +| source | format | timestamps | session | prefix identity | fixture | +|---|---|---|---|---|---| +| **BurstGPT_1/_2** | CSV, 6 columns: `Timestamp,Model,Request tokens,Response tokens,Total tokens,Log Type` | **seconds** | none | none | `fixtures/burstgpt_slice.csv` | +| **BurstGPT_3** | CSV, 8 columns: `Timestamp,`**`Session ID,Elapsed time,`**`Model,Request tokens,Response tokens,Total tokens,Log Type` | **seconds** | `Session ID` (UUID) | none | `fixtures/burstgpt3_slice.csv` | +| **Mooncake** | JSONL, `{timestamp, input_length, output_length, hash_ids}` | **milliseconds** | none | `hash_ids`, **512-token** blocks | `fixtures/mooncake_slice.jsonl` | + +All three formats were read off the real published files, not off a paper. +Sources: `HPMLL/BurstGPT` — `data/BurstGPT_1.csv` on `main`, and +`BurstGPT_3.csv` from **release v2.0** (the `_3` files exist only there; `main` +carries `_1` alone) — and `kvcache-ai/Mooncake` +(`FAST25-release/traces/conversation_trace.jsonl`). + +**BurstGPT_3 inserts its two columns at positions 1 and 2 — it does not append +them.** A positional reader does not merely miss them; it reads `Session ID` as +the model and `Elapsed time` as the request length. `read_burstgpt` therefore +goes by column **name**: the six core columns are required, `Session ID` and +`Elapsed time` are used when present, and an unrecognized extra column is +recorded in `TraceMeta.notes` rather than rejected — so a future `BurstGPT_4` +loads instead of raising. `BurstGPT_without_fails_3.csv` carries the same eight. + +Real fixtures are the **first 400 rows of each real file**, unmodified, plus +three hand-authored dirty files that exercise every drop reason. All six are +pinned by sha256 in `manifest.yaml` (`gitm.bench.manifest/v1`, the same contract +the other benchmarks use). + +## 2. The canonical schema + +`gitm.traffic.schema`. Per request: `arrival_s` (**seconds offset from trace +start**, never an epoch), `input_tokens`, `output_tokens`, `session_id`, +`prefix_blocks`. Absent-from-source behaviour is documented per field in the +docstring and is *never* a silent default — a row that cannot supply a required +field is dropped under a named reason. + +`prefix_blocks` is a **chain**, not one hash: two requests share a prefix exactly +as far as their leading block ids agree, and a single digest of the whole chain +would only match identical prompts — the case that does not need measuring. + +`TraceMeta` carries the raw file's sha256, `rows_read`, `rows_emitted` and the +per-reason drop counts. **`Trace` refuses to exist unless those reconcile** +(`rows_read == rows_emitted + dropped`), so "we dropped some bad rows" can never +be a hand-wave. + +## 3. What the real data actually contains + +Pinned in `gitm/traffic/_selftest.py`, measured not assumed: + +| | BurstGPT_1 slice | BurstGPT_3 slice | Mooncake slice | +|---|---|---|---| +| rows read / emitted | 400 / **383** | 400 / **399** | 400 / 400 | +| drops | `zero_input_tokens=17` | `zero_input_tokens=1` | none | +| span | 37,269 s | 41,453 s | 141.0 s | +| session rows / sessions | — | **393 / 134** | — | +| regime label | `prod/io1/in256/out128/burst-poisson/copen` | `prod/io2/in256/out64/burst-poisson/copen` | `prod/io32/in8k/out256/burst-hi/copen` | +| input p50 / p95 | 353 / 1,638 | — | 9,075 / 49,904 | +| output p50 / p95 | 238 / 841 | — | 370 / 662 | +| burstiness (D @ 1 s) | 1.01 | — | 6.74 | + +### Session identity is partial by design, so it is counted, not flagged + +In the full `BurstGPT_3` file, `Session ID` is populated on **exactly** the +`Conversation log` rows and empty on **exactly** the `API log` rows — 528 with, +5,115 without, in the first 5,643. So **an empty `Session ID` is normal data, not +a defect**: the row is emitted with `session_id=None`. Dropping them would throw +away 90 % of a real v3 trace. + +That is also why `TraceMeta` carries `session_rows` and `sessions` and not just +`has_session_identity`. The boolean says "yes" on a trace that is 90 % single-shot; +a multi-turn experiment needs the counts to decide whether the trace can carry it. +In the committed slice: 393 of 399 emitted rows across **134 conversations**, +longest 24 turns. + +`Elapsed time` becomes `CanonicalRequest.source_e2e_latency_s` — deliberately +long-named. It is the **source system's end-to-end submission-to-final-response +time on OpenAI's hardware**: not TTFT, not ITL, and not ours. It must never be +compared against a measured latency or used to promote a playbook row. It is +carried rather than discarded because losing real data at the adapter boundary is +unrecoverable; its one legitimate use is bounding think-time between turns of a +session. An unparseable value is read as *absent* (the row survives, the count +lands in `TraceMeta.notes`) — an optional annotation being junk is no reason to +throw away a valid request. + +**7.9 % of the full BurstGPT file carries zero input *and* zero output tokens** +(4.3 % of this slice). A loader that kept them would put empty prefills into +every regime fit. This is the "handle real data" requirement firing on the first +source, not a hypothetical. + +## 4. Regime axes + +`gitm.traffic.regime`. Prefill/decode token ratio, input and output length +distributions, arrival burstiness, concurrency — plus `source_kind`. + +Burstiness is the **index of dispersion** (variance/mean of arrival counts per +1 s bin), not the CV of interarrival times. Both are 1 for a Poisson process, but +two traces with the same mean rate and different bunching collapse to one CV, and +bunching is the axis the customer's traffic actually varies on. + +`source_kind ∈ {production, synthetic, scoreboard}` is a schema field, not a +naming convention. **Artificial Analysis's fixed-length workload is a +`scoreboard` regime** and its label starts `board/`, so it can never be read as +production traffic after being copied into a spreadsheet. + +`Regime.label()` is bucketed on purpose — two runs of the same workload must +produce the same label, and raw quantiles never repeat. Out-of-envelope sampled +points are suffixed `/xenv`. + +## 5. Replay mode — no custom load generator + +vLLM's bench-serve has a native `timed_trace` dataset that consumes exactly +`{timestamp, input_length, output_length, hash_ids}` JSONL and, under +`--self-timed`, schedules **each request at its own timestamp**. That is faithful +replay, already written and maintained, so `gitm.traffic.replay` writes that file +and builds that command line. Nothing here fires traffic itself. + +Two silent failures this module exists to make loud: + +1. **Block coverage.** `timed_trace` expands each `hash_ids` entry to + `--timed-trace-chunk-hash-size` tokens and *stops when the ids run out*. Pass + vLLM's default of 16 against Mooncake's 512-token blocks and every prompt is + 32× short while every count still looks right. `write_timed_trace` checks + `len(blocks) * block_tokens >= input_tokens` per request and **refuses the + whole file** rather than truncating. +2. **Sources with no prefix identity.** BurstGPT has no `hash_ids`; an empty list + produces a *zero-length* prompt. Blocks are therefore **synthesized** — a + fresh, globally unique run per request, so lengths are honoured and **no + prefix sharing is invented that the source never had**. `ReplayPlan` + records `prefix_synthesized=True`, and a prefix-cache experiment must reject + such a plan. + +## 6. Parameterized mode + +`gitm.traffic.parameterize`. Fits each trace along the regime axes, then samples +the grid — including points beyond any single trace, because the customer's next +hour is never the trace's next hour. + +- Length distributions are fitted **empirically** (101-point quantile grid, + inverse-CDF sampling). No parametric family: production length distributions + are multi-modal — short chat turns and long document prompts in one trace — and + a lognormal fit would smear the modes and quietly move the prefill/decode ratio + the whole exercise turns on. +- Arrival burstiness is generated by drawing **per-bin counts from a negative + binomial**, whose index of dispersion is `1 + m/r` and so can be set directly to + the target. Poisson is the `D = 1` case. `D < 1` clamps to Poisson and says so. +- Every point outside the fitted envelope is `in_envelope=False` and labelled + `/xenv`. An extrapolation that cannot be told from a measurement is worse than + no extrapolation. +- A synthetic trace's identity is the **sha256 of its generating parameters** — + same digest, same trace, reproducible without storing the file. + +## 7. Validation — the deliverable, shown + +`gitm.traffic.validate` compares the replayed stream against the source on +arrival timing, both length distributions, mean rate and burstiness, and renders +the check table beside both arrival-rate profiles. Any mismatch is **explained in +prose**, not just printed. + +What is compared is the **file bench-serve will actually read**: the emitter +writes it, `read_timed_trace` reads it back, and `compare` puts it beside the +adapter's output. So "the pipeline preserves the trace" is a measurement of the +artifact, not an argument about the code. + +Two standards, named rather than implied: + +| | `REPLAY_THRESHOLDS` | `SAMPLED_THRESHOLDS` | +|---|---|---| +| KS (arrival, lengths) | 0.001 | 0.15 | +| mean rate | 1e-6 | 0.25 | +| burstiness | 1e-6 | 0.60 | +| request count | exact | 0.35 | +| arrival timeline compared | **yes** | **no** | + +A replay must *reproduce* the trace — the emitter is a format change, so every +statistic comes back identical. A parameterized sample is a *draw* from the +envelope: it reproduces the rate and the dispersion by construction and the +timeline by nothing, so comparing timelines there would only measure that a +sample is not a copy. + +Arrival times are compared at **microsecond resolution** +(`ARRIVAL_RESOLUTION_S`). Sub-microsecond replay fidelity is meaningless next to +millisecond network jitter, and without the quantization the KS reports the +6e-16 s residue of `5999 / 1000` as a real distribution gap — which it did, on +the Mooncake fixture, before this was added. + +Measured, on the committed fixtures: **every replay check returns exactly 0.0** +for both adapters. + +## 8. The check + +```bash +python -m gitm.traffic --selftest # 10 checks, 2 real traces, 7 drop reasons +python -m pytest tests/test_traffic.py -q # the same assertions, as pytest cases +``` + +The assertions live in `gitm/traffic/_selftest.py` and both entry points call +them, so the runnable check named here and the one CI runs cannot drift apart. + +## 9. Not in v1, and why + +- **Adapters beyond these two.** Azure LLM inference, the Prism provider traces, + TraceLab, ShareGPT/LMSYS lengths — "later" in the brief. Two adapters are what + proves the canonical schema is real; the third would only prove it again. +- **Session-aware *firing*.** The adapter is session-aware; the replay path + cannot be. vLLM's `timed_trace` format has **no session field**, so conversation + identity stops at the emitter — pinned by + `check_session_trace_replay_understates_reuse`, not left as folklore. Sessions + are available for analysis and regime characterization today. Making them flow + would mean deriving prefix blocks from session membership, i.e. asserting how + much each turn re-sends: an *invented* cache hit, which is the one thing this + module will not do. Consequence, stated on every such plan: prefix reuse on a + BurstGPT_3 replay is **understated, never overstated** — a floor, not an + estimate. +- **A streaming `Trace`.** The whole trace is materialized; fine to a few million + rows (BurstGPT_1 is ~1.4 M). Named ceiling, not an oversight. +- **Firing at a real endpoint.** vLLM is not installed on the authoring box. The + emitted file and the argv are built against vLLM's current `timed_trace` + contract, read from source; the first run against a live server is the + outstanding confirmation. +- **Concurrency as a fitted axis.** It is an offered-load *setting*, not a + property of a trace: it is carried on `Regime` and set by the caller. diff --git a/docs/playbook-schema.md b/docs/playbook-schema.md new file mode 100644 index 0000000..ea5aa84 --- /dev/null +++ b/docs/playbook-schema.md @@ -0,0 +1,343 @@ +# Playbook schema + +The contract between detection and apply. A **row** is one promoted tuning +result; a **lookup** decides whether a past row applies to the workload in front +of you. Types in `gitm/playbook/schema.py`, matching in `gitm/playbook/match.py`, +worked rows in `benchmarks/playbook/examples.json`. + +> **Read this first.** The distance *metric* below is designed and implemented. +> The distance *threshold* is **not calibrated**, no measurement exists that +> could calibrate it yet, and the shipped policy carries no number at all. Under +> `UNCALIBRATED_POLICY` the only automatic match is an **exact** regime match; +> everything else routes to conservative discovery. That is the honest state, and +> §5 says exactly what would change it. + +```bash +python -m gitm.playbook --selftest # 17 checks +python -m gitm.playbook --show benchmarks/playbook/examples.json +python -m gitm.playbook --distance benchmarks/playbook/examples.json ex1-… ex2-… +python -m gitm.playbook --lookup benchmarks/playbook/examples.json ex1-… +``` + +--- + +## 1. What a row is + +``` +(model + revision, GPU SKU, environment, workload regime, knob set) + -> measured delta + provenance +``` + +| part | type | matched how | +|---|---|---| +| `model`, `model_revision` | `str` | **exact** | +| `gpu_sku` | `str` | **exact** | +| `env` (`EnvCapture`) | engine + engine version | **exact** (§3) | +| `regime.source_kind` | `production \| synthetic \| scoreboard` | **exact** | +| `regime.concurrency` | `int \| None` | **exact**, by policy | +| `knobs` | `dict[str, bool\|int\|float\|str]` | **exact**, key set | +| `regime` numeric axes | `Regime` | **distance** (§2) | + +`Regime` is deliverable 1's type, **imported**. There is no second copy — a +second one would drift inside a week and the distance would be measured in two +different coordinate systems. `check_regime_is_imported_not_redeclared` asserts +the field's annotation is literally `gitm.traffic.regime.Regime`. + +### What the types refuse to hold + +A type cannot see how a number was produced, so it cannot enforce "came through +the promotion rule". What it does instead is refuse rows that **could not** have: + +| refused | why | +|---|---| +| `knobs={}` | a row with no knob set says nothing | +| `repeats < 2` | a single run has no variance and cannot clear D2-1 | +| a delta with throughput but no TTFT/ITL | D2 criterion 3 would be unenforceable | +| `trace_sha256=""` | a claim that cannot be traced back to bytes | +| any unknown field | `extra="forbid"` everywhere except `EnvCapture` (§3) | + +### Two fields that are states, not absences + +- **`evidence`** — `measured` or `illustrative`. The worked examples ship in the + same format as real rows, so a field has to separate them. Every row in + `examples.json` is `illustrative` and **none of them is selectable**; a lookup + against the example file returns `no_match` with that as the stated reason. +- **`invalidated`** — an `Invalidation(reason, at, by)`, never a deletion. A + deleted row leaves no record that the claim was ever made, which is the first + thing a reviewer asks for. + +### `delta_is_floor` — the D1-11 guard + +BurstGPT has no prefix identity. When D1 replays it, `write_timed_trace` +**synthesizes** prefix blocks, unique per request, so lengths hold and *no prefix +sharing is invented that the source never had*. A prefix-cache knob measured on +such a trace therefore saw the **least** reuse the real traffic could have had. + +`PlaybookRow.delta_is_floor` is `True` when `provenance.prefix_synthesized` is +set **and** the knob set touches prefix caching. Such a row's delta is a lower +bound — usable as "at least this much", never quotable as the gain — and +`summary()` prints `[FLOOR: prefixes synthesized]`. `ex6` in the examples is that +case; `ex2` is the control (same synthesized trace, a knob that does not depend +on reuse, so not a floor). + +### Replay conditions in provenance + +`replay_chunk_hash_size` is a field because it is deliverable 1's finding with +the worst failure mode: at vLLM's default of 16 against Mooncake's 512-token +blocks, every prompt is 32× short **while every count in the result still reads +correctly**. A row that does not record the block size cannot be checked for it. + +### Where a row comes from — `row_from_runs` + +A `BenchRun` (deliverable 1, seam 3) is **one arm**. A row is the **difference +between two**, so the two are joined here and nowhere else: + +```python +from gitm.playbook import row_from_runs + +row = row_from_runs( + "prefix-cache-mooncake-h100", baseline_runs, treatment_runs, + model="Qwen/Qwen3.6-35B-A3B-FP8", model_revision="95a723d0", + gpu_sku="NVIDIA H100 80GB", env=EnvCapture(engine="vllm", engine_version="0.28.0"), + knobs={"enable_prefix_caching": True}, +) +``` + +Provenance is lifted off the runs, never retyped by the caller: the trace +checksum, drop counts, regime label and replay conditions come from the arms that +actually ran, which is the only way the row's checksum and the run's checksum +cannot drift apart. + +**It refuses, rather than producing a row, when:** + +| refusal | why | +|---|---| +| the arms did not run the same trace bytes or label to the same regime | a delta across two workloads measures the workloads | +| any run is not `promotable` (did not reconcile, or had failures) | below that bar there is nothing to take a difference of | +| the arms have unequal repeat counts | D2 interleaves A/B/A/B; unequal means the interleave broke | +| `repeats < 2` | delegated to `MeasuredDelta` — a single run has no variance | +| either arm is missing `output_throughput`, `p99_ttft_ms` or `p99_itl_ms` | D2's criterion 3 is unenforceable without both sides | + +**What it deliberately does not do.** `throughput_pct` is a percentage of +`output_throughput` (`THROUGHPUT_METRIC`, stated once so two readers cannot mean +different numbers), computed on **medians, never means**. `throughput_ci95_pct` +and `latency_blowout` stay `None`: D2 owns the variance rule and the blowout +predicate, and inventing either here is the mistake `AxisTolerance` refuses to +make with a distance threshold. And "same config minus exactly one knob" is D2's, +enforced by diffing two config-capture records that do not exist yet (**R1**) — +until they do, the caller asserts it and the row carries a note saying so, which +disappears on its own the moment `config_capture` is real. + +--- + +## 2. Regime distance + +### The metric + +Numeric axes are compared as **log2 ratios**, because what matters for token +counts and rates is the *factor*, not the difference: + +``` +|log2(a / b)| + +1,024 vs 2,048 tokens -> 1.00 (a 2x change) +1,024 vs 1,536 tokens -> 0.58 +1,024 vs 1,024 tokens -> 0.00 +``` + +Scale-free by construction: 100 vs 200 and 10,000 vs 20,000 are the same +distance, which is the property the axis needs. Symmetric. Two exact zeros are +equal; **one** zero is `inf` — "no output tokens at all" is not a small version +of "some output tokens", and infinity is what makes the combination say so +without a special case. + +**Burstiness** uses a shifted ratio, `|log2((1+a)/(1+b))|`. A perfectly paced +trace has `D = 0` and a bare ratio would make it incomparable to everything +including another paced trace. The shift anchors the axis on the Poisson +reference: + +``` +flat (0.00) vs poisson (1.00) -> 1.00 +burstgpt (1.01) vs mooncake (6.74) -> 1.94 +moderate (5.00) vs mooncake (6.74) -> 0.37 +``` + +### Combination: L-infinity + +``` +regime_distance = max(input_p50, input_p95, output_p50, output_p95, + io_ratio, burstiness) +``` + +Not a mean, not a Euclidean norm. **A row is as far away as its worst axis.** The +case this exists for, asserted in `check_linf_is_the_worst_axis`: a candidate +identical on five axes and 8× off on `input_p95`. The mean calls that a 0.5 +mismatch and would apply the row; L-inf calls it 3.0 and does not. That is a +long-context workload against a short-context row. + +For reference, the two real traces deliverable 1 measured are **L-inf 4.929 +apart, limited by `input_p95`** (49,904 vs 1,638). If those two collapsed to a +small distance, the axes would be decoration. + +### `rate_rps` is deliberately not an axis + +It exists on `Regime` and is **not** in `DEFAULT_AXES`. Adding it because it is +there would be exactly the mistake this module is written to avoid. + +| | | +|---|---| +| **include it if** | knob outcomes are shown to depend materially on offered load *after* concurrency and burstiness are accounted for | +| **omit it if** | the selected knobs are insensitive to rate once those two are fixed | +| **either way** | the decision is recorded with the experiment that settled it | + +The decision is material, not cosmetic: two regimes identical except for an 8× +difference in offered rate are distance **0.0** by default and **3.0** with the +axis on (`check_rate_is_not_in_the_default_axes`). Turning it on is one field: + +```python +MatchPolicy(name="with-rate", axes=(*DEFAULT_AXES, "rate_rps"), tolerances=…) +``` + +--- + +## 3. The exact-match gates + +Checked **before** any distance is computed, and each returns a stated reason +rather than a large number. + +| gate | rule | why not a distance | +|---|---|---| +| `model`, `model_revision` | exact | "nearly the same weights" is not a thing | +| `gpu_sku` | exact | ditto for silicon | +| `env.engine`, `env.engine_version` | exact | scheduler rewrites ship in point releases; a version bump is the most common way a knob's effect changes with nothing in the workload changing | +| `source_kind` | exact | a `scoreboard` row can never satisfy a production query, even when every numeric axis is identical (asserted) | +| `concurrency` | exact, `match_concurrency=True` | open-loop and a capped-concurrency run are different experiments | +| knob key set | exact | a lookup asks about a *specific* knob | + +Loosening any of them is an edit to a named field on `MatchPolicy`, not an +accident inside a comparison. `EnvCapture` is the one model with `extra="allow"`, +so a capture record from a newer engine round-trips without being truncated — +but `compatible_with` reads the **named** fields only, so an unknown extra key +can never change a match decision. + +When the shared config-capture schema lands (**R1**), `EnvCapture` is deleted and +Adit's types are **imported verbatim**. No translation layer: two schemas that +translate into each other are two schemas that drift. Everything waiting on this +is marked `pending-adit` and is grep-able. + +--- + +## 4. Lookup, precedence, and the handoff + +```python +result = lookup(playbook, query_identity, UNCALIBRATED_POLICY) +if result.route_to_discovery: + ... # conservative discovery +else: + apply(result.row.identity.knobs) +``` + +| status | meaning | returns a row? | +|---|---|---| +| `exact_regime` | distance 0.0 on every axis | ✅ | +| `near_regime` | inside every calibrated tolerance | ✅ | +| `uncalibrated` | candidates exist; some nonzero axis has no calibrated tolerance | ❌ → discovery | +| `no_match` | nothing passed the gates, or the nearest exceeds its tolerance | ❌ → discovery | + +`route_to_discovery` is `True` for everything that is not a returned row. A +status that is sometimes a row and sometimes a suggestion is how a wrong row gets +applied inside a 72-hour window. + +A miss carries what it rejected and why — `rejected: {row_id -> reason}` plus the +ranked `candidates` with their per-axis distances. Discovery starts warm, and a +human reading a miss learns which axis was the problem, not just that there was +one. **Discovery mode itself is not designed here**; only the handoff. + +### Precedence + +1. **Distance** — nearest first. +2. **In envelope** — among equally near rows, a row measured inside the observed + envelope beats one sampled beyond it (`/xenv`). *This sits below distance, + where `todo.md` had it above*: a nearby extrapolated point was still genuinely + run, and preferring a 4×-away in-envelope row over it answers the wrong + question. +3. **Recency** — most recently verified. +4. **Smallest claim** — conservative. A wrong row applied in the live window + costs more than a missed opportunity. + +Each of the four is asserted separately. + +--- + +## 5. Calibration — what would remove `uncalibrated` + +`AxisTolerance` **refuses a number without the experiment that produced it**: + +```python +AxisTolerance(max_distance=1.0) # ValidationError +AxisTolerance(max_distance=1.0, calibration="prereg E4, 2026-09-14: " + "sign flip at 1.4 on input_p95") # ok +``` + +That is the enforcement behind "the threshold is currently unknown". A +placeholder cannot quietly become a production constant. + +**Why `PLAYBOOK_MATCH_MAX_DISTANCE = 1.0` was not shipped.** Under log2 it means +"accept up to a 2× mismatch on every axis at once". That may well be safe for +`output_p50`. It is not obviously safe for long-context `input_p95`, for +prefix-cache reuse, or for a queue-sensitive scheduling policy — and nothing +measured says which. A number that reads as derived when it was picked is the +kind of thing that gets defended in front of a customer and then collapses. + +### The procedure + +1. Run the same knob across nearby regimes, varying **one** axis at a time. +2. Find where the effect changes sign, or where D2's latency-percentile criterion + flips from pass to fail. +3. Set that axis's tolerance strictly inside the distance at which it flipped, + and cite the run in `calibration`. +4. The L-inf limit is then **the strictest relevant per-axis tolerance, by + construction** — there is no separate global number to choose. + +### What blocks it today + +- No run against a live endpoint **with a real knob** has happened. Seam 3 is + closed and `row_from_runs` (§ 6) builds a row from two arms, but calibration + needs the *same knob measured across nearby regimes*, which needs a GPU. +- D2 does not exist, so "the effect flipped" has no agreed predicate. + +Until then the cost of `uncalibrated` is a discovery run, which is the cheap +failure. `MatchPolicy.uncalibrated_axes` reports the current state; today it is +all six. + +--- + +## 6. Worked examples + +`benchmarks/playbook/examples.json` — 6 rows, **0 selectable**, regenerated by +`python benchmarks/playbook/make_examples.py` from the repo root, with `gitm` +importable (`pip install -e .`, or `PYTHONPATH=.` — a bare script path does not +put the root on `sys.path` the way `python -m` does). The **regimes are real**, measured +off deliverable 1's sha256-pinned fixtures. The **deltas are invented** and every +row says so. + +| row | what it demonstrates | +|---|---| +| `ex1-prefix-cache-mooncake` | the ordinary case: long-input, prefix-sharing production traffic | +| `ex2-max-num-seqs-burstgpt` | the counter-example — same model and GPU, no prefix identity in the source. Synthesized prefixes but **not** a floor, because the knob does not depend on reuse | +| `ex3-chunked-prefill-qwen` | the row D3 would produce; stays illustrative until Phase B runs (needs one 80 GB card) | +| `ex4-retired-engine-bump` | invalidated by an engine bump — **kept, with a reason** | +| `ex5-scoreboard-not-production` | the biggest claimed delta in the file, gated out of every production lookup **by equality, not by distance** | +| `ex6-prefix-cache-on-a-synthesized-trace` | `delta_is_floor` — a prefix-cache knob on a source with no prefix identity | + +--- + +## 7. Not in scope + +- **Discovery mode.** The handoff is defined; the mode is not designed here. +- **The apply runtime** (Seojun's) and **env capture** (Adit's). +- **The promotion rule** (D2). D4 says what a row *is*; D2 says when one may be + created. `MeasuredDelta.latency_blowout` is the stored field where D2's + criterion-3 verdict lands, so the promotion gate and the live revert trigger + read the same value instead of each re-deriving it. +- **A calibrated threshold.** See §5. diff --git a/gitm/playbook/__init__.py b/gitm/playbook/__init__.py new file mode 100644 index 0000000..b9f9e11 --- /dev/null +++ b/gitm/playbook/__init__.py @@ -0,0 +1,82 @@ +"""Playbook schema — the contract between detection and apply. + +Deliverable 4 of the validation infrastructure. A playbook row says *this knob, +on this model revision and this GPU, under this workload regime, moved these +numbers by this much, and here is everything needed to check it.* + + from gitm.playbook import Playbook, lookup, UNCALIBRATED_POLICY + + result = lookup(playbook, query_identity, UNCALIBRATED_POLICY) + if result.route_to_discovery: + ... # conservative discovery; the handoff is defined, not the mode + else: + apply(result.row.identity.knobs) + +Matching is a **split**: exact equality on the categorical fields (model, +revision, GPU, environment, source_kind, concurrency), log2-ratio distance +combined with L-infinity on the numeric regime axes. The distance threshold is +**not calibrated yet** and the shipped policy says so — see +:mod:`gitm.playbook.match`. + +CPU-only. ``python -m gitm.playbook --selftest`` is the check. +""" + +from gitm.playbook.match import ( + AXIS_METRICS, + DEFAULT_AXES, + RATE_AXIS_DECISION, + UNCALIBRATED_POLICY, + AxisTolerance, + Candidate, + MatchPolicy, + MatchResult, + MatchStatus, + RegimeDistance, + dispersion_distance, + log2_ratio, + lookup, + regime_distance, +) +from gitm.playbook.schema import ( + PENDING_ADIT, + SCHEMA, + THROUGHPUT_METRIC, + EnvCapture, + Evidence, + Invalidation, + MeasuredDelta, + Playbook, + PlaybookRow, + Provenance, + RowIdentity, + row_from_runs, +) + +__all__ = [ + "AXIS_METRICS", + "DEFAULT_AXES", + "PENDING_ADIT", + "RATE_AXIS_DECISION", + "SCHEMA", + "THROUGHPUT_METRIC", + "UNCALIBRATED_POLICY", + "AxisTolerance", + "Candidate", + "EnvCapture", + "Evidence", + "Invalidation", + "MatchPolicy", + "MatchResult", + "MatchStatus", + "MeasuredDelta", + "Playbook", + "PlaybookRow", + "Provenance", + "RegimeDistance", + "RowIdentity", + "dispersion_distance", + "log2_ratio", + "lookup", + "regime_distance", + "row_from_runs", +] diff --git a/gitm/playbook/__main__.py b/gitm/playbook/__main__.py new file mode 100644 index 0000000..839b866 --- /dev/null +++ b/gitm/playbook/__main__.py @@ -0,0 +1,89 @@ +"""CLI for the playbook: inspect a file, measure a distance, run a lookup. + + python -m gitm.playbook --selftest + python -m gitm.playbook --show benchmarks/playbook/examples.json + python -m gitm.playbook --distance benchmarks/playbook/examples.json ex1-... ex2-... + python -m gitm.playbook --lookup benchmarks/playbook/examples.json ex2-... + +``--lookup`` takes a row id and asks the playbook what it would select *for that +row's own workload*, which is the honest way to demo a lookup without a live +server: the query is a real regime, and the answer is whatever the shipped policy +says. On the example file the answer is always "nothing" — every row is +illustrative — and that is the demonstration. + +All CPU-only. Nothing here applies a knob to anything. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +from gitm._banner import add_banner_argument, show_banner +from gitm.playbook._selftest import run_all +from gitm.playbook.match import UNCALIBRATED_POLICY, lookup, regime_distance +from gitm.playbook.schema import Playbook + + +def _load(path: str) -> Playbook: + return Playbook.model_validate(json.loads(Path(path).read_text(encoding="utf-8"))) + + +def _find(book: Playbook, row_id: str): + for row in book.rows: + if row.row_id == row_id: + return row + raise SystemExit(f"no row {row_id!r}; have: {', '.join(r.row_id for r in book.rows)}") + + +def main(argv: list[str] | None = None) -> int: + if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding="utf-8", errors="replace") + + p = argparse.ArgumentParser(prog="python -m gitm.playbook") + add_banner_argument(p) + p.add_argument("--selftest", action="store_true", help="run every check and exit") + p.add_argument("--show", metavar="PLAYBOOK") + p.add_argument("--distance", nargs=3, metavar=("PLAYBOOK", "ROW_A", "ROW_B")) + p.add_argument("--lookup", nargs=2, metavar=("PLAYBOOK", "ROW_ID")) + a = p.parse_args(argv) + show_banner(suppressed=a.no_banner) + + if a.selftest: + return run_all() + + if a.show: + book = _load(a.show) + print(f"{a.show}: {len(book.rows)} rows, {len(book.selectable())} selectable") + for row in book.rows: + print(f" {row.summary()}") + for note in row.notes: + print(f" note: {note}") + return 0 + + if a.distance: + path, a_id, b_id = a.distance + book = _load(path) + d = regime_distance( + _find(book, a_id).identity.regime, _find(book, b_id).identity.regime, UNCALIBRATED_POLICY + ) + print(f"{a_id} vs {b_id}") + print(f" {d.render()}") + return 0 + + if a.lookup: + path, row_id = a.lookup + book = _load(path) + result = lookup(book, _find(book, row_id).identity, UNCALIBRATED_POLICY) + print(result.render()) + print(f"\nroute_to_discovery: {result.route_to_discovery}") + return 0 + + p.print_help() + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/gitm/playbook/_selftest.py b/gitm/playbook/_selftest.py new file mode 100644 index 0000000..b888568 --- /dev/null +++ b/gitm/playbook/_selftest.py @@ -0,0 +1,568 @@ +"""The check that fails if the playbook schema or its match semantics break. + +One runnable thing, ``python -m gitm.playbook --selftest``, and the same +functions are the pytest cases in ``tests/test_playbook.py`` — the check a reader +is told about and the check CI runs are the *same* check. + +The regime coordinates below are the **real** ones measured off deliverable 1's +committed fixtures, so the distances asserted here describe the gap between two +real production traces rather than between two made-up ones. The *deltas* are +invented and every shipped example row says so. +""" + +from __future__ import annotations + +import json +import math +import os +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from gitm.playbook.match import ( + DEFAULT_AXES, + UNCALIBRATED_POLICY, + AxisTolerance, + MatchPolicy, + MatchStatus, + dispersion_distance, + log2_ratio, + lookup, + regime_distance, +) +from gitm.playbook.schema import ( + EnvCapture, + Evidence, + Invalidation, + MeasuredDelta, + Playbook, + PlaybookRow, + Provenance, + RowIdentity, +) +from gitm.traffic.regime import Regime, SourceKind + +#: The shipped worked examples. Data, not package content — same rule as the +#: traffic fixtures. ``$GITM_PLAYBOOK_EXAMPLES`` overrides for an installed +#: checkout. +EXAMPLES = Path( + os.environ.get( + "GITM_PLAYBOOK_EXAMPLES", + Path(__file__).resolve().parents[2] / "benchmarks" / "playbook" / "examples.json", + ) +) + +EXAMPLE_ROWS = 6 + +# --- real regime coordinates, measured off the D1 fixtures ------------------- +MOONCAKE = Regime( + source_kind=SourceKind.PRODUCTION, trace="mooncake", requests=400, + rate_rps=2.8368794326241136, io_ratio=39.09367234191124, + input_p50=9075, input_p95=49904, output_p50=370, output_p95=662, + burstiness=6.738120567375886, +) +BURSTGPT = Regime( + source_kind=SourceKind.PRODUCTION, trace="burstgpt", requests=383, + rate_rps=0.010276637419839545, io_ratio=1.8821556431490254, + input_p50=353, input_p95=1638, output_p50=238, output_p95=841, + burstiness=1.0106110910396904, +) + +#: How far apart the two real traces are, on the default axes. Pinned because it +#: is the sanity check on the whole metric: if these two collapsed to a small +#: distance the axes would not be separating anything. +BURSTGPT_VS_MOONCAKE_LINF = 4.929 # limited by input_p95 (49904 vs 1638, ~30x) + +H100 = "NVIDIA H100 80GB" +MODEL = "Qwen/Qwen3.6-35B-A3B-FP8" +REV = "95a723d0" +ENV = EnvCapture(engine="vllm", engine_version="0.11.0") +T0 = datetime(2026, 9, 2, tzinfo=timezone.utc) + + +def _identity(regime: Regime, **kw) -> RowIdentity: + base = dict( + model=MODEL, model_revision=REV, gpu_sku=H100, env=ENV, + regime=regime, knobs={"enable_prefix_caching": True}, + ) + base.update(kw) + return RowIdentity(**base) + + +def _row(row_id: str, regime: Regime, *, tput: float = 10.0, verified: datetime | None = T0, + **kw) -> PlaybookRow: + """A *measured* row — the selftest needs selectable rows, which the shipped + examples deliberately are not.""" + identity_kw = kw.pop("identity_kw", {}) + return PlaybookRow( + row_id=row_id, + identity=_identity(regime, **identity_kw), + delta=MeasuredDelta(throughput_pct=tput, ttft_p99_ms=-5.0, itl_p99_ms=0.2, repeats=5), + provenance=Provenance( + trace_source=regime.trace, trace_sha256="0" * 64, + regime_label=regime.label(), verified_at=verified, + ), + evidence=Evidence.MEASURED, + **kw, + ) + + +def _calibrated(max_distance: float, axes: tuple[str, ...] = DEFAULT_AXES) -> MatchPolicy: + return MatchPolicy( + name="test-calibrated", + axes=axes, + tolerances={ + a: AxisTolerance(max_distance=max_distance, calibration="selftest fixture, not a real run") + for a in axes + }, + ) + + +# --- the checks -------------------------------------------------------------- + +def check_log2_ratio_is_scale_free() -> None: + """The three worked numbers from the design, plus the properties behind them.""" + assert log2_ratio(1024, 2048) == 1.0 # a 2x change is exactly 1 + assert abs(log2_ratio(1024, 1536) - 0.5849625007211562) < 1e-12 + assert log2_ratio(1024, 1024) == 0.0 + + # scale-free: the same factor at any magnitude is the same distance. This is + # why a raw difference would be the wrong metric — 100 vs 200 tokens and + # 10,000 vs 20,000 are the same *kind* of mismatch. + assert log2_ratio(100, 200) == log2_ratio(10_000, 20_000) == 1.0 + # symmetric + assert log2_ratio(2048, 1024) == log2_ratio(1024, 2048) + # zero is not a small number: one side zero is incomparable, both zero is equal + assert math.isinf(log2_ratio(0, 256)) + assert log2_ratio(0, 0) == 0.0 + + +def check_dispersion_distance_handles_flat_traces() -> None: + """``D = 0`` is a real trace (perfectly paced), not an incomparable one.""" + assert dispersion_distance(0.0, 0.0) == 0.0 + assert dispersion_distance(0.0, 1.0) == 1.0 # flat vs Poisson, one unit + # the two real traces are far apart on this axis, correctly + d = dispersion_distance(BURSTGPT.burstiness, MOONCAKE.burstiness) + assert 1.9 < d < 2.0, d + # and a nearby dispersion is near + assert dispersion_distance(5.0, MOONCAKE.burstiness) < 0.4 + # a raw log2 ratio would have blown up here; the shift is what prevents it + assert math.isinf(log2_ratio(0.0, 1.0)) + + +def check_linf_is_the_worst_axis() -> None: + """A mean would let four close axes hide one that breaks the row.""" + d = regime_distance(BURSTGPT, MOONCAKE, UNCALIBRATED_POLICY) + assert d.limiting_axis == "input_p95", d.per_axis + assert abs(d.linf - BURSTGPT_VS_MOONCAKE_LINF) < 0.001, d.render() + assert d.linf == max(d.per_axis.values()) + + # The case L-inf exists for: identical on five axes, 8x off on the sixth. + # The mean calls that a 0.5 mismatch; L-inf calls it a 3.0 mismatch. The + # workload is a long-context one against a short-context row, and the mean + # is the reading that would apply the row. + long_ctx = MOONCAKE.model_copy(update={"input_p95": MOONCAKE.input_p95 * 8}) + one_axis = regime_distance(MOONCAKE, long_ctx, UNCALIBRATED_POLICY) + assert one_axis.limiting_axis == "input_p95" and one_axis.linf == 3.0 + assert sum(one_axis.per_axis.values()) / len(one_axis.per_axis) == 0.5 + + # a regime against itself is exactly zero on every axis + same = regime_distance(MOONCAKE, MOONCAKE, UNCALIBRATED_POLICY) + assert same.exact and set(same.per_axis.values()) == {0.0} + + +def check_rate_is_not_in_the_default_axes() -> None: + """``rate_rps`` is excluded by decision, and the decision is material.""" + assert "rate_rps" not in DEFAULT_AXES + assert "rate_rps" in regime_distance( + BURSTGPT, MOONCAKE, MatchPolicy(name="with-rate", axes=(*DEFAULT_AXES, "rate_rps")) + ).per_axis + + # Two regimes identical except for offered rate: distance 0 by default, and + # a large distance the moment rate is included. If including it were a + # no-op the decision would not need making. + slower = MOONCAKE.model_copy(update={"rate_rps": MOONCAKE.rate_rps / 8}) + assert regime_distance(MOONCAKE, slower, UNCALIBRATED_POLICY).exact + with_rate = MatchPolicy(name="with-rate", axes=(*DEFAULT_AXES, "rate_rps")) + assert abs(regime_distance(MOONCAKE, slower, with_rate).linf - 3.0) < 1e-9 + + +def check_tolerance_requires_calibration() -> None: + """A threshold cannot enter a policy without the experiment that set it.""" + with pytest.raises(ValidationError, match="max_distance without calibration"): + AxisTolerance(max_distance=1.0) + with pytest.raises(ValidationError): + AxisTolerance(max_distance=-0.5, calibration="whatever") + ok = AxisTolerance(max_distance=1.0, calibration="prereg E4: sign flip at 1.4 on input_p95") + assert ok.calibrated + # and the shipped policy has none of them + assert UNCALIBRATED_POLICY.uncalibrated_axes == DEFAULT_AXES + assert not any(UNCALIBRATED_POLICY.tolerance(a).calibrated for a in DEFAULT_AXES) + + +def check_exact_gates_reject_before_distance() -> None: + """Categorical mismatches are rejections, never a large distance.""" + book = Playbook(rows=[_row("r-mooncake", MOONCAKE)]) + for label, kw in [ + ("gpu_sku", {"gpu_sku": "NVIDIA A100 80GB"}), + ("model", {"model": "meta-llama/Llama-4-70B"}), + ("model_revision", {"model_revision": "deadbeef"}), + ("engine_version", {"env": EnvCapture(engine="vllm", engine_version="0.12.0")}), + ("knob set", {"knobs": {"max_num_seqs": 64}}), + ]: + res = lookup(book, _identity(MOONCAKE, **kw), UNCALIBRATED_POLICY) + assert res.status is MatchStatus.NO_MATCH, (label, res.render()) + assert res.route_to_discovery + assert label.split("_")[0] in res.rejected["r-mooncake"], (label, res.rejected) + + # source_kind: a scoreboard row is gated out of a production query even + # though every numeric axis is identical. + board = MOONCAKE.model_copy(update={"source_kind": SourceKind.SCOREBOARD}) + assert regime_distance(MOONCAKE, board, UNCALIBRATED_POLICY).exact + res = lookup(Playbook(rows=[_row("r-board", board)]), _identity(MOONCAKE), UNCALIBRATED_POLICY) + assert res.status is MatchStatus.NO_MATCH + assert "scoreboard" in res.rejected["r-board"] + + # concurrency: exact by policy, and turning the gate off is a named edit + capped = MOONCAKE.model_copy(update={"concurrency": 64}) + book_capped = Playbook(rows=[_row("r-capped", capped)]) + assert lookup(book_capped, _identity(MOONCAKE), UNCALIBRATED_POLICY).status is MatchStatus.NO_MATCH + loose = MatchPolicy(name="ignore-concurrency", match_concurrency=False) + assert lookup(book_capped, _identity(MOONCAKE), loose).status is MatchStatus.EXACT_REGIME + + +def check_uncalibrated_policy_routes_to_discovery() -> None: + """The headline: a near miss is not a match until an axis is calibrated.""" + near = MOONCAKE.model_copy(update={"output_p50": 400}) # ~0.11 away, one axis + res = lookup(Playbook(rows=[_row("r-near", near)]), _identity(MOONCAKE), UNCALIBRATED_POLICY) + assert res.status is MatchStatus.UNCALIBRATED + assert res.route_to_discovery and res.row is None + assert "not yet calibrated" in res.reason + assert "output_p50" in res.reason # the reason names the axis, not just a number + # the candidate is still reported — a miss has to be actionable + assert [c.row.row_id for c in res.candidates] == ["r-near"] + assert res.candidates[0].distance.limiting_axis == "output_p50" + + +def check_exact_regime_matches_without_any_calibration() -> None: + """Distance 0 needs no threshold, so the schema is usable on day one.""" + res = lookup(Playbook(rows=[_row("r-exact", MOONCAKE)]), _identity(MOONCAKE), UNCALIBRATED_POLICY) + assert res.status is MatchStatus.EXACT_REGIME + assert not res.route_to_discovery + assert res.row is not None and res.row.row_id == "r-exact" + assert res.distance is not None and res.distance.exact + + +def check_calibrated_policy_matches_inside_and_rejects_outside() -> None: + """Once an axis is calibrated, near means near — and far still means no.""" + near = MOONCAKE.model_copy(update={"output_p50": 400}) # log2(400/370) = 0.112 + far = MOONCAKE.model_copy(update={"output_p50": 1480}) # log2(1480/370) = 2.0 + book_near = Playbook(rows=[_row("r-near", near)]) + book_far = Playbook(rows=[_row("r-far", far)]) + policy = _calibrated(0.5) + + ok = lookup(book_near, _identity(MOONCAKE), policy) + assert ok.status is MatchStatus.NEAR_REGIME and ok.row is not None + assert not ok.route_to_discovery + + no = lookup(book_far, _identity(MOONCAKE), policy) + assert no.status is MatchStatus.NO_MATCH and no.route_to_discovery + assert "output_p50" in no.reason and "2.0" in no.reason + + # the two real traces are far outside any plausible tolerance + assert lookup( + Playbook(rows=[_row("r-burstgpt", BURSTGPT)]), _identity(MOONCAKE), _calibrated(2.0) + ).status is MatchStatus.NO_MATCH + + +def check_precedence_exact_then_recent_then_conservative() -> None: + """Three tie-breaks, in order, each demonstrated on its own.""" + near = MOONCAKE.model_copy(update={"output_p50": 400}) + policy = _calibrated(0.5) + + # 1. exact regime beats nearest regime, even with a smaller claimed delta + res = lookup( + Playbook(rows=[_row("near-big", near, tput=99.0), _row("exact-small", MOONCAKE, tput=1.0)]), + _identity(MOONCAKE), policy, + ) + assert res.row is not None and res.row.row_id == "exact-small" + assert res.status is MatchStatus.EXACT_REGIME + + # 2. among equals, the most recently verified wins + res = lookup( + Playbook(rows=[ + _row("stale", MOONCAKE, verified=T0 - timedelta(days=30)), + _row("fresh", MOONCAKE, verified=T0), + ]), + _identity(MOONCAKE), policy, + ) + assert res.row is not None and res.row.row_id == "fresh" + + # 3. equally close and equally fresh: the smaller claim wins, because a + # wrong row inside the live window costs more than a missed one + res = lookup( + Playbook(rows=[_row("bold", MOONCAKE, tput=40.0), _row("modest", MOONCAKE, tput=3.0)]), + _identity(MOONCAKE), policy, + ) + assert res.row is not None and res.row.row_id == "modest" + + +def check_synthesized_prefixes_make_a_prefix_cache_delta_a_floor() -> None: + """D1-11: BurstGPT has no prefix identity, so its reuse number is a bound. + + D1 synthesizes unique blocks per request, which invents no sharing the source + never had. A prefix-cache knob measured there saw the *least* reuse possible, + and the row has to say so or the floor gets quoted as the gain. + """ + row = _row("r-synth", BURSTGPT) + assert not row.delta_is_floor # prefix_synthesized defaults False + synth = row.model_copy(update={ + "provenance": row.provenance.model_copy(update={"prefix_synthesized": True}) + }) + assert synth.delta_is_floor + assert "FLOOR" in synth.summary() + + # a non-prefix knob on the same synthesized trace is unaffected — the + # synthesis only distorts what depends on reuse + other = synth.model_copy(update={ + "identity": _identity(BURSTGPT, knobs={"max_num_seqs": 64}) + }) + assert not other.delta_is_floor + + # the marker is a substring, so a renamed flag still trips it + renamed = synth.model_copy(update={ + "identity": _identity(BURSTGPT, knobs={"enable-prefix-caching-v2": True}) + }) + assert renamed.delta_is_floor + + +def check_extrapolated_rows_lose_ties_but_not_distance() -> None: + """``/xenv`` is a tie-break below distance, and the ordering is asserted.""" + xenv = MOONCAKE.model_copy(update={"in_envelope": False}) + policy = _calibrated(1.0) + + # equal distance: the in-envelope row wins + res = lookup( + Playbook(rows=[_row("extrapolated", xenv), _row("measured", MOONCAKE)]), + _identity(MOONCAKE), policy, + ) + assert res.row is not None and res.row.row_id == "measured" + + # nearer-but-extrapolated still beats a far in-envelope row: distance is the + # workload question and comes first (this is where we depart from todo.md) + far = MOONCAKE.model_copy(update={"output_p50": 640}) # 0.79 away + res = lookup( + Playbook(rows=[_row("near-xenv", xenv), _row("far-inenv", far)]), + _identity(MOONCAKE), policy, + ) + assert res.row is not None and res.row.row_id == "near-xenv" + + +def check_replay_conditions_are_carried_into_provenance() -> None: + """The 512-vs-16 finding is a field, so a wrong-block-size row is checkable.""" + row = _row("r-mooncake", MOONCAKE) + assert row.provenance.replay_chunk_hash_size is None # unrecorded, not assumed + pinned = row.model_copy(update={ + "provenance": row.provenance.model_copy( + update={"replay_chunk_hash_size": 512, "replay_self_timed": True} + ) + }) + assert pinned.provenance.replay_chunk_hash_size == 512 + # and D2's stored verdict has somewhere to go without being recomputed + assert pinned.delta.latency_blowout is None + assert pinned.delta.model_copy(update={"latency_blowout": True}).latency_blowout is True + + +def check_invalidated_row_is_kept_and_never_selected() -> None: + """Retirement is a field with a reason, not a deletion.""" + dead = _row("r-dead", MOONCAKE) + dead = dead.model_copy(update={ + "invalidated": Invalidation(reason="vLLM 0.11 -> 0.12 scheduler rewrite", at=T0) + }) + book = Playbook(rows=[dead]) + assert len(book.rows) == 1 and book.selectable() == [] + res = lookup(book, _identity(MOONCAKE), UNCALIBRATED_POLICY) + assert res.status is MatchStatus.NO_MATCH + assert "scheduler rewrite" in res.rejected["r-dead"] # the reason survives the miss + + +def check_a_row_cannot_exist_without_its_evidence() -> None: + """The type refuses rows that could not have come through the promotion rule.""" + with pytest.raises(ValidationError, match="knobs is required"): + _identity(MOONCAKE, knobs={}) + with pytest.raises(ValidationError, match="single run has no variance"): + MeasuredDelta(throughput_pct=8.0, ttft_p99_ms=-1.0, itl_p99_ms=0.0, repeats=1) + with pytest.raises(ValidationError): # trace_sha256 is required + Provenance(trace_source="mooncake", trace_sha256="", regime_label="x") + with pytest.raises(ValidationError): # throughput alone cannot be promoted (D2 criterion 3) + MeasuredDelta(throughput_pct=8.0, repeats=5) + + +def check_examples_ship_nothing_selectable() -> None: + """Every worked example is labelled, and none of them can be applied.""" + book = Playbook.model_validate(json.loads(EXAMPLES.read_text(encoding="utf-8"))) + assert len(book.rows) == EXAMPLE_ROWS + assert all(r.evidence is Evidence.ILLUSTRATIVE for r in book.rows) + assert book.selectable() == [] # an example in a live file is the failure mode + + # round-trips byte-identically: the file is the schema, not a rendering of it + assert json.loads(json.dumps(book.model_dump(mode="json"))) == json.loads( + EXAMPLES.read_text(encoding="utf-8") + ) + + # the illustrative rows carry real regimes and real trace checksums + by_id = {r.row_id: r for r in book.rows} + assert by_id["ex1-prefix-cache-mooncake"].identity.regime.input_p50 == MOONCAKE.input_p50 + assert by_id["ex2-max-num-seqs-burstgpt"].identity.regime.input_p50 == BURSTGPT.input_p50 + assert all(len(r.provenance.trace_sha256) == 64 for r in book.rows) + # the biggest claimed delta in the file is the scoreboard row, gated by equality + assert max(book.rows, key=lambda r: r.delta.throughput_pct).row_id == ( + "ex5-scoreboard-not-production" + ) + # exactly one example is a floor, and it is the prefix-cache knob on the + # source with no prefix identity — both halves of D1-11 have an example + floors = [r.row_id for r in book.rows if r.delta_is_floor] + assert floors == ["ex6-prefix-cache-on-a-synthesized-trace"], floors + assert by_id["ex2-max-num-seqs-burstgpt"].provenance.prefix_synthesized + assert not by_id["ex2-max-num-seqs-burstgpt"].delta_is_floor # synthesized != floor + assert all(r.provenance.replay_chunk_hash_size == 512 for r in book.rows) + + # and lookup refuses all of them even for a perfectly matching query + res = lookup(book, _identity(MOONCAKE), UNCALIBRATED_POLICY) + assert res.status is MatchStatus.NO_MATCH + assert all("never selectable" in w or "invalidated" in w for w in res.rejected.values()) + + +def check_regime_is_imported_not_redeclared() -> None: + """One coordinate system. A second Regime here would drift within a week.""" + import gitm.traffic.regime as d1 + + assert RowIdentity.model_fields["regime"].annotation is d1.Regime + # and every distance axis is a real field on it, by name + from gitm.playbook.match import AXIS_METRICS + + assert set(AXIS_METRICS) <= set(d1.Regime.model_fields) + + +# --- the last mile: seam 3's records become a row ---------------------------- +#: Deltas injected into the treatment arm's copy of the real result JSON. The +#: *runs* are real (the committed 0.28.0 result); the difference between the arms +#: is constructed, because a second real arm needs a knob and a GPU. What is +#: under test is the arithmetic and the refusals, not the numbers. +ARM_TPUT_GAIN = 1.10 +ARM_TTFT_DELTA_MS = -40.0 + + +def _runs(**bump): + """N ``BenchRun`` records off the committed real result, optionally bumped.""" + from gitm.traffic._selftest import _real_run + from gitm.traffic.results import join_result + + result, plan, reg, _ = _real_run() + return result, plan, reg, (lambda n=2, **kw: [join_result({**result, **kw}, plan, reg) + for _ in range(n)]) + + +def check_two_bench_runs_become_a_row() -> None: + """The last mile. Seam 3 makes one arm; a row is the difference between two. + + Before this existed, nothing could populate a row end to end no matter how + complete the join was — which is why every shipped example is illustrative. + """ + from gitm.playbook.schema import PENDING_ADIT, row_from_runs + + result, _, reg, mk = _runs() + base = mk(2) + treat = mk(2, + output_throughput=result["output_throughput"] * ARM_TPUT_GAIN, + p99_ttft_ms=result["p99_ttft_ms"] + ARM_TTFT_DELTA_MS) + + row = row_from_runs( + "r-from-real-runs", base, treat, + model=MODEL, model_revision=REV, gpu_sku=H100, env=ENV, + knobs={"enable_prefix_caching": True}, + ) + + # the arithmetic + assert row.delta.repeats == 2 + assert math.isclose(row.delta.throughput_pct, (ARM_TPUT_GAIN - 1) * 100, rel_tol=1e-9) + assert math.isclose(row.delta.ttft_p99_ms, ARM_TTFT_DELTA_MS, rel_tol=1e-9) + + # provenance came off the runs, not off the caller — the whole point + assert row.provenance.trace_sha256 == base[0].source.sha256 + assert len(row.provenance.trace_sha256) == 64 + assert row.provenance.regime_label == reg.label() + assert row.provenance.replay_chunk_hash_size == 512 + assert row.provenance.replay_self_timed is True + assert row.identity.regime == reg + + # measured, and therefore selectable — the first row in the repo that is + assert row.evidence is Evidence.MEASURED and row.selectable + + # ...and it still says what it does not know + assert row.provenance.config_capture == PENDING_ADIT + assert row.provenance.promotion_rule.startswith(PENDING_ADIT) + assert any("R1" in n for n in row.notes), row.notes + assert row.delta.latency_blowout is None # D2 owns the predicate + assert row.delta.throughput_ci95_pct is None # D2 owns the variance rule + + +def check_a_row_refuses_arms_that_are_not_one_experiment() -> None: + """Every way a pair of runs is not a delta. Each is a wrong row prevented.""" + from gitm.playbook.schema import row_from_runs + + _, _, reg, mk = _runs() + ident = dict(model=MODEL, model_revision=REV, gpu_sku=H100, env=ENV, + knobs={"enable_prefix_caching": True}) + + def refused(base, treat, needle): + with pytest.raises((ValueError, ValidationError)) as e: + row_from_runs("r-bad", base, treat, **ident) + assert needle in str(e.value), str(e.value) + + refused(mk(2), mk(1), "unequal arms") # the interleave broke + refused(mk(1), mk(1), "repeats=1") # delegated to MeasuredDelta + refused(mk(2), mk(2, failed=1), "promotable") # a run with failures + refused(mk(2), mk(2, completed=39), "promotable") # did not replay the trace + refused([], mk(2), "both arms are required") + + # two different workloads: same result JSON, a regime that labels differently + from gitm.traffic.results import join_result + result, plan, _, _ = _runs() + other = reg.model_copy(update={"input_p50": 100, "input_p95": 200}) + assert other.label() != reg.label() + refused(mk(2), [join_result(result, plan, other) for _ in range(2)], + "did not run the same workload") + + +CHECKS = ( + check_log2_ratio_is_scale_free, + check_dispersion_distance_handles_flat_traces, + check_linf_is_the_worst_axis, + check_rate_is_not_in_the_default_axes, + check_tolerance_requires_calibration, + check_exact_gates_reject_before_distance, + check_uncalibrated_policy_routes_to_discovery, + check_exact_regime_matches_without_any_calibration, + check_calibrated_policy_matches_inside_and_rejects_outside, + check_precedence_exact_then_recent_then_conservative, + check_extrapolated_rows_lose_ties_but_not_distance, + check_synthesized_prefixes_make_a_prefix_cache_delta_a_floor, + check_replay_conditions_are_carried_into_provenance, + check_invalidated_row_is_kept_and_never_selected, + check_a_row_cannot_exist_without_its_evidence, + check_examples_ship_nothing_selectable, + check_regime_is_imported_not_redeclared, + check_two_bench_runs_become_a_row, + check_a_row_refuses_arms_that_are_not_one_experiment, +) + + +def run_all() -> int: + for fn in CHECKS: + fn() + print(f" ok {fn.__name__}") + print(f"selftest ok -- {len(CHECKS)} checks, 2 real regimes, 0 calibrated axes") + return 0 diff --git a/gitm/playbook/match.py b/gitm/playbook/match.py new file mode 100644 index 0000000..2e81044 --- /dev/null +++ b/gitm/playbook/match.py @@ -0,0 +1,452 @@ +"""Match semantics — which past result applies to the workload in front of you. + +The whole deliverable turns on this module, because "close enough" is where a +wrong row gets applied. The design is a **split**, not a single score: + + exact equality model + revision, GPU SKU, environment, source_kind, + concurrency policy -> a gate: pass or reject + distance the numeric regime axes -> a ranking among survivors + +Categorical fields are gated because "nearly an H100" is not a thing, and a +distance that mixed a GPU mismatch into the same number as a token-count +mismatch would let a large enough workload similarity outvote running on +different silicon. + +**The numeric axes are compared as log2 ratios**, which is the natural metric for +token counts and rates — what matters is the *factor*, not the difference: + + 1,024 vs 2,048 tokens -> 1.0 (a 2x change) + 1,024 vs 1,536 tokens -> 0.58 + 1,024 vs 1,024 tokens -> 0.0 + +and combined with **L-infinity** (the max across axes), not a mean or a Euclidean +norm. A mean lets a close match on four axes hide a 4x mismatch on the fifth, and +the fifth is the one that breaks the row. L-inf says: a row is as far away as its +worst axis. + +**The threshold is not known, and this module says so rather than picking one.** +A number like ``max_distance = 1.0`` reads as calibrated and is not: under log2 +it means "accept up to a 2x mismatch on every axis at once", which may well be +safe for output p50 and is certainly not safe for long-context input p95, for +prefix-cache reuse, or for a queue-sensitive scheduling policy. So +:class:`AxisTolerance` **refuses to hold a number without the experiment that +produced it**, the shipped :data:`UNCALIBRATED_POLICY` has no numbers at all, and +a lookup that would need one returns :attr:`MatchStatus.UNCALIBRATED` and routes +to conservative discovery. + +Calibrating an axis, which is what removes that status: + +1. Run the same knob across nearby regimes, varying **one** axis at a time. +2. Find where the effect changes sign, or where the latency percentile criterion + from deliverable 2 flips from pass to fail. +3. Set that axis's tolerance strictly inside the distance at which it flipped. +4. The L-inf limit is then the strictest relevant per-axis tolerance, by + construction — there is no separate global number to choose. + +Until step 1 has data, the honest state is "not yet calibrated", and the cost of +that state is a discovery run, which is the cheap failure. +""" + +from __future__ import annotations + +import math +from datetime import datetime, timezone +from enum import Enum + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from gitm.playbook.schema import Playbook, PlaybookRow, RowIdentity +from gitm.traffic.regime import Regime + +#: Distance returned when an axis cannot be compared at all — one side is zero, +#: the other is not. Not 0.0 and not "skip": an incomparable axis is a reason to +#: reject a row, and infinity is what makes L-inf say so without a special case. +INCOMPARABLE = math.inf + + +def log2_ratio(a: float, b: float) -> float: + """``|log2(a / b)|`` — the distance between two positive magnitudes. + + Symmetric, zero at equality, 1.0 at a factor of two, and scale-free: 100 vs + 200 tokens is the same distance as 10,000 vs 20,000, which is the property a + token-count axis needs. Two exact zeros are equal; one zero is + :data:`INCOMPARABLE`, because "no output tokens at all" is not a small + version of "some output tokens". + """ + if a == b: + return 0.0 + if a <= 0 or b <= 0: + return INCOMPARABLE + return abs(math.log2(a / b)) + + +def dispersion_distance(a: float, b: float) -> float: + """Distance between two index-of-dispersion values, ``|log2((1+a)/(1+b))|``. + + Burstiness cannot use :func:`log2_ratio` directly: a perfectly paced trace + has ``D = 0`` and a bare ratio would make it incomparable to everything, + including another paced trace. The ``1 +`` shift anchors the axis so that the + Poisson reference ``D = 1`` sits one unit from flat ``D = 0``, and the two + real traces deliverable 1 measured land where intuition puts them: + + flat (0.0) vs poisson (1.0) -> 1.00 + burstgpt (1.01) vs mooncake (6.74) -> 1.95 (far apart, correctly) + moderate (5.0) vs mooncake (6.74) -> 0.37 (near, correctly) + """ + return log2_ratio(1.0 + a, 1.0 + b) + + +#: The numeric axes, and how each is compared. Names match +#: :class:`~gitm.traffic.regime.Regime` fields exactly — an axis that cannot be +#: read off a Regime by name is an axis that will silently stop being computed. +AXIS_METRICS = { + "input_p50": log2_ratio, + "input_p95": log2_ratio, + "output_p50": log2_ratio, + "output_p95": log2_ratio, + "io_ratio": log2_ratio, + "burstiness": dispersion_distance, + "rate_rps": log2_ratio, +} + +#: Axes on by default. **``rate_rps`` is deliberately absent.** It exists on +#: ``Regime`` and adding it to the distance because it is there would be exactly +#: the mistake this module is written to avoid: offered rate is largely captured +#: by burstiness plus the concurrency gate, and a knob that is insensitive to +#: rate would then be rejected for a workload it fits. Turning it on is a +#: decision with evidence behind it — see :data:`RATE_AXIS_DECISION`. +DEFAULT_AXES = ("input_p50", "input_p95", "output_p50", "output_p95", "io_ratio", "burstiness") + +RATE_AXIS_DECISION = """\ +rate_rps is not in the distance by default. Include it when knob outcomes are +shown to depend materially on offered load *after* concurrency and burstiness are +accounted for; omit it when they are not. Either way the decision is recorded +with the experiment that settled it, not inferred from the field existing.""" + + +class AxisTolerance(BaseModel): + """How far this axis may differ, and the experiment that says so. + + ``max_distance=None`` means **uncalibrated** — the axis is compared and + reported, but no nonzero distance on it can be accepted automatically. + + A number without ``calibration`` is rejected at construction. That is the + enforcement behind "the threshold is currently unknown": the only way to get + a tolerance into a policy is to name the run that produced it, so a + placeholder can never quietly become a production constant. + """ + + model_config = ConfigDict(extra="forbid") + + max_distance: float | None = None + #: What measured it. Free text pointing at a run or a spec section, e.g. + #: "prereg_rank1 E4, 2026-09-14: sign flip at 1.4 on input_p95". + calibration: str | None = None + + @model_validator(mode="after") + def _a_number_needs_a_reason(self) -> AxisTolerance: + if self.max_distance is not None and not self.calibration: + raise ValueError( + "max_distance without calibration: a tolerance is a measured " + "quantity, not a default. Run the knob across nearby regimes, " + "find where the effect flips, and cite it here." + ) + if self.max_distance is not None and self.max_distance < 0: + raise ValueError("max_distance must be >= 0") + return self + + @property + def calibrated(self) -> bool: + return self.max_distance is not None + + +class MatchPolicy(BaseModel): + """Which axes count, how far each may stray, and what is gated exactly.""" + + model_config = ConfigDict(extra="forbid") + + name: str + axes: tuple[str, ...] = DEFAULT_AXES + tolerances: dict[str, AxisTolerance] = Field(default_factory=dict) + #: Exact-match gates. Each is a policy choice, listed so that loosening one + #: is an edit to a named field rather than an accident in a comparison. + match_source_kind: bool = True + match_concurrency: bool = True + match_env: bool = True + + @model_validator(mode="after") + def _axes_are_real(self) -> MatchPolicy: + unknown = [a for a in self.axes if a not in AXIS_METRICS] + if unknown: + raise ValueError(f"unknown regime axes {unknown}; known: {sorted(AXIS_METRICS)}") + stray = [a for a in self.tolerances if a not in self.axes] + if stray: + raise ValueError(f"tolerance set for axes not in the policy: {stray}") + return self + + def tolerance(self, axis: str) -> AxisTolerance: + return self.tolerances.get(axis, AxisTolerance()) + + @property + def uncalibrated_axes(self) -> tuple[str, ...]: + return tuple(a for a in self.axes if not self.tolerance(a).calibrated) + + +#: The policy that ships. Every axis uncalibrated, so the only automatic match is +#: an **exact** regime match and everything else routes to discovery. This is not +#: a placeholder to be edited in passing — replacing it means calibrating the +#: axes, and :class:`AxisTolerance` will not let a number in without the run. +UNCALIBRATED_POLICY = MatchPolicy(name="uncalibrated") + + +class MatchStatus(str, Enum): + """The outcome of a lookup. Four states, and three of them are not a row.""" + + EXACT_REGIME = "exact_regime" # distance 0 on every axis; safe to apply + NEAR_REGIME = "near_regime" # within calibrated tolerances + UNCALIBRATED = "uncalibrated" # candidates exist, but no axis is calibrated + NO_MATCH = "no_match" # nothing passed the exact gates + + +class RegimeDistance(BaseModel): + """Per-axis distances and the L-inf that summarizes them. + + Both halves are kept. The L-inf is what a threshold compares against; the + per-axis dict is what tells a human *which* axis put the row out of range, + which is the only actionable half when a lookup misses. + """ + + model_config = ConfigDict(extra="forbid") + + per_axis: dict[str, float] + linf: float + limiting_axis: str | None + + @property + def exact(self) -> bool: + return self.linf == 0.0 + + def render(self) -> str: + parts = " ".join( + f"{a}={'inf' if math.isinf(d) else format(d, '.3f')}" + for a, d in sorted(self.per_axis.items()) + ) + lim = f" (limited by {self.limiting_axis})" if self.limiting_axis else "" + return f"L-inf {self.linf:.3f}{lim} [{parts}]" + + +def regime_distance(a: Regime, b: Regime, policy: MatchPolicy = UNCALIBRATED_POLICY) -> RegimeDistance: + """Distance between two regimes on the policy's axes, combined with L-inf.""" + per_axis = {axis: AXIS_METRICS[axis](getattr(a, axis), getattr(b, axis)) for axis in policy.axes} + if not per_axis: + return RegimeDistance(per_axis={}, linf=0.0, limiting_axis=None) + limiting = max(per_axis, key=lambda k: per_axis[k]) + # An exact match has no limiting axis. Naming one would read as "this is the + # axis that nearly failed", which is the opposite of what a 0.0 means. + return RegimeDistance( + per_axis=per_axis, + linf=per_axis[limiting], + limiting_axis=limiting if per_axis[limiting] > 0 else None, + ) + + +def _gate(query: RowIdentity, row: RowIdentity, policy: MatchPolicy) -> str: + """Exact-match gate. Returns ``""`` when the row passes, else why it did not.""" + if query.model != row.model: + return f"model {row.model!r} != {query.model!r}" + if query.model_revision != row.model_revision: + return f"model_revision {row.model_revision} != {query.model_revision}" + if query.gpu_sku != row.gpu_sku: + return f"gpu_sku {row.gpu_sku!r} != {query.gpu_sku!r}" + if policy.match_env: + ok, why = query.env.compatible_with(row.env) + if not ok: + return why + if policy.match_source_kind and query.regime.source_kind is not row.regime.source_kind: + return ( + f"source_kind {row.regime.source_kind.value} != {query.regime.source_kind.value}" + " — a scoreboard result is not evidence about production traffic" + ) + if policy.match_concurrency and query.regime.concurrency != row.regime.concurrency: + return f"concurrency {row.regime.concurrency} != {query.regime.concurrency}" + if query.knobs.keys() != row.knobs.keys(): + return f"knob set {sorted(row.knobs)} != {sorted(query.knobs)}" + return "" + + +class Candidate(BaseModel): + """A row that passed the gates, with how far its regime is from the query.""" + + model_config = ConfigDict(extra="forbid") + + row: PlaybookRow + distance: RegimeDistance + + +class MatchResult(BaseModel): + """What a lookup found, and — when it found nothing — exactly why. + + ``rejected`` is not diagnostics-for-later. A miss that cannot say which gate + it failed sends someone to read the whole playbook by hand, and a miss is the + common case for a schema this young. + """ + + model_config = ConfigDict(extra="forbid") + + status: MatchStatus + row: PlaybookRow | None = None + distance: RegimeDistance | None = None + candidates: list[Candidate] = Field(default_factory=list) + rejected: dict[str, str] = Field(default_factory=dict) # row_id -> why + policy: str = UNCALIBRATED_POLICY.name + reason: str = "" + + @property + def route_to_discovery(self) -> bool: + """Whether the caller must fall back to conservative discovery mode. + + True for everything that is not a returned row. Deliverable 4 defines + this handoff and **not** discovery itself: what a caller needs from the + schema is an unambiguous "I have nothing for you", and a status that is + sometimes a row and sometimes a suggestion is how a wrong row gets + applied in a 72-hour window. + """ + return self.row is None + + def render(self) -> str: + head = f"{self.status.value} (policy: {self.policy})" + if self.row is not None and self.distance is not None: + return f"{head}\n {self.row.summary()}\n {self.distance.render()}" + lines = [head, f" {self.reason}"] if self.reason else [head] + for c in self.candidates: + lines.append(f" candidate {c.row.row_id}: {c.distance.render()}") + for row_id, why in sorted(self.rejected.items()): + lines.append(f" rejected {row_id}: {why}") + return "\n".join(lines) + + +def _precedence(c: Candidate) -> tuple[float, int, float, float]: + """Sort key: nearest, then measured-in-envelope, then freshest, then smallest. + + Four terms, in this order and for these reasons: + + 1. **Distance.** The workload question comes first; everything else is a + tie-break among rows that answer it equally well. + 2. **In envelope.** A row measured at a point deliberately sampled *beyond* + any observed trace (D1 marks these ``/xenv``) is weaker evidence than one + measured inside it. This sits below distance rather than above it — + ``todo.md`` had it above — because a nearby extrapolated point was still + genuinely run, and preferring a 4x-away in-envelope row over it answers + the wrong question. + 3. **Recency.** Among equals, the most recently verified. + 4. **Smallest claim.** The conservative tie-break from the plan: prefer the + row claiming **less**, because a wrong row applied inside the live window + costs more than a missed opportunity. + """ + verified = c.row.provenance.verified_at + recency = -(verified or datetime.min.replace(tzinfo=timezone.utc)).timestamp() + extrapolated = 0 if c.row.identity.regime.in_envelope else 1 + return (c.distance.linf, extrapolated, recency, abs(c.row.delta.throughput_pct)) + + +def lookup( + playbook: Playbook, + query: RowIdentity, + policy: MatchPolicy = UNCALIBRATED_POLICY, +) -> MatchResult: + """Find the row that applies to ``query``, or say why none does. + + Order of operations, and each step can only ever *reject*: + + 1. **Gate** on the exact fields. Model, revision, GPU, environment, + source_kind, concurrency, and the knob set being asked about. + 2. **Measure** the regime distance for the survivors. + 3. **Decide.** Distance 0 on every axis is an exact regime match and is + returned. A nonzero distance needs a calibrated tolerance on every axis it + is nonzero along; without one the result is + :attr:`MatchStatus.UNCALIBRATED` and the caller goes to discovery. + 4. **Break ties** by :func:`_precedence`. + + Unselectable rows — invalidated, or worked examples — never reach step 2. + """ + result_rejected: dict[str, str] = {} + candidates: list[Candidate] = [] + + for row in playbook.rows: + if not row.selectable: + result_rejected[row.row_id] = ( + "invalidated: " + row.invalidated.reason + if row.invalidated + else f"evidence={row.evidence.value}; never selectable" + ) + continue + why = _gate(query, row.identity, policy) + if why: + result_rejected[row.row_id] = why + continue + candidates.append(Candidate(row=row, distance=regime_distance(query.regime, row.identity.regime, policy))) + + if not candidates: + return MatchResult( + status=MatchStatus.NO_MATCH, + rejected=result_rejected, + policy=policy.name, + reason="no row passed the exact-match gates", + ) + + candidates.sort(key=_precedence) + best = candidates[0] + + if best.distance.exact: + return MatchResult( + status=MatchStatus.EXACT_REGIME, + row=best.row, + distance=best.distance, + candidates=candidates, + rejected=result_rejected, + policy=policy.name, + ) + + # Nonzero distance: every axis it is nonzero along must have a calibrated + # tolerance, and must be inside it. An uncalibrated axis is not "probably + # fine" — it is an axis nobody has measured the knob across. + uncalibrated = [ + a for a, d in best.distance.per_axis.items() if d > 0 and not policy.tolerance(a).calibrated + ] + if uncalibrated: + return MatchResult( + status=MatchStatus.UNCALIBRATED, + candidates=candidates, + rejected=result_rejected, + policy=policy.name, + reason=( + f"nearest row {best.row.row_id} is {best.distance.linf:.3f} away, limited by " + f"{best.distance.limiting_axis}; no calibrated tolerance for {sorted(uncalibrated)}. " + "Automatic regime matching is not yet calibrated — routing to discovery." + ), + ) + + over = [ + (a, d) for a, d in best.distance.per_axis.items() if d > (policy.tolerance(a).max_distance or 0.0) + ] + if over: + axis, dist = max(over, key=lambda t: t[1]) + return MatchResult( + status=MatchStatus.NO_MATCH, + candidates=candidates, + rejected=result_rejected, + policy=policy.name, + reason=( + f"nearest row {best.row.row_id} exceeds its tolerance on {axis}: " + f"{dist:.3f} > {policy.tolerance(axis).max_distance}" + ), + ) + + return MatchResult( + status=MatchStatus.NEAR_REGIME, + row=best.row, + distance=best.distance, + candidates=candidates, + rejected=result_rejected, + policy=policy.name, + ) diff --git a/gitm/playbook/schema.py b/gitm/playbook/schema.py new file mode 100644 index 0000000..1acb21b --- /dev/null +++ b/gitm/playbook/schema.py @@ -0,0 +1,470 @@ +"""Playbook row contracts — what a promoted tuning result *is*. + +Deliverable 4. One row says: + + (model+revision, GPU SKU, workload regime, knob set, environment) + -> measured delta + the provenance to re-verify it from scratch + +The schema is the contract between Adit's detection and Seojun's apply runtime, +so it ships as **types**, not as a doc that two implementations read differently. + +Three rules the types enforce rather than describe: + +* **A row cannot exist without provenance.** :class:`Provenance` is required and + ``extra="forbid"``, and its trace fields are the ones deliverable 1 already + emits in :class:`~gitm.traffic.schema.TraceMeta`. A tuning claim without the + raw trace checksum, the drop counts and the repeat data is not defensible, and + the type is where that stops being a convention. +* **A row is retired by a field, never by a deletion.** :class:`Invalidation` + carries a reason. A row deleted from a file leaves no record that the claim was + ever made, which is exactly what a reviewer asks for. +* **Regime is imported, never re-declared.** :class:`~gitm.traffic.regime.Regime` + is deliverable 1's type. A second copy here would drift within a week, and the + distance metric in :mod:`gitm.playbook.match` would be measuring two different + coordinate systems. + +**R1, stated in the types:** the shared config-capture schema does not exist yet. +:class:`EnvCapture` below is the *named subset* this deliverable needs, marked +``pending-adit``. When Adit's types land they are **imported verbatim** and this +class is deleted — there is no translation layer, per the brief, because two +schemas that translate into each other are two schemas that drift. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from datetime import datetime +from enum import Enum +from statistics import median + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from gitm.traffic.regime import Regime +from gitm.traffic.results import BenchRun + +#: Schema identity, in the style of ``gitm.bench.manifest.SCHEMA``. Bump on any +#: field change that is not purely additive. +SCHEMA = "gitm.playbook.row/v1" + +#: Marker for every field that is waiting on the shared config-capture schema +#: (risk R1). Grep-able on purpose: when Adit's types land, this string is the +#: work list. +PENDING_ADIT = "pending-adit" + +#: Knob-name fragments that make a row's delta depend on prefix-cache reuse. +#: Substring match, deliberately: engines rename these flags between versions, +#: and a list of exact names would go stale silently while a substring stays +#: right for ``enable_prefix_caching``, ``prefix_caching``, and the next +#: spelling after that. +PREFIX_CACHE_KNOB_MARKERS = ("prefix_cach", "prefix-cach", "kv_reuse") + + +class Evidence(str, Enum): + """Whether a row's *delta* was measured or is an illustration. + + The worked examples ship in the same file format as real rows, so a field + has to separate them. Without it, an example row copied into a live playbook + is indistinguishable from a promoted one — and the whole point of the schema + is that a row's standing is readable from the row. + """ + + MEASURED = "measured" # produced by a real A/B under the promotion rule + ILLUSTRATIVE = "illustrative" # a worked example; never selectable + + +class EnvCapture(BaseModel): + """The environment fields a playbook row must pin. ``pending-adit`` (R1). + + Deliberately thin. These are the values that, if they differ between the run + that produced a row and the box about to apply it, make the row's number + meaningless. Everything else Adit's capture records is welcome and arrives by + *import*, not by being re-typed here. + + ``extra="allow"`` is the one place in this module that permits unknown keys: + a capture record from a newer engine must round-trip through a playbook file + without being silently truncated. Comparison uses :meth:`compatible_with`, + which reads the named fields only — an unknown extra key never changes a + match decision, it just survives the trip. + """ + + model_config = ConfigDict(extra="allow") + + schema_id: str = f"{PENDING_ADIT}/env-capture" + engine: str # e.g. "vllm" + engine_version: str # exact, e.g. "0.11.0" + driver_version: str | None = None + torch_version: str | None = None + cuda_version: str | None = None + + def compatible_with(self, other: EnvCapture) -> tuple[bool, str]: + """Exact on engine and engine version. Returns ``(ok, why_not)``. + + An engine version bump is the single most common way a knob's effect + changes without anything in the workload changing — scheduler rewrites + ship in point releases. So the default policy is **exact**, and anything + looser has to be written down as a policy rather than assumed by a + comparison that used ``startswith``. + """ + if self.engine != other.engine: + return False, f"engine {self.engine!r} != {other.engine!r}" + if self.engine_version != other.engine_version: + return False, f"engine_version {self.engine_version} != {other.engine_version}" + return True, "" + + +class RowIdentity(BaseModel): + """The lookup key. Everything here is matched, nothing here is a result. + + Split deliberately into fields matched **exactly** and one field matched by + **distance** (:attr:`regime`). The split is the design: model, GPU and + environment are categorical — "nearly an H100" is not a thing — while the + workload is continuous and live traffic never lands on a measured point. + :mod:`gitm.playbook.match` implements exactly that split and nothing else. + """ + + # ``protected_namespaces=()`` because the fields really are called ``model`` + # and ``model_revision`` — that is the vocabulary everyone else uses, and + # renaming them to dodge a pydantic warning would make the schema wrong in + # the one place it is read by hand. + model_config = ConfigDict(extra="forbid", protected_namespaces=()) + + model: str # e.g. "Qwen/Qwen3.6-35B-A3B-FP8" + model_revision: str # exact commit/revision, e.g. "95a723d0" + gpu_sku: str # e.g. "NVIDIA H100 80GB" + env: EnvCapture + regime: Regime # deliverable 1's type, imported + #: ``bool`` first in the union on purpose: pydantic coerces left to right, + #: and ``True`` arriving as ``1.0`` turns a boolean knob into a number in + #: every rendering of the row. + knobs: dict[str, bool | int | float | str] = Field(default_factory=dict) + + @model_validator(mode="after") + def _knobs_are_not_empty(self) -> RowIdentity: + if not self.knobs: + raise ValueError("a playbook row with no knob set says nothing; knobs is required") + return self + + def exact_key(self) -> tuple[str, str, str]: + """The part of the key compared by equality, for grouping and printing.""" + return (self.model, self.model_revision, self.gpu_sku) + + +class MeasuredDelta(BaseModel): + """What the knob did, relative to the baseline arm of the same experiment. + + Throughput **and** latency percentiles, always both. Deliverable 2's + criterion 3 is that a knob which gains throughput while blowing out latency + percentiles fails promotion — a schema that let a row carry throughput alone + would make that criterion unenforceable at the point it matters. + + Signs are stated once, here, because a sign error in a playbook is a knob + applied backwards: **throughput is percent, higher is better; the latency + fields are milliseconds of change, negative is better.** + """ + + model_config = ConfigDict(extra="forbid") + + throughput_pct: float # % change vs baseline; + is faster + ttft_p99_ms: float # ms change vs baseline; - is better + itl_p99_ms: float # ms change vs baseline; - is better + repeats: int = Field(ge=1) # per arm, interleaved A/B/A/B per D2 + #: Bootstrap 95 % CI on the median difference in throughput, per D2-1. The + #: promotion rule owns the predicate; the row carries the numbers it needs. + throughput_ci95_pct: tuple[float, float] | None = None + #: D2 criterion 3's verdict, **stored rather than recomputed**: did this knob + #: blow out a latency percentile? The promotion gate and the live-window + #: revert trigger must read the same value, and two callers each re-deriving + #: "blowout" from the raw percentiles is two predicates that drift. D2 owns + #: the predicate and writes the answer here; ``None`` means D2 has not run, + #: which is every row today because D2 does not exist yet. + latency_blowout: bool | None = None + + @model_validator(mode="after") + def _repeats_are_plural(self) -> MeasuredDelta: + if self.repeats < 2: + raise ValueError( + f"repeats={self.repeats}: a single run has no variance and cannot be " + "promoted under D2. Record it as evidence=illustrative if it is an example." + ) + return self + + +class Provenance(BaseModel): + """Everything needed to re-run this row from scratch and get it again. + + The trace fields are deliverable 1's :class:`~gitm.traffic.schema.TraceMeta` + verbatim in meaning: ``trace_sha256`` pins the raw bytes, ``trace_drops`` + says what was rejected getting to them. A row whose trace no longer hashes to + ``trace_sha256`` is not the same experiment, and the field is what lets + anyone find that out. + """ + + model_config = ConfigDict(extra="forbid") + + trace_source: str # adapter name, e.g. "mooncake" + trace_sha256: str # of the raw file, via gitm.bench.manifest + trace_drops: dict[str, int] = Field(default_factory=dict) + regime_label: str # Regime.label() as it stood when measured + #: Where the per-repeat raw numbers live. D2 requires them; a summarized + #: delta with the repeats thrown away cannot be re-analyzed under a different + #: variance rule, and D2-1's thresholds are explicitly not settled yet. + repeat_raw_data: list[str] = Field(default_factory=list) + promotion_rule: str = f"{PENDING_ADIT}/promotion-rule" # D2 doc + version + config_capture: str = PENDING_ADIT # R1 + verified_at: datetime | None = None # last time the row was re-measured + + #: The conditions the replay actually ran under, from D1's ``ReplayPlan``. + #: ``replay_chunk_hash_size`` is here because it is the deliverable-1 finding + #: with the worst failure mode: at vLLM's default of 16 against Mooncake's + #: 512-token blocks every prompt is 32x short while every count in the result + #: still reads correctly. A row that does not record it cannot be checked. + replay_chunk_hash_size: int | None = None + replay_self_timed: bool | None = None + #: True when D1 *synthesized* prefix blocks because the source had none — + #: BurstGPT. See :attr:`PlaybookRow.delta_is_floor`: a prefix-cache knob + #: measured on a synthesized-prefix trace saw no sharing the source never + #: had, so its delta is a lower bound and may not be quoted as a gain. + prefix_synthesized: bool = False + + @model_validator(mode="after") + def _label_matches_nothing_yet(self) -> Provenance: + if not self.trace_sha256: + raise ValueError("trace_sha256 is required — a row that cannot be traced to bytes") + return self + + @classmethod + def from_bench_run(cls, run: BenchRun, **overrides) -> Provenance: + """Build the trace half of a row's provenance from a seam-3 record. + + Every field here already exists on :class:`~gitm.traffic.results.BenchRun` + because that is what seam 3 was for; copying them by hand at each call + site is how the checksum and the label that ran drift apart. What is + *not* here is the config-capture half — ``config_capture`` comes across + still marked ``pending-adit`` (R1), which is the truthful value. + + ``overrides`` covers the fields no single run knows: ``repeat_raw_data``, + ``promotion_rule``, ``verified_at``. + """ + return cls( + trace_source=run.source.source, + trace_sha256=run.source.sha256, + trace_drops=dict(run.source.drops), + regime_label=run.regime_label, + replay_chunk_hash_size=run.chunk_hash_size, + replay_self_timed=run.self_timed, + prefix_synthesized=run.prefix_synthesized, + config_capture=run.config_capture, + **overrides, + ) + + +class Invalidation(BaseModel): + """Why a row stopped being usable. A field, never a deletion.""" + + model_config = ConfigDict(extra="forbid") + + reason: str # free text, e.g. "vLLM 0.11 -> 0.12 scheduler rewrite" + at: datetime + by: str | None = None + + +class PlaybookRow(BaseModel): + """One promoted tuning result. + + Rows enter **only** through deliverable 2's promotion rule. That is not + enforceable in a type — a type cannot see how a number was produced — so what + the type does instead is refuse to hold a row that *could not* have come + through it: no knobs, no provenance, a single repeat, or a delta missing its + latency percentiles all fail construction. + """ + + model_config = ConfigDict(extra="forbid") + + schema_id: str = SCHEMA + row_id: str + identity: RowIdentity + delta: MeasuredDelta + provenance: Provenance + evidence: Evidence = Evidence.MEASURED + invalidated: Invalidation | None = None + notes: list[str] = Field(default_factory=list) + + @property + def delta_is_floor(self) -> bool: + """True when this row's delta is a **lower bound**, not a measurement. + + D1 synthesizes prefix blocks for a source that has none (BurstGPT), with + every request's ids unique so that lengths hold and **no prefix sharing + is invented**. A prefix-cache knob measured on such a trace therefore saw + the *least* reuse the real traffic could have had. The number is a floor: + useful as "at least this much", never quotable as the gain. + + A property rather than a stored flag because it is a function of two + fields already on the row, and a stored copy is a third place to get it + wrong. + """ + if not self.provenance.prefix_synthesized: + return False + return any( + m in k.lower() for k in self.identity.knobs for m in PREFIX_CACHE_KNOB_MARKERS + ) + + @property + def selectable(self) -> bool: + """Whether a lookup may return this row at all. + + Two ways to be unselectable, and both are states rather than absences: + the row was invalidated, or it is a worked example that was never + measured. An example row in a live file is the failure mode this guards. + """ + return self.invalidated is None and self.evidence is Evidence.MEASURED + + def summary(self) -> str: + k = ", ".join(f"{k}={v}" for k, v in sorted(self.identity.knobs.items())) + mark = "" if self.selectable else f" [{'invalid' if self.invalidated else 'example'}]" + mark += " [FLOOR: prefixes synthesized]" if self.delta_is_floor else "" + return ( + f"{self.row_id}: {k} on {self.identity.gpu_sku} / " + f"{self.identity.regime.label()} -> " + f"tput {self.delta.throughput_pct:+.1f}%, " + f"ttft p99 {self.delta.ttft_p99_ms:+.1f}ms, " + f"itl p99 {self.delta.itl_p99_ms:+.1f}ms{mark}" + ) + + +class Playbook(BaseModel): + """A file of rows. Thin on purpose — the matching lives in ``match.py``.""" + + model_config = ConfigDict(extra="forbid") + + schema_id: str = SCHEMA + rows: list[PlaybookRow] = Field(default_factory=list) + + def selectable(self) -> list[PlaybookRow]: + return [r for r in self.rows if r.selectable] + + +#: The throughput a row's ``throughput_pct`` is a percentage of. ``bench serve`` +#: reports three (request, output-token, total-token); a row that does not say +#: which one it means is a row two readers compare differently. Output tokens per +#: second is the serving number, and it is the one deliverable 2's criterion 3 +#: pairs against the latency percentiles. +THROUGHPUT_METRIC = "output_throughput" + +#: Metrics both arms must carry before a delta can be computed. TTFT and ITL are +#: here because deliverable 2's criterion 3 is unenforceable without them — the +#: schema already refuses a throughput-only :class:`MeasuredDelta`, and this +#: refuses to *build* one rather than failing later with a pydantic error. +REQUIRED_METRICS = (THROUGHPUT_METRIC, "p99_ttft_ms", "p99_itl_ms") + + +def row_from_runs( + row_id: str, + baseline: Sequence[BenchRun], + treatment: Sequence[BenchRun], + *, + model: str, + model_revision: str, + gpu_sku: str, + env: EnvCapture, + knobs: dict[str, bool | int | float | str], + repeat_raw_data: Sequence[str] = (), + verified_at: datetime | None = None, + notes: Sequence[str] = (), +) -> PlaybookRow: + """Build one row from the two arms of a real experiment. + + This is the last mile deliverable 4 was missing: seam 3 turns a + ``bench serve`` result into a :class:`~gitm.traffic.results.BenchRun`, and + this turns a *pair* of arms into a row. A ``BenchRun`` is one arm; a row is a + **difference between two**, which is why nothing before this could populate a + row end to end no matter how complete the join was. + + What it enforces, because these are deliverable 4's business: + + * **Both arms ran the same workload** — same trace bytes, same regime label. + A delta across two different traces measures the traces. + * **Every run is** :attr:`~gitm.traffic.results.BenchRun.promotable` — it + reconciles against its trace and had no failures. Below that bar there is + nothing to take a difference of. + * **Equal repeat counts**, because deliverable 2 interleaves A/B/A/B. Unequal + arms mean the interleave broke, and the shorter arm is the one that was + cut short. + * **Medians, never means**, per deliverable 2's criterion 2. + + What it does **not** enforce, and why: + + * **"Same config minus exactly one knob."** That is deliverable 2's, and it + is enforced by diffing two config-capture records — which do not exist + (R1). Until they do, the caller asserts it and the row says so in a note + that disappears on its own the moment ``config_capture`` is real. + * ``throughput_ci95_pct`` and ``latency_blowout`` stay ``None``. Deliverable + 2 owns the variance rule and the blowout predicate; inventing either here + would be the same mistake :class:`~gitm.playbook.match.AxisTolerance` + refuses to make with a distance threshold. + """ + arms = list(baseline) + list(treatment) + if not baseline or not treatment: + raise ValueError("both arms are required — a row is a difference between two") + if len(baseline) != len(treatment): + raise ValueError( + f"unequal arms: {len(baseline)} baseline vs {len(treatment)} treatment. " + "D2 interleaves A/B/A/B, so unequal counts mean the interleave broke." + ) + + unpromotable = [r.summary() for r in arms if not r.promotable] + if unpromotable: + raise ValueError( + "a row cannot be built from runs that are not promotable " + "(reconciled, no failures): " + "; ".join(unpromotable) + ) + + traces = {r.source.sha256 for r in arms} + labels = {r.regime_label for r in arms} + if len(traces) != 1 or len(labels) != 1: + raise ValueError( + f"the arms did not run the same workload: {len(traces)} distinct trace " + f"checksum(s), regime labels {sorted(labels)} — a delta across two " + "workloads measures the workloads" + ) + + for r in arms: + missing = [m for m in REQUIRED_METRICS if r.metrics.get(m) is None] + if missing: + raise ValueError(f"{r.summary()}: result is missing {missing}") + + def med(runs: Sequence[BenchRun], metric: str) -> float: + return median(float(r.metrics[metric]) for r in runs) + + base_tput = med(baseline, THROUGHPUT_METRIC) + if base_tput == 0: + raise ValueError(f"baseline {THROUGHPUT_METRIC} is 0 — no percentage to take") + + notes = list(notes) + prov = Provenance.from_bench_run( + treatment[0], + repeat_raw_data=list(repeat_raw_data), + verified_at=verified_at, + ) + if prov.config_capture == PENDING_ADIT: + notes.append( + "R1: no config-capture record exists, so 'same config minus exactly one knob' " + "is asserted by the caller here, not diffed. D2 owns that check." + ) + + return PlaybookRow( + row_id=row_id, + identity=RowIdentity( + model=model, model_revision=model_revision, gpu_sku=gpu_sku, env=env, + regime=treatment[0].regime, knobs=knobs, + ), + delta=MeasuredDelta( + throughput_pct=(med(treatment, THROUGHPUT_METRIC) / base_tput - 1.0) * 100.0, + ttft_p99_ms=med(treatment, "p99_ttft_ms") - med(baseline, "p99_ttft_ms"), + itl_p99_ms=med(treatment, "p99_itl_ms") - med(baseline, "p99_itl_ms"), + repeats=len(treatment), + ), + provenance=prov, + evidence=Evidence.MEASURED, + notes=notes, + ) diff --git a/gitm/traffic/__init__.py b/gitm/traffic/__init__.py new file mode 100644 index 0000000..628e544 --- /dev/null +++ b/gitm/traffic/__init__.py @@ -0,0 +1,73 @@ +"""Traffic replay library — production traces in, harness workloads out. + +Deliverable 1 of the validation infrastructure. Real production traces normalized +into one canonical format, fired through the harness by a tool that already +exists (vLLM's ``bench serve``), and tagged with the workload regime every result +row is keyed on. + + from gitm.traffic import read_burstgpt, Regime, SourceKind, write_timed_trace + + trace = read_burstgpt("BurstGPT_1.csv") + print(trace.meta.summary()) # provenance and drops + print(Regime.from_trace(trace).label()) # the result-row key + plan = write_timed_trace(trace, "replay.jsonl") # what bench serve consumes + print(" ".join(plan.bench_serve_argv(model="..."))) + +CPU-only: nothing here needs a GPU, and only the final firing needs a server. +Run ``python -m gitm.traffic --selftest`` for the check that fails if any of it +regresses. +""" + +from gitm.traffic.adapters import ADAPTERS, read_burstgpt, read_mooncake +from gitm.traffic.parameterize import RegimeFit, fit, grid, sample_trace +from gitm.traffic.regime import Regime, SourceKind, index_of_dispersion +from gitm.traffic.replay import ( + VLLM_MIN_VERSION, + ReplayPlan, + read_timed_trace, + write_timed_trace, +) +from gitm.traffic.results import BenchRun, join_result, unjoined_keys +from gitm.traffic.runner import RunResult, VllmUnavailable, check_vllm, run_replay +from gitm.traffic.schema import SCHEMA, CanonicalRequest, DropReason, Trace, TraceMeta +from gitm.traffic.validate import ( + REPLAY_THRESHOLDS, + SAMPLED_THRESHOLDS, + ValidationReport, + compare, + ks_statistic, +) + +__all__ = [ + "ADAPTERS", + "REPLAY_THRESHOLDS", + "SAMPLED_THRESHOLDS", + "SCHEMA", + "BenchRun", + "CanonicalRequest", + "DropReason", + "Regime", + "RegimeFit", + "ReplayPlan", + "RunResult", + "VLLM_MIN_VERSION", + "VllmUnavailable", + "SourceKind", + "Trace", + "TraceMeta", + "ValidationReport", + "compare", + "fit", + "grid", + "index_of_dispersion", + "join_result", + "check_vllm", + "ks_statistic", + "read_burstgpt", + "read_mooncake", + "read_timed_trace", + "run_replay", + "sample_trace", + "unjoined_keys", + "write_timed_trace", +] diff --git a/gitm/traffic/__main__.py b/gitm/traffic/__main__.py new file mode 100644 index 0000000..5187e86 --- /dev/null +++ b/gitm/traffic/__main__.py @@ -0,0 +1,147 @@ +"""CLI for the traffic library: describe a trace, emit a replay, validate, selftest. + + python -m gitm.traffic --selftest + python -m gitm.traffic --describe burstgpt BurstGPT_1.csv + python -m gitm.traffic --replay mooncake trace.jsonl --out replay.jsonl --model Qwen/... + python -m gitm.traffic --sweep burstgpt BurstGPT_1.csv + python -m gitm.traffic --replay mooncake t.jsonl --fire --model Qwen/... --result-dir runs/ + python -m gitm.traffic --gui + +Everything except ``--fire`` is CPU-only and needs no vLLM. ``--replay`` writes +the file, validates the round trip and prints the ``vllm bench serve`` command; +``--fire`` then runs it, which needs vLLM >= 0.23.0 and a server. Without +``--fire`` the command is printed and not run, because this box may have neither. +""" + +from __future__ import annotations + +import argparse +import sys + +from gitm._banner import add_banner_argument, show_banner +from gitm.traffic._selftest import run_all +from gitm.traffic.adapters import ADAPTERS +from gitm.traffic.parameterize import fit, grid +from gitm.traffic.regime import Regime, SourceKind +from gitm.traffic.replay import VLLM_MIN_VERSION, read_timed_trace, write_timed_trace +from gitm.traffic.runner import VllmUnavailable, run_replay +from gitm.traffic.validate import REPLAY_THRESHOLDS, compare + + +def main(argv: list[str] | None = None) -> int: + # The validation render uses block characters; a cp1252 console would raise + # on them mid-report rather than at the start. + if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding="utf-8", errors="replace") + + p = argparse.ArgumentParser(prog="python -m gitm.traffic") + add_banner_argument(p) + p.add_argument("--selftest", action="store_true", help="run every check and exit") + p.add_argument("--describe", nargs=2, metavar=("ADAPTER", "PATH")) + p.add_argument("--replay", nargs=2, metavar=("ADAPTER", "PATH")) + p.add_argument("--sweep", nargs=2, metavar=("ADAPTER", "PATH")) + p.add_argument("--gui", action="store_true", + help="serve the localhost viewer (read-only; 127.0.0.1 only)") + p.add_argument("--gui-port", type=int, default=8765) + p.add_argument("--gui-root", default=None, + help="directory traces may be read from (default: the committed fixtures)") + p.add_argument("--no-open", action="store_true", help="do not open a browser") + p.add_argument("--fire", action="store_true", + help="after --replay, actually run the command (needs vLLM and a server)") + p.add_argument("--result-dir", default=None, + help="where --fire saves bench serve's result JSON") + p.add_argument("--dry-run", action="store_true", + help="with --fire: build and check everything, launch nothing") + p.add_argument("--out", default="replay.jsonl") + p.add_argument("--model", default="MODEL") + p.add_argument("--tokenizer", default=None, + help="tokenizer id; needed when the served model name is not " + "resolvable on HuggingFace (a stub, or --served-model-name)") + p.add_argument("--base-url", default="http://127.0.0.1:8000") + p.add_argument("--max-rows", type=int, default=None) + p.add_argument("--kind", default="production", choices=[k.value for k in SourceKind]) + a = p.parse_args(argv) + show_banner(suppressed=a.no_banner) + + if a.selftest: + return run_all() + + if a.gui: + from pathlib import Path + + from gitm.traffic.gui import serve + + return serve(port=a.gui_port, + root=Path(a.gui_root) if a.gui_root else None, + open_browser=not a.no_open) + + spec = a.describe or a.replay or a.sweep + if spec is None: + p.print_help() + return 2 + adapter, path = spec + if adapter not in ADAPTERS: + p.error(f"unknown adapter {adapter!r}; known: {', '.join(sorted(ADAPTERS))}") + trace = ADAPTERS[adapter](path, max_rows=a.max_rows) + regime = Regime.from_trace(trace, source_kind=SourceKind(a.kind)) + + print(trace.meta.summary()) + print(f"regime: {regime.summary()}") + for note in trace.meta.notes: + print(f" note: {note}") + + if a.replay: + plan = write_timed_trace(trace, a.out) + print(f"\nwrote {plan.path} ({plan.requests} requests, " + f"{plan.chunk_hash_size}-token blocks)") + for note in plan.notes: + print(f" note: {note}") + report = compare(trace, read_timed_trace(a.out), thresholds=REPLAY_THRESHOLDS) + print() + print(report.render()) + # not `argv` — that is main()'s own parameter, and shadowing it here reads + # like a bug even though parse_args has already run. + cmd = plan.bench_serve_argv(model=a.model, base_url=a.base_url, + tokenizer=a.tokenizer) + print("\n" + " ".join(cmd)) + if not a.fire: + print(f"\n(not run — pass --fire to launch it; needs vLLM >= " + f"{VLLM_MIN_VERSION} and a server at {a.base_url})") + return 0 if report.passed else 1 + + # The version guard runs inside run_replay, before anything launches, so + # a too-old vLLM is a sentence here rather than an argparse error there. + try: + res = run_replay(plan, model=a.model, base_url=a.base_url, + result_dir=a.result_dir, regime=regime, + tokenizer=a.tokenizer, dry_run=a.dry_run) + except VllmUnavailable as exc: + print(f"\nnot fired: {exc}") + return 2 + print(f"\n{res.summary()}") + for note in res.notes: + print(f" note: {note}") + # Seam 3: the result joined to the workload that produced it. + if res.joined is not None: + print() + print(res.joined.render()) + if not res.ok: + print(f"\nstderr tail:\n{res.stderr_tail}") + # A result that does not reconcile with its trace is not evidence, so it + # fails the command even when bench serve itself exited 0. + reconciled = res.joined is None or res.joined.reconciled + return 0 if (report.passed and res.ok and reconciled) else 1 + + if a.sweep: + f = fit(trace) + print(f"\nfitted envelope: {f.rate_rps:.4f} rps, D={f.burstiness:.2f}, " + f"span {f.span_s:.0f}s") + print(f"{'regime label':<48} {'req':>6} {'rps':>8} {'D':>6}") + for sampled, reg in grid(f): + print(f"{reg.label():<48} {len(sampled):>6} {reg.rate_rps:>8.3f} " + f"{reg.burstiness:>6.2f}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/gitm/traffic/_selftest.py b/gitm/traffic/_selftest.py new file mode 100644 index 0000000..6725d24 --- /dev/null +++ b/gitm/traffic/_selftest.py @@ -0,0 +1,682 @@ +"""The check that fails if the traffic library breaks. + +One runnable thing, ``python -m gitm.traffic --selftest``, and the same functions +are the pytest cases in ``tests/test_traffic.py`` — no assertions written twice. + +Every expected number below was measured on the committed fixtures and pinned +here. That is the point: a pinned count is a regression guard, an unpinned one is +a comment. The fixtures are **real published bytes** (the first 400 rows of each +source), so these numbers describe real production traffic, not a mock. +""" + +from __future__ import annotations + +import json +import os +import sys +import tempfile +from pathlib import Path + +from gitm.traffic.adapters import ADAPTERS, read_burstgpt, read_mooncake +from gitm.traffic.parameterize import fit, sample_trace +from gitm.traffic.regime import Regime, SourceKind +from gitm.traffic.replay import read_timed_trace, write_timed_trace +from gitm.traffic.schema import CanonicalRequest, DropReason, Trace, TraceMeta +from gitm.traffic.validate import REPLAY_THRESHOLDS, SAMPLED_THRESHOLDS, compare + +#: Fixtures live beside the benchmark spec, not inside the package: they are +#: data, and the wheel ships ``gitm`` only. ``$GITM_TRAFFIC_FIXTURES`` overrides +#: for an installed checkout. +FIXTURES = Path( + os.environ.get( + "GITM_TRAFFIC_FIXTURES", + Path(__file__).resolve().parents[2] / "benchmarks" / "traffic_replay" / "fixtures", + ) +) + +# --- pinned on the committed fixtures --------------------------------------- +BURSTGPT_ROWS = 400 +BURSTGPT_EMITTED = 383 +BURSTGPT_DROPS = {"zero_input_tokens": 17} # real: 4.3% of the slice is 0-in/0-out +BURSTGPT_LABEL = "prod/io1/in256/out128/burst-poisson/copen" + +#: BurstGPT_3 (release v2.0) — the eight-column layout with Session ID and +#: Elapsed time inserted at positions 1 and 2. +BURSTGPT3_ROWS = 400 +BURSTGPT3_EMITTED = 399 +BURSTGPT3_DROPS = {"zero_input_tokens": 1} +BURSTGPT3_SESSION_ROWS = 393 # the rest are API-log rows with no conversation +BURSTGPT3_SESSIONS = 134 +BURSTGPT3_LABEL = "prod/io2/in256/out64/burst-poisson/copen" + +#: The v3 dirty fixture: 6 rows, 3 emitted, 3 defects, 1 junk Elapsed time. +BURSTGPT3_DIRTY_EMITTED = 3 +BURSTGPT3_DIRTY_DROPS = {"malformed_row": 1, "non_monotonic_arrival": 1, "zero_input_tokens": 1} + +MOONCAKE_ROWS = 400 +MOONCAKE_EMITTED = 400 +MOONCAKE_SPAN_S = 141.0 +MOONCAKE_LABEL = "prod/io32/in8k/out256/burst-hi/copen" + +#: GPT-4 rows in the BurstGPT slice, and the split that must reconcile. +BURSTGPT_GPT4_EMITTED = 77 +BURSTGPT_GPT4_FILTERED = 320 + +#: Every defect reason, each firing exactly once across the two dirty fixtures. +DIRTY_ROWS = 9 +DIRTY_EMITTED = 2 + + +def _fixture(name: str) -> Path: + p = FIXTURES / name + if not p.exists(): + raise FileNotFoundError( + f"fixture {p} not found. Fixtures live in benchmarks/traffic_replay/fixtures/; " + "set $GITM_TRAFFIC_FIXTURES if this is an installed checkout." + ) + return p + + +def check_burstgpt_fixture() -> None: + """The BurstGPT adapter on real bytes, counts and regime pinned.""" + t = read_burstgpt(_fixture("burstgpt_slice.csv")) + assert t.meta.rows_read == BURSTGPT_ROWS, t.meta.rows_read + assert t.meta.rows_emitted == BURSTGPT_EMITTED, t.meta.rows_emitted + assert t.meta.drops == BURSTGPT_DROPS, t.meta.drops + assert t.meta.raw_time_unit == "s" + assert not t.meta.has_prefix_identity # BurstGPT has no hash_ids + assert t.requests[0].arrival_s == 0.0 # clock anchored on the first row read + assert Regime.from_trace(t).label() == BURSTGPT_LABEL, Regime.from_trace(t).label() + + +def check_burstgpt3_layout() -> None: + """The eight-column BurstGPT_3 layout, read by column name rather than index. + + The two extra columns are *inserted* at positions 1 and 2, so a positional + reader does not merely miss them — it reads Session ID as the model and + Elapsed time as the request length. This is the check that the reader is + name-based. + """ + t = read_burstgpt(_fixture("burstgpt3_slice.csv")) + assert t.meta.rows_read == BURSTGPT3_ROWS, t.meta.rows_read + assert t.meta.rows_emitted == BURSTGPT3_EMITTED, t.meta.rows_emitted + assert t.meta.drops == BURSTGPT3_DROPS, t.meta.drops + + # Session identity: present, and quantified. The flag alone would say "yes" + # on a trace that is 90% single-shot API traffic, which the full v3 file is. + assert t.meta.has_session_identity + assert t.meta.session_rows == BURSTGPT3_SESSION_ROWS, t.meta.session_rows + assert t.meta.sessions == BURSTGPT3_SESSIONS, t.meta.sessions + assert 0 < t.meta.session_rows <= t.meta.rows_emitted + + # An empty Session ID is by design (API-log rows), never a drop. + blank = [r for r in t.requests if r.session_id is None] + assert blank, "no API-log rows survived; an empty session id must not drop a row" + assert len(blank) == BURSTGPT3_EMITTED - BURSTGPT3_SESSION_ROWS + + # A real multi-turn conversation is visible, which is the point of the column. + from collections import Counter + + turns = Counter(r.session_id for r in t.requests if r.session_id) + assert max(turns.values()) > 1, "no multi-turn session in the fixture" + + # Elapsed time is carried, in seconds, and is not confused with a token count. + latencies = [r.source_e2e_latency_s for r in t.requests if r.source_e2e_latency_s] + assert latencies and all(0 <= v <= 3600 for v in latencies), latencies[:5] + assert t.requests[0].source_e2e_latency_s == 43.0 # first row of the real file + + assert Regime.from_trace(t).label() == BURSTGPT3_LABEL, Regime.from_trace(t).label() + + +def check_burstgpt3_defects() -> None: + """v3-specific handling: junk in an optional column does not lose the row.""" + t = read_burstgpt(_fixture("burstgpt3_dirty.csv")) + assert t.meta.rows_emitted == BURSTGPT3_DIRTY_EMITTED, t.meta.rows_emitted + assert t.meta.drops == BURSTGPT3_DIRTY_DROPS, t.meta.drops + # An unparseable Elapsed time is read as absent and said out loud... + assert any("unparseable" in n for n in t.meta.notes), t.meta.notes + # ...and the request itself survives with its lengths intact. + survived = [r for r in t.requests if r.source_e2e_latency_s is None and r.session_id] + assert survived and survived[0].input_tokens == 120, survived + + +def check_burstgpt_layouts_are_read_by_name() -> None: + """A six-column file and an eight-column file both load; junk does not.""" + v1 = read_burstgpt(_fixture("burstgpt_slice.csv")) + v3 = read_burstgpt(_fixture("burstgpt3_slice.csv")) + assert not v1.meta.has_session_identity and v3.meta.has_session_identity + assert v1.meta.session_rows == 0 and v1.meta.sessions == 0 + # v1 is unchanged by v3 support — the regression this whole check exists for. + assert v1.meta.rows_emitted == BURSTGPT_EMITTED and v1.meta.drops == BURSTGPT_DROPS + # Every layout records the columns it actually saw. + assert any(n.startswith("columns: ") for n in v3.meta.notes) + # A file missing a core column is still rejected, and says which. + try: + read_burstgpt(_fixture("mooncake_slice.jsonl")) + except ValueError as e: + assert "missing column" in str(e) or "not a BurstGPT" in str(e), e + else: + raise AssertionError("a non-BurstGPT file was accepted as one") + + +def check_session_trace_replay_understates_reuse() -> None: + """A session-aware source with no prefix hashes must say so on the plan. + + BurstGPT_3 knows which requests are turns of one conversation but not what + they share. Synthesized unique blocks therefore understate real prefix reuse + — the safe direction, but only if it is stated. + """ + t = read_burstgpt(_fixture("burstgpt3_slice.csv")) + with tempfile.TemporaryDirectory() as td: + out = Path(td) / "bg3.jsonl" + plan = write_timed_trace(t, out) + replayed = read_timed_trace(out) + # The load shape crosses intact... + assert compare(t, replayed, thresholds=REPLAY_THRESHOLDS).passed + # ...but session identity does not, because timed_trace has no field for + # it. Pinned so the limitation is a fact in the suite, not folklore. + assert not any(r.session_id for r in replayed.requests) + assert not any(r.source_e2e_latency_s for r in replayed.requests) + assert plan.prefix_synthesized + assert any("UNDERSTATED" in n for n in plan.notes), plan.notes + assert any(f"{t.meta.sessions} sessions" in n for n in plan.notes), plan.notes + assert any("NOT carried into the replay file" in n for n in plan.notes), plan.notes + + +def check_mooncake_fixture() -> None: + """The Mooncake adapter, including the 512-token block reading.""" + t = read_mooncake(_fixture("mooncake_slice.jsonl")) + assert t.meta.rows_read == MOONCAKE_ROWS, t.meta.rows_read + assert t.meta.rows_emitted == MOONCAKE_EMITTED, t.meta.rows_emitted + assert t.meta.drops == {}, t.meta.drops + assert t.meta.raw_time_unit == "ms" + assert abs(t.meta.span_s - MOONCAKE_SPAN_S) < 1e-9, t.meta.span_s + assert t.meta.has_prefix_identity and t.meta.prefix_block_tokens == 512 + # The blocks tile the prompt exactly — the property that makes 512 the right + # value and 16 (vLLM's default) a silent 32x truncation. + for r in t.requests: + assert len(r.prefix_blocks) * 512 >= r.input_tokens + assert (len(r.prefix_blocks) - 1) * 512 < r.input_tokens + assert Regime.from_trace(t).label() == MOONCAKE_LABEL, Regime.from_trace(t).label() + + +def check_every_drop_reason_fires() -> None: + """Both dirty fixtures exercise all seven defect reasons, one row each.""" + bg = read_burstgpt(_fixture("burstgpt_dirty.csv")) + mc = read_mooncake(_fixture("mooncake_dirty.jsonl")) + defects = {r.value for r in DropReason} - {DropReason.FILTERED_OUT.value} + for t in (bg, mc): + assert t.meta.rows_read == DIRTY_ROWS, t.meta.rows_read + assert t.meta.rows_emitted == DIRTY_EMITTED, t.meta.rows_emitted + assert set(t.meta.drops) == defects, set(t.meta.drops) + assert all(v == 1 for v in t.meta.drops.values()), t.meta.drops + + +def check_filtering_is_not_a_defect() -> None: + """A caller-supplied filter counts separately from bad data, and reconciles.""" + t = read_burstgpt(_fixture("burstgpt_slice.csv"), model="GPT-4") + assert t.meta.rows_emitted == BURSTGPT_GPT4_EMITTED, t.meta.rows_emitted + assert t.meta.drops["filtered_out"] == BURSTGPT_GPT4_FILTERED, t.meta.drops + assert t.meta.defects == t.meta.dropped - BURSTGPT_GPT4_FILTERED + assert t.meta.rows_read == BURSTGPT_ROWS # the whole file was still read + + +def check_provenance_must_reconcile() -> None: + """A Trace whose counts do not add up cannot be constructed at all.""" + meta = TraceMeta(source="x", path="x", sha256="0" * 64, rows_read=5, rows_emitted=2) + try: + Trace(meta=meta, requests=[CanonicalRequest(0.0, 10, 10)] * 2) + except ValueError as e: + assert "unattributed" in str(e), e + else: + raise AssertionError("a trace with 3 unaccounted rows was accepted") + + +def check_replay_roundtrip() -> None: + """The emitted timed_trace file reproduces the source, on both adapters. + + This is the brief's validation deliverable: the comparison is against the + file bench-serve will actually read, so it is evidence about the artifact. + """ + with tempfile.TemporaryDirectory() as td: + for name, trace in ( + ("burstgpt", read_burstgpt(_fixture("burstgpt_slice.csv"))), + ("mooncake", read_mooncake(_fixture("mooncake_slice.jsonl"))), + ): + out = Path(td) / f"{name}.jsonl" + plan = write_timed_trace(trace, out) + report = compare(trace, read_timed_trace(out), thresholds=REPLAY_THRESHOLDS) + assert report.passed, f"{name}:\n{report.render()}" + assert plan.self_timed and plan.sec_multiplier == 1.0 + assert plan.requests == len(trace) + assert plan.source.sha256 == trace.meta.sha256 # provenance survives + + +def check_replay_refuses_to_truncate() -> None: + """The two silent failures the replay path exists to make loud.""" + mc = read_mooncake(_fixture("mooncake_slice.jsonl")) + with tempfile.TemporaryDirectory() as td: + # vLLM's default chunk size against 512-token Mooncake blocks would + # silently emit prompts 32x short. It must refuse instead. + try: + write_timed_trace(mc, Path(td) / "bad.jsonl", block_tokens=16) + except ValueError as e: + assert "truncated" in str(e), e + else: + raise AssertionError("a 32x prompt truncation was accepted") + + # A trace with no output lengths cannot be replayed as-is. + meta = TraceMeta(source="x", path="x", sha256="0" * 64, rows_read=1, rows_emitted=1) + no_out = Trace(meta=meta, requests=[CanonicalRequest(0.0, 128, None)]) + try: + write_timed_trace(no_out, Path(td) / "noout.jsonl") + except ValueError as e: + assert "output length" in str(e), e + else: + raise AssertionError("a trace with no output lengths was replayed as-is") + + # BurstGPT has no prefix identity, so blocks are synthesized and said so. + bg = read_burstgpt(_fixture("burstgpt_slice.csv")) + plan = write_timed_trace(bg, Path(td) / "bg.jsonl") + assert plan.prefix_synthesized + assert any("SYNTHESIZED" in n for n in plan.notes) + # ...and synthesized ids never collide, so no prefix sharing is invented. + seen: set[int] = set() + for req in read_timed_trace(Path(td) / "bg.jsonl").requests: + assert not seen & set(req.prefix_blocks) + seen |= set(req.prefix_blocks) + + +def check_regime_axes_separate_the_traces() -> None: + """The axes have to tell the two real traces apart, or they are decoration.""" + bg = Regime.from_trace(read_burstgpt(_fixture("burstgpt_slice.csv"))) + mc = Regime.from_trace(read_mooncake(_fixture("mooncake_slice.jsonl"))) + assert mc.burstiness > 5.0 > bg.burstiness, (mc.burstiness, bg.burstiness) + assert mc.input_p50 > 10 * bg.input_p50 # 9075 vs 353 tokens + assert bg.label() != mc.label() + # A scoreboard workload can never be read as production traffic. + board = Regime.from_trace( + read_burstgpt(_fixture("burstgpt_slice.csv")), source_kind=SourceKind.SCOREBOARD + ) + assert board.label().startswith("board/") and bg.label().startswith("prod/") + + +def check_parameterized_envelope() -> None: + """A sample matches the envelope it was fitted on; margin is labelled margin.""" + src = read_burstgpt(_fixture("burstgpt_slice.csv")) + f = fit(src) + inside, reg = sample_trace(f, seed=1) + report = compare(src, inside, thresholds=SAMPLED_THRESHOLDS) + assert report.passed, report.render() + assert reg.in_envelope and "xenv" not in reg.label() + + outside, reg_out = sample_trace(f, rate_mult=4.0, burstiness=16.0, seed=2) + assert not reg_out.in_envelope + assert reg_out.label().endswith("/xenv") + assert len(outside) > 3 * len(inside) # 4x the rate really is 4x the load + # The dispersion target is honoured, not merely requested. + assert reg_out.burstiness > 4.0, reg_out.burstiness + # A sample is reproducible from its parameters alone. + again, _ = sample_trace(f, rate_mult=4.0, burstiness=16.0, seed=2) + assert again.meta.sha256 == outside.meta.sha256 + + +def check_bench_serve_argv() -> None: + """The command line carries the flags that decide whether a replay is real.""" + mc = read_mooncake(_fixture("mooncake_slice.jsonl")) + with tempfile.TemporaryDirectory() as td: + plan = write_timed_trace(mc, Path(td) / "mc.jsonl") + argv = plan.bench_serve_argv(model="Qwen/Qwen3.6-35B-A3B-FP8", max_concurrency=64) + assert argv[:3] == ["vllm", "bench", "serve"] + assert "--self-timed" in argv # without this, timestamps are ignored + assert argv[argv.index("--dataset-name") + 1] == "timed_trace" + assert argv[argv.index("--timed-trace-chunk-hash-size") + 1] == "512" + assert argv[argv.index("--timed-trace-sec-multiplier") + 1] == "1" + assert argv[argv.index("--max-concurrency") + 1] == "64" + assert argv[argv.index("--num-prompts") + 1] == str(MOONCAKE_EMITTED) + + +def check_banner_can_never_corrupt_stdout() -> None: + """The banner's three guards. Cosmetic feature, real failure mode. + + A banner on stdout breaks `jq`, a `> results.json`, and every CI step that + parses output — and it breaks them far from here, as a JSON parse error. + """ + import io + + from gitm import _banner + + sink = io.StringIO() + + class _Tty(io.StringIO): + def isatty(self): + return True + + real_stdout, real_env = sys.stdout, os.environ.get(_banner.ENV_VAR) + try: + # not a TTY -> silent, which is the redirected/CI case + sys.stdout = io.StringIO() + os.environ.pop(_banner.ENV_VAR, None) + assert _banner.show_banner(stream=sink) is False + assert sink.getvalue() == "" + + # a TTY -> shown, and on the stream we were given, never on stdout + sys.stdout = _Tty() + assert _banner.show_banner(stream=sink) is True + assert "GitM" in sink.getvalue() or "git machines" in sink.getvalue() + assert sys.stdout.getvalue() == "", "banner reached stdout" + + # both escapes work even on a TTY + sink.truncate(0), sink.seek(0) + assert _banner.show_banner(suppressed=True, stream=sink) is False + os.environ[_banner.ENV_VAR] = "1" + assert _banner.show_banner(stream=sink) is False + assert sink.getvalue() == "" + finally: + sys.stdout = real_stdout + os.environ.pop(_banner.ENV_VAR, None) + if real_env is not None: + os.environ[_banner.ENV_VAR] = real_env + + # and the flag exists on every entry point, so one of them cannot drift + import argparse + + p = argparse.ArgumentParser() + _banner.add_banner_argument(p) + assert p.parse_args(["--no-banner"]).no_banner is True + assert p.parse_args([]).no_banner is False + + +def check_gui_refuses_paths_it_was_not_configured_for() -> None: + """The viewer is browser-reachable, so its trace root is a trust boundary.""" + from gitm.traffic import gui + + root = FIXTURES.resolve() + + # a name from the server's own list resolves + assert gui._resolve(root, "burstgpt_slice.csv") == root / "burstgpt_slice.csv" + + # anything that is not a bare name inside the root does not + for bad in ( + "../../../etc/passwd", + "..\\..\\windows\\win.ini", + "/etc/passwd", + "subdir/trace.csv", + ".hidden", + "", + "does_not_exist.csv", + ): + try: + gui._resolve(root, bad) + except gui._Rejected: + continue + raise AssertionError(f"path traversal not refused: {bad!r}") + + # loopback only, and a module constant rather than a default someone can pass + assert gui.HOST == "127.0.0.1" + + # the listing only offers files it knows an adapter for + names = {t["name"] for t in gui._list_traces(root)} + assert "burstgpt_slice.csv" in names and "mooncake_slice.jsonl" in names + assert all(t["adapter"] in ADAPTERS for t in gui._list_traces(root)) + + # a Host header that is not loopback is refused, so DNS rebinding cannot + # reach the filesystem reader through a browser on another origin + h = gui._Handler.__new__(gui._Handler) + for host, ok in [("127.0.0.1:8765", True), ("localhost", True), + ("evil.example.com", False), ("", False)]: + h.headers = {"Host": host} + assert h._host_is_loopback() is ok, host + + +def check_version_guard_fires_before_launching() -> None: + """Seam 2's whole point: a too-old vLLM must fail as a sentence, not argparse. + + Below the floor there is no ``timed_trace`` dataset and vLLM complains about + an unknown dataset name — which reads like a typo in *our* command. The guard + turns that into a message naming the version and the flag, before anything is + launched. + """ + from gitm.traffic import runner + from gitm.traffic.replay import VLLM_MIN_VERSION + + # the release comparison, including the two ways a version string lies + assert runner._release("0.23.0") == (0, 23, 0) + assert runner._release("0.23.0+cu128") == (0, 23, 0) # a build tag is not older + assert runner._release("0.23.0rc1") == (0, 23, 0) # nor an rc newer + assert runner._release("0.22.1") < runner._release(VLLM_MIN_VERSION) + assert runner._release("0.6") < runner._release(VLLM_MIN_VERSION) # the old floor + assert runner._release("1.0.0") > runner._release(VLLM_MIN_VERSION) + + real = runner.installed_vllm_version + try: + runner.installed_vllm_version = lambda: None + try: + runner.check_vllm() + except runner.VllmUnavailable as e: + assert "not installed" in str(e) and VLLM_MIN_VERSION in str(e) + else: + raise AssertionError("a missing vllm was not refused") + + runner.installed_vllm_version = lambda: "0.22.1" + try: + runner.check_vllm() + except runner.VllmUnavailable as e: + # the message has to name the flag, or it is the same puzzle as the + # argparse error it exists to replace + assert "timed_trace" in str(e) and "0.22.1" in str(e), e + else: + raise AssertionError("a too-old vllm was not refused") + + runner.installed_vllm_version = lambda: "0.23.0" + assert runner.check_vllm() == "0.23.0" + finally: + runner.installed_vllm_version = real + + +def check_runner_builds_the_pinned_argv_and_keeps_provenance() -> None: + """A dry run exercises everything except the subprocess — argv and shape.""" + from gitm.traffic import runner + + t = read_mooncake(_fixture("mooncake_slice.jsonl")) + with tempfile.TemporaryDirectory() as d: + plan = write_timed_trace(t, Path(d) / "replay.jsonl") + res = runner.run_replay(plan, model="Qwen/Qwen3.6-35B-A3B-FP8", + result_dir=d, dry_run=True) + + assert res.ok and res.returncode == 0 + assert res.argv[:3] == ["vllm", "bench", "serve"] + assert res.argv[res.argv.index("--dataset-name") + 1] == "timed_trace" + assert "--self-timed" in res.argv + # the 512-vs-16 finding survives into the command that actually runs + assert res.argv[res.argv.index("--timed-trace-chunk-hash-size") + 1] == "512" + assert "--save-result" in res.argv + + # provenance is carried, and it is the source trace's own, byte-identical + assert res.source == plan.source == t.meta + assert res.source.sha256 == t.meta.sha256 and res.source.raw_time_unit == "ms" + # nothing was joined: seam 3 is not this module + assert res.result is None + assert any("dry run" in n for n in res.notes) + + +#: The REAL bench serve result JSON, from the 0.28.0 run in session 8. Committed +#: rather than mocked: the two fields the joiner drops are only wrong in a way a +#: mock would have gotten right by accident, and ``request_rate`` arrives as the +#: STRING "inf" because json.dumps cannot write a bare Infinity. +BENCHSERVE_RESULT = "benchserve_result.json" +REAL_RUN_REQUESTS = 40 +REAL_RUN_SPAN_S = 12.0 +REAL_RUN_INPUT_TOKENS = 506_280 +REAL_RUN_DURATION_S = 12.00791824299995 +#: The regime of the 40 rows that actually ran — NOT MOONCAKE_LABEL, which is +#: pinned for the whole 400-row fixture. A joined record must carry the regime of +#: the workload that ran, and a 40-row head of a trace is a different workload: +#: input p50 7,323 (in4k) against the full slice's 9,075 (in8k). +REAL_RUN_LABEL = "prod/io32/in4k/out256/burst-hi/copen" + + +def _real_run(): + """The committed result JSON, plus the plan and regime it came from.""" + result = json.loads(_fixture(BENCHSERVE_RESULT).read_text(encoding="utf-8")) + t = read_mooncake(_fixture("mooncake_slice.jsonl"), max_rows=REAL_RUN_REQUESTS) + with tempfile.TemporaryDirectory() as d: + plan = write_timed_trace(t, Path(d) / "r.jsonl") + return result, plan, Regime.from_trace(t), t + + +def check_join_drops_the_two_fields_that_are_wrong() -> None: + """Seam 3's core claim, against the real result JSON. + + Under ``--self-timed`` the trace decides the schedule, so vLLM never consults + ``request_rate`` or ``burstiness`` — but records them anyway, untouched, next + to real metrics and on exactly the two axes the playbook keys on. + """ + from gitm.traffic.results import MISLEADING_UNDER_SELF_TIMED, is_infinite, join_result + + result, plan, reg, _ = _real_run() + # what the file actually says, before anything touches it + assert result["request_rate"] == "inf" # a STRING, not a float + assert result["burstiness"] == 1.0 + assert is_infinite(result["request_rate"]) + + run = join_result(result, plan, reg) + + # dropped, and visibly so + assert set(run.dropped) == set(MISLEADING_UNDER_SELF_TIMED) + assert "request_rate" not in run.metrics and "burstiness" not in run.metrics + assert run.dropped_values["request_rate"] == "inf" + assert run.dropped_values["burstiness"] == 1.0 + + # and the truth is on the record instead, differing by a lot + assert reg.burstiness > 5.0 and run.dropped_values["burstiness"] == 1.0 + assert reg.rate_rps > 1.0 and is_infinite(run.dropped_values["request_rate"]) + + # nothing was lost: the raw JSON survives the join + assert run.raw == result + assert "request_rate" in run.raw + + +def check_join_attaches_the_identity_the_result_lacks() -> None: + """D1's stated purpose: the regime label reaches a measured number.""" + from gitm.traffic.results import join_result + + result, plan, reg, t = _real_run() + for k in ("regime", "regime_label", "trace", "sha256", "source"): + assert k not in result, f"result JSON unexpectedly carries {k}" + + run = join_result(result, plan, reg) + assert run.regime_label == reg.label() == REAL_RUN_LABEL, run.regime_label + assert run.source.sha256 == t.meta.sha256 and len(run.source.sha256) == 64 + assert run.source.raw_time_unit == "ms" + assert run.chunk_hash_size == 512 + assert run.config_capture == "pending-adit" and run.knobs == {} # R1 + assert run.metrics["p99_ttft_ms"] == result["p99_ttft_ms"] + + +def check_join_reconciles_the_real_run() -> None: + """The real run must pass every check; a mismatch means it is not evidence.""" + from gitm.traffic.results import join_result + + result, plan, reg, _ = _real_run() + # the plan carries the totals a result is reconciled against + assert plan.input_tokens_total == REAL_RUN_INPUT_TOKENS, plan.input_tokens_total + assert abs(plan.span_s - REAL_RUN_SPAN_S) < 1e-9 + + run = join_result(result, plan, reg) + assert run.reconciled, run.render() + assert run.promotable + names = [c.name for c in run.checks] + assert "input_tokens_match_trace" in names and "paced_to_trace_span" in names + assert result["total_input_tokens"] == plan.input_tokens_total == REAL_RUN_INPUT_TOKENS + assert abs(result["duration"] - REAL_RUN_DURATION_S) < 1e-9 + + +def check_join_catches_the_32x_truncation_after_the_fact() -> None: + """The emitter refuses to WRITE a truncating file; this catches one that ran. + + At vLLM's default 16-token blocks every prompt is 32x short while completed, + duration, throughput and every percentile still read perfectly. Input tokens + are the only number that moves. + """ + from gitm.traffic.results import join_result + + result, plan, reg, _ = _real_run() + truncated = dict(result, total_input_tokens=result["total_input_tokens"] // 32) + run = join_result(truncated, plan, reg) + assert not run.reconciled and not run.promotable + bad = [c for c in run.failures() if c.name == "input_tokens_match_trace"] + assert bad and "chunk-hash-size" in bad[0].detail, run.render() + + # every OTHER check still passes, which is exactly why this one is needed + assert [c.name for c in run.failures()] == ["input_tokens_match_trace"] + + +def check_join_reads_both_directions_of_pacing_failure() -> None: + """Too fast and too slow are different failures and must not be conflated.""" + from gitm.traffic.results import join_result + + result, plan, reg, _ = _real_run() + + def pacing(dur): + run = join_result(dict(result, duration=dur), plan, reg) + return next(c for c in run.checks if c.name == "paced_to_trace_span") + + fast = pacing(0.4) # a client that ignored the timestamps + assert not fast.ok and "FASTER" in fast.detail + + slow = pacing(40.0) # a server that saturated + assert not slow.ok and "drifted" in slow.detail + + assert pacing(REAL_RUN_DURATION_S).ok # the real run + + # a run that was NOT self-timed has no schedule to hold, so no such check + unpaced = plan.model_copy(update={"self_timed": False}) + run = join_result(result, unpaced, reg) + assert "paced_to_trace_span" not in [c.name for c in run.checks] + assert "NOT self-timed" in run.dropped["request_rate"] + + +def check_join_accounts_for_every_result_key() -> None: + """A new vLLM field must be a decision, not a silent omission.""" + from gitm.traffic.results import unjoined_keys + + result = json.loads(_fixture(BENCHSERVE_RESULT).read_text(encoding="utf-8")) + assert len(result) == 34, len(result) + assert unjoined_keys(result) == [], unjoined_keys(result) + + +CHECKS = ( + check_burstgpt_fixture, + check_burstgpt3_layout, + check_burstgpt3_defects, + check_burstgpt_layouts_are_read_by_name, + check_session_trace_replay_understates_reuse, + check_mooncake_fixture, + check_every_drop_reason_fires, + check_filtering_is_not_a_defect, + check_provenance_must_reconcile, + check_replay_roundtrip, + check_replay_refuses_to_truncate, + check_regime_axes_separate_the_traces, + check_parameterized_envelope, + check_bench_serve_argv, + check_banner_can_never_corrupt_stdout, + check_gui_refuses_paths_it_was_not_configured_for, + check_version_guard_fires_before_launching, + check_runner_builds_the_pinned_argv_and_keeps_provenance, + check_join_drops_the_two_fields_that_are_wrong, + check_join_attaches_the_identity_the_result_lacks, + check_join_reconciles_the_real_run, + check_join_catches_the_32x_truncation_after_the_fact, + check_join_reads_both_directions_of_pacing_failure, + check_join_accounts_for_every_result_key, +) + + +def run_all() -> int: + for fn in CHECKS: + fn() + print(f" ok {fn.__name__}") + print(f"selftest ok -- {len(CHECKS)} checks, 3 real traces, 7 drop reasons") + return 0 diff --git a/gitm/traffic/adapters.py b/gitm/traffic/adapters.py new file mode 100644 index 0000000..f0a9689 --- /dev/null +++ b/gitm/traffic/adapters.py @@ -0,0 +1,407 @@ +"""Source adapters: raw trace file in, :class:`~gitm.traffic.schema.Trace` out. + +Two sources for v1. Both formats were read off the real published files, not off +a paper: + +* **BurstGPT** — real Azure OpenAI traffic, the burstiness reference. + ``Timestamp,Model,Request tokens,Response tokens,Total tokens,Log Type``, + timestamps in **seconds**. No prefix or session identity. + https://github.com/HPMLL/BurstGPT +* **Mooncake** — Kimi production serving traces. JSONL, + ``{"timestamp", "input_length", "output_length", "hash_ids"}``, timestamps in + **milliseconds**, ``hash_ids`` are **512-token** cache blocks (vLLM's own + ``--timed-trace-chunk-hash-size`` help names 512 for the Moonshot traces; on + the published slice ``len(hash_ids) * 512`` covers ``input_length`` exactly, + never over-covering by a whole block). + https://github.com/kvcache-ai/Mooncake + +Both adapters share the same discipline: **nothing is defaulted, everything +rejected is counted**. Real data is the work here — 7.9 % of the BurstGPT rows +carry zero input *and* zero output tokens, and a loader that quietly kept them +would put empty prefills into every regime fit. +""" + +from __future__ import annotations + +import csv +import json +from collections import Counter +from pathlib import Path + +from gitm.bench.manifest import sha256_file +from gitm.traffic.schema import CanonicalRequest, DropReason, Trace, TraceMeta + +#: Tokens per ``hash_ids`` entry in the published Mooncake traces. +MOONCAKE_BLOCK_TOKENS = 512 + +#: The six columns every BurstGPT release has carried. Required. +_BURSTGPT_COLUMNS = ( + "Timestamp", + "Model", + "Request tokens", + "Response tokens", + "Total tokens", + "Log Type", +) + +#: Columns BurstGPT_3 (release v2.0) adds, at positions 1 and 2 — inserted, not +#: appended, which is why the reader goes by column *name* and never by index. +#: ``BurstGPT_without_fails_3.csv`` carries the same eight. +_BURSTGPT_SESSION = "Session ID" +_BURSTGPT_ELAPSED = "Elapsed time" +_BURSTGPT_OPTIONAL = (_BURSTGPT_SESSION, _BURSTGPT_ELAPSED) + + +class _Collector: + """Accumulates survivors and drop counts so an adapter never loses a row. + + ``last_ts`` tracks the newest *parsed* raw timestamp, which is what the + monotonicity check compares against: a row that was dropped for some other + defect still tells us where the file's clock had reached. + """ + + def __init__(self) -> None: + self.requests: list[CanonicalRequest] = [] + self.drops: Counter[str] = Counter() + self.rows_read = 0 + self.last_ts: float | None = None + + def drop(self, reason: DropReason) -> None: + self.drops[reason.value] += 1 + + def check_lengths(self, inp: int, out: int | None) -> DropReason | None: + """Shared length validation — every adapter routes through here. + + One place, so a new source cannot invent a different definition of + "unusable row" and quietly widen the envelope. + """ + if inp < 0 or (out is not None and out < 0): + return DropReason.NEGATIVE_VALUE + if inp == 0: + return DropReason.ZERO_INPUT_TOKENS + if out == 0: + return DropReason.ZERO_OUTPUT_TOKENS + return None + + def check_monotonic(self, ts: float) -> DropReason | None: + if self.last_ts is not None and ts < self.last_ts: + return DropReason.NON_MONOTONIC_ARRIVAL + return None + + +def _finish( + coll: _Collector, + *, + source: str, + path: Path, + source_url: str | None, + raw_time_unit: str, + prefix_block_tokens: int | None, + has_prefix_identity: bool, + has_session_identity: bool, + notes: list[str], + session_rows: int = 0, + sessions: int = 0, +) -> Trace: + arrivals = [r.arrival_s for r in coll.requests] + span = (arrivals[-1] - arrivals[0]) if len(arrivals) > 1 else 0.0 + digest, nbytes = sha256_file(path) + if any(r.output_tokens is None for r in coll.requests): + notes = [*notes, "some requests carry no output length; replay-as-is is not valid"] + meta = TraceMeta( + source=source, + path=str(path), + sha256=digest, + source_url=source_url, + rows_read=coll.rows_read, + rows_emitted=len(coll.requests), + drops=dict(coll.drops), + span_s=span, + raw_time_unit=raw_time_unit, + prefix_block_tokens=prefix_block_tokens, + has_prefix_identity=has_prefix_identity, + has_session_identity=has_session_identity, + session_rows=session_rows, + sessions=sessions, + notes=[*notes, f"raw file {nbytes} bytes"], + ) + return Trace(meta=meta, requests=coll.requests) + + +def read_burstgpt( + path: str | Path, + *, + model: str | None = None, + log_type: str | None = None, + max_rows: int | None = None, + source_url: str | None = None, +) -> Trace: + """Read a BurstGPT CSV into canonical form. Handles every published layout. + + Two layouts exist. ``BurstGPT_1`` / ``_2`` carry six columns; + **``BurstGPT_3`` (release v2.0) carries eight**, inserting ``Session ID`` and + ``Elapsed time`` at positions 1 and 2 — *inserted*, not appended. So the + reader goes by column **name**: the six core columns are required, the two + extras are used when present, and an unrecognized extra column is recorded in + ``TraceMeta.notes`` rather than rejected. A future ``BurstGPT_4`` that adds a + column will load rather than raise. + + ``Session ID`` is populated **only for ``Conversation log`` rows** — in the + published v3 file every ``API log`` row has it empty, and those are 90 % of + the trace. An empty session id is therefore *by design and not a defect*: + the row is emitted with ``session_id=None``. Dropping them would discard most + of a real v3 trace. ``TraceMeta.session_rows`` / ``.sessions`` report how much + conversation identity actually survived, which is what a multi-turn + experiment must check — the boolean flag alone would say "yes" on a trace + that is 90 % single-shot. + + ``Elapsed time`` becomes ``CanonicalRequest.source_e2e_latency_s``. Read that + field's docstring before using it: it is end-to-end latency on the *source* + system, not TTFT, and not ours. An unparseable value is treated as absent + (the row survives; the count lands in ``TraceMeta.notes``), because an + optional annotation being junk is no reason to throw away a valid request. + + ``model`` (``"ChatGPT"`` / ``"GPT-4"``) and ``log_type`` (``"Conversation + log"`` / ``"API log"``) select a subset; excluded rows count as + ``FILTERED_OUT``, never as defects. + + The arrival clock is anchored on the **first row read**, before any filtering, + so narrowing the selection shifts which requests appear but never shifts when + they appear. + """ + path = Path(path) + coll = _Collector() + t0: float | None = None + sessions: set[str] = set() + session_rows = 0 + bad_elapsed = 0 + + with path.open(newline="", encoding="utf-8") as fh: + reader = csv.reader(fh) + header = next(reader, None) + if header is None: + raise ValueError(f"{path}: empty file, not a BurstGPT CSV") + columns = [h.strip() for h in header] + missing = [c for c in _BURSTGPT_COLUMNS if c not in columns] + if missing: + raise ValueError( + f"{path}: not a BurstGPT CSV — missing column(s) {missing}; " + f"header was {columns!r}" + ) + idx = {name: i for i, name in enumerate(columns)} + has_session = _BURSTGPT_SESSION in idx + has_elapsed = _BURSTGPT_ELAPSED in idx + unknown = [c for c in columns if c not in _BURSTGPT_COLUMNS + _BURSTGPT_OPTIONAL] + + for row in reader: + if max_rows is not None and coll.rows_read >= max_rows: + break + coll.rows_read += 1 + + if len(row) != len(columns): + coll.drop(DropReason.MALFORMED_ROW) + continue + cells = [c.strip() for c in row] + ts_raw = cells[idx["Timestamp"]] + row_model = cells[idx["Model"]] + inp_raw = cells[idx["Request tokens"]] + out_raw = cells[idx["Response tokens"]] + row_log = cells[idx["Log Type"]] + + # The timestamp is parsed first and alone. A row whose *lengths* are + # junk still says where the file's clock had reached, and folding both + # parses into one try lets a bad length hide a backwards jump from the + # monotonicity check entirely. + if not ts_raw: + coll.drop(DropReason.MISSING_FIELD) + continue + try: + ts = float(ts_raw) + except ValueError: + coll.drop(DropReason.NON_NUMERIC) + continue + if t0 is None: + t0 = ts + reason = coll.check_monotonic(ts) + coll.last_ts = ts if coll.last_ts is None else max(coll.last_ts, ts) + if reason is not None: + coll.drop(reason) + continue + + if not inp_raw or not out_raw: + coll.drop(DropReason.MISSING_FIELD) + continue + try: + inp = int(inp_raw) + out = int(out_raw) + except ValueError: + coll.drop(DropReason.NON_NUMERIC) + continue + if (model is not None and row_model != model) or ( + log_type is not None and row_log != log_type + ): + coll.drop(DropReason.FILTERED_OUT) + continue + reason = coll.check_lengths(inp, out) + if reason is not None: + coll.drop(reason) + continue + + # Optional columns. Absent or blank is normal, never a drop: v3 + # leaves Session ID empty on every API-log row by design. + session_id = cells[idx[_BURSTGPT_SESSION]] if has_session else "" + if session_id: + sessions.add(session_id) + session_rows += 1 + elapsed: float | None = None + if has_elapsed and cells[idx[_BURSTGPT_ELAPSED]]: + try: + elapsed = float(cells[idx[_BURSTGPT_ELAPSED]]) + except ValueError: + bad_elapsed += 1 + + coll.requests.append( + CanonicalRequest( + arrival_s=ts - t0, + input_tokens=inp, + output_tokens=out, + session_id=session_id or None, + source_e2e_latency_s=elapsed, + ) + ) + + notes = [f"columns: {', '.join(columns)}"] + if session_rows: + notes.append( + f"session identity on {session_rows}/{len(coll.requests)} emitted rows " + f"({len(sessions)} sessions) — the rest are single-shot API traffic" + ) + else: + notes.append("no session identity in this layout (BurstGPT_1/_2)") + notes.append("BurstGPT carries no prefix identity in any layout") + if bad_elapsed: + notes.append(f"{bad_elapsed} unparseable '{_BURSTGPT_ELAPSED}' values read as absent") + if unknown: + notes.append(f"unrecognized columns ignored: {', '.join(unknown)}") + if model or log_type: + notes.append(f"filtered: model={model!r} log_type={log_type!r}") + return _finish( + coll, + source="burstgpt", + path=path, + source_url=source_url, + raw_time_unit="s", + prefix_block_tokens=None, + has_prefix_identity=False, + has_session_identity=bool(session_rows), + session_rows=session_rows, + sessions=len(sessions), + notes=notes, + ) + + +def read_mooncake( + path: str | Path, + *, + block_tokens: int = MOONCAKE_BLOCK_TOKENS, + max_rows: int | None = None, + source_url: str | None = None, + time_scale: float = 0.001, + source: str = "mooncake", +) -> Trace: + """Read a Mooncake JSONL trace into canonical form. + + ``block_tokens`` is how many tokens one ``hash_ids`` entry stands for. It is + **512 for the published Moonshot traces** and getting it wrong is silent: the + replay path expands each block to that many tokens, so a wrong value produces + prompts that are a clean multiple too short while every count still looks + right. :func:`gitm.traffic.replay.write_timed_trace` checks the coverage and + refuses rather than truncating. + + ``time_scale`` converts the source's timestamps to seconds (0.001 for + Mooncake's milliseconds). :func:`gitm.traffic.replay.read_timed_trace` reuses + this reader at ``time_scale=1.0`` — the emitted replay file is the same shape, + so re-parsing it needs one parameter, not a second parser that can drift. + """ + path = Path(path) + coll = _Collector() + t0: float | None = None + + with path.open(encoding="utf-8") as fh: + for line in fh: + if not line.strip(): + continue + if max_rows is not None and coll.rows_read >= max_rows: + break + coll.rows_read += 1 + + try: + rec = json.loads(line) + except json.JSONDecodeError: + coll.drop(DropReason.MALFORMED_ROW) + continue + if not isinstance(rec, dict): + coll.drop(DropReason.MALFORMED_ROW) + continue + # Timestamp first and alone — see the note in read_burstgpt. + if "timestamp" not in rec: + coll.drop(DropReason.MISSING_FIELD) + continue + try: + ts_raw = float(rec["timestamp"]) + except (TypeError, ValueError): + coll.drop(DropReason.NON_NUMERIC) + continue + if t0 is None: + t0 = ts_raw + reason = coll.check_monotonic(ts_raw) + coll.last_ts = ts_raw if coll.last_ts is None else max(coll.last_ts, ts_raw) + if reason is not None: + coll.drop(reason) + continue + + if not {"input_length", "output_length"} <= rec.keys(): + coll.drop(DropReason.MISSING_FIELD) + continue + try: + inp = int(rec["input_length"]) + out = int(rec["output_length"]) + except (TypeError, ValueError): + coll.drop(DropReason.NON_NUMERIC) + continue + reason = coll.check_lengths(inp, out) + if reason is not None: + coll.drop(reason) + continue + + blocks = rec.get("hash_ids") or [] + if not isinstance(blocks, list) or any(not isinstance(b, int) for b in blocks): + coll.drop(DropReason.MALFORMED_ROW) + continue + + coll.requests.append( + CanonicalRequest( + arrival_s=(ts_raw - t0) * time_scale, + input_tokens=inp, + output_tokens=out, + prefix_blocks=tuple(blocks), + ) + ) + + has_prefix = any(r.prefix_blocks for r in coll.requests) + return _finish( + coll, + source=source, + path=path, + source_url=source_url, + raw_time_unit="ms" if time_scale == 0.001 else "s", + prefix_block_tokens=block_tokens if has_prefix else None, + has_prefix_identity=has_prefix, + has_session_identity=False, + notes=[f"hash_ids read as {block_tokens}-token cache blocks"], + ) + + +#: Adapter registry — name to reader. Keeps the CLI and the selftest from +#: growing an if-chain per source. +ADAPTERS = {"burstgpt": read_burstgpt, "mooncake": read_mooncake} diff --git a/gitm/traffic/gui.py b/gitm/traffic/gui.py new file mode 100644 index 0000000..0379876 --- /dev/null +++ b/gitm/traffic/gui.py @@ -0,0 +1,391 @@ +"""A localhost viewer for the traffic library. stdlib only, one page, read-only. + +``python -m gitm.traffic --gui`` and open the printed URL. Describe a trace, +replay it and see the validation table with the arrival profiles drawn properly, +or sweep the parameterized grid. + +**It is a viewer, not a control panel.** Everything here is CPU-only. It does not +fire traffic — that is :mod:`gitm.traffic.runner` (``--fire`` on the CLI), which +needs vLLM and a live server. The viewer deliberately does not reach for it: +firing from a browser page means a long-running subprocess behind a synchronous +handler and a result nobody is waiting for. Saying so up front is the point — a +page with a "replay" button that only writes a file is a page someone will assume +hit a server. + +No Flask, no React, no build step: the library's functions already return +pydantic models that serialize straight to JSON, so the server is a thin shell +around ``model_dump`` and the page is one string. + +Three things this gets right because it is browser-reachable, even on loopback: + +* **Binds 127.0.0.1 only.** Never ``0.0.0.0`` — that would expose a filesystem + reader to the network the moment someone runs it on a shared box. +* **Never accepts a path from the form.** The client sends a *name*, chosen from + a list the server produced; the server joins it under one configured root and + re-checks containment after resolving. Path traversal is a real vector here, + and "it's only localhost" has never been a defence. +* **Checks the Host header.** A page on any other origin can still POST to + ``127.0.0.1`` via DNS rebinding. Requests whose Host is not loopback are + refused. +""" + +from __future__ import annotations + +import json +import os +import tempfile +import webbrowser +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +from gitm.traffic.adapters import ADAPTERS +from gitm.traffic.parameterize import fit, grid +from gitm.traffic.regime import DEFAULT_BIN_S, Regime, SourceKind +from gitm.traffic.replay import read_timed_trace, write_timed_trace +from gitm.traffic.validate import REPLAY_THRESHOLDS, compare + +#: Loopback only. Not a default to be overridden — a constant, so binding +#: anywhere else is an edit somebody has to justify in a diff. +HOST = "127.0.0.1" + +#: Upper bound on rows read per request. The page is a viewer; a 1.4 M-row file +#: behind a synchronous handler is a hung browser tab, not a feature. +MAX_ROWS_CAP = 20_000 + +#: Where traces may be read from. One directory, resolved once at startup. +DEFAULT_ROOT = Path(__file__).resolve().parents[2] / "benchmarks" / "traffic_replay" / "fixtures" + +_SUFFIX_ADAPTER = {".csv": "burstgpt", ".jsonl": "mooncake"} + + +class _Rejected(Exception): + """A request that will not be served, with the reason shown to the user.""" + + +def _resolve(root: Path, name: str) -> Path: + """Resolve a client-supplied trace *name* under ``root``, or refuse. + + Two independent guards, because either alone has a bypass: rejecting + separators stops the obvious ``../..``, and the containment check after + ``resolve()`` catches what symlinks and Windows path quirks let through. + """ + if not name or "/" in name or "\\" in name or name.startswith("."): + raise _Rejected(f"not a trace name: {name!r}") + path = (root / name).resolve() + if path.parent != root or not path.is_file(): + raise _Rejected(f"{name!r} is not a file under the configured trace root") + return path + + +def _list_traces(root: Path) -> list[dict]: + return sorted( + ( + { + "name": p.name, + "bytes": p.stat().st_size, + "adapter": _SUFFIX_ADAPTER.get(p.suffix, ""), + } + for p in root.iterdir() + if p.is_file() and p.suffix in _SUFFIX_ADAPTER + ), + key=lambda d: d["name"], + ) + + +def _load(root: Path, body: dict): + """Adapter + trace, from a validated request body.""" + adapter = body.get("adapter", "") + if adapter not in ADAPTERS: + raise _Rejected(f"unknown adapter {adapter!r}") + path = _resolve(root, body.get("trace", "")) + max_rows = body.get("max_rows") or None + if max_rows is not None: + max_rows = max(1, min(int(max_rows), MAX_ROWS_CAP)) + return ADAPTERS[adapter](path, max_rows=max_rows) + + +def _regime_of(trace, body: dict) -> Regime: + concurrency = body.get("concurrency") or None + return Regime.from_trace( + trace, + source_kind=SourceKind(body.get("source_kind", "production")), + bin_s=float(body.get("bin_s") or DEFAULT_BIN_S), + concurrency=int(concurrency) if concurrency else None, + ) + + +def _describe(root: Path, body: dict) -> dict: + trace = _load(root, body) + regime = _regime_of(trace, body) + return { + "meta": trace.meta.model_dump(mode="json"), + "summary": trace.meta.summary(), + "regime": regime.model_dump(mode="json"), + "label": regime.label(), + "regime_summary": regime.summary(), + } + + +def _replay(root: Path, body: dict) -> dict: + trace = _load(root, body) + # The output path is ours, never the client's: writing where a form says to + # is the same vulnerability as reading where it says to. + with tempfile.TemporaryDirectory(prefix="gitm-gui-") as tmp: + out = Path(tmp) / "replay.jsonl" + plan = write_timed_trace(trace, out) + report = compare(trace, read_timed_trace(out), thresholds=REPLAY_THRESHOLDS) + return { + "plan": plan.model_dump(mode="json"), + "argv": plan.bench_serve_argv(model=body.get("model") or "MODEL"), + "report": report.model_dump(mode="json"), + "passed": report.passed, + "explain": report.explain(), + } + + +def _sweep(root: Path, body: dict) -> dict: + trace = _load(root, body) + fitted = fit(trace, bin_s=float(body.get("bin_s") or DEFAULT_BIN_S)) + rows = [ + { + "label": reg.label(), + "requests": len(sampled), + "rate_rps": reg.rate_rps, + "burstiness": reg.burstiness, + "input_p50": reg.input_p50, + "output_p50": reg.output_p50, + "in_envelope": reg.in_envelope, + } + for sampled, reg in grid(fitted) + ] + return {"fit": fitted.model_dump(mode="json"), "rows": rows} + + +ROUTES = {"/api/describe": _describe, "/api/replay": _replay, "/api/sweep": _sweep} + + +class _Handler(BaseHTTPRequestHandler): + server_version = "gitm-traffic-viewer" + root: Path = DEFAULT_ROOT + + def log_message(self, fmt, *args): # noqa: A003 - stdlib hook name + pass # the terminal belongs to whatever else is running + + def _host_is_loopback(self) -> bool: + host = (self.headers.get("Host") or "").rsplit(":", 1)[0].strip("[]") + return host in {"127.0.0.1", "localhost", "::1"} + + def _send(self, code: int, body: bytes, ctype: str) -> None: + self.send_response(code) + self.send_header("Content-Type", ctype) + self.send_header("Content-Length", str(len(body))) + # The page is self-contained; nothing should be able to load anything. + self.send_header("Content-Security-Policy", "default-src 'none'; style-src 'unsafe-inline'; script-src 'unsafe-inline'") + self.send_header("X-Content-Type-Options", "nosniff") + self.end_headers() + self.wfile.write(body) + + def _json(self, code: int, payload: dict) -> None: + self._send(code, json.dumps(payload).encode("utf-8"), "application/json") + + def do_GET(self) -> None: # noqa: N802 - stdlib hook name + if not self._host_is_loopback(): + return self._json(403, {"error": "non-loopback Host header refused"}) + if self.path == "/": + return self._send(200, PAGE.encode("utf-8"), "text/html; charset=utf-8") + if self.path == "/api/traces": + return self._json(200, {"root": str(self.root), "traces": _list_traces(self.root)}) + self._json(404, {"error": "no such path"}) + + def do_POST(self) -> None: # noqa: N802 - stdlib hook name + if not self._host_is_loopback(): + return self._json(403, {"error": "non-loopback Host header refused"}) + handler = ROUTES.get(self.path) + if handler is None: + return self._json(404, {"error": "no such path"}) + try: + n = min(int(self.headers.get("Content-Length") or 0), 64 * 1024) + body = json.loads(self.rfile.read(n) or b"{}") + return self._json(200, handler(self.root, body)) + except _Rejected as exc: + return self._json(400, {"error": str(exc)}) + except Exception as exc: # a bad adapter/threshold error belongs on the page + return self._json(400, {"error": f"{type(exc).__name__}: {exc}"}) + + +def serve(port: int = 8765, root: Path | None = None, *, open_browser: bool = True) -> int: + """Run the viewer until interrupted.""" + _Handler.root = (root or Path(os.environ.get("GITM_TRAFFIC_FIXTURES", DEFAULT_ROOT))).resolve() + if not _Handler.root.is_dir(): + raise SystemExit(f"trace root does not exist: {_Handler.root}") + httpd = ThreadingHTTPServer((HOST, port), _Handler) + url = f"http://{HOST}:{httpd.server_address[1]}/" + print(f"traffic viewer on {url}\n trace root: {_Handler.root}\n" + f" read-only: it does not fire traffic (that is --fire, which needs vLLM)\n" + f" ctrl-c to stop") + if open_browser: + webbrowser.open(url) + try: + httpd.serve_forever() + except KeyboardInterrupt: + print("\nstopped") + finally: + httpd.server_close() + return 0 + + +PAGE = r""" +gitm traffic viewer + +
+

gitm — traffic viewer

+
read-only. describe, replay-and-validate, sweep. it does not fire traffic — that is --fire on the CLI, which needs vLLM and a live server.
+
+
+
+ + + + + + + + + +
+
+
+ +""" diff --git a/gitm/traffic/parameterize.py b/gitm/traffic/parameterize.py new file mode 100644 index 0000000..4d8372e --- /dev/null +++ b/gitm/traffic/parameterize.py @@ -0,0 +1,216 @@ +"""Parameterized mode — fit a trace's envelope, then sample the grid around it. + +Replay answers "what happens under this exact traffic". Parameterized mode +answers "what happens across the region this traffic lives in, **and beyond it**" +— which is the question a playbook row has to survive, because the customer's +next hour is never the trace's next hour. + +The traces define the realistic envelope; the sampler covers the envelope plus +margin, and every point outside it is labelled ``in_envelope=False`` and carries +``/xenv`` in its regime label. An extrapolation that cannot be told apart from a +measurement is worse than no extrapolation. + +Distributions are fitted **empirically** — a 101-point quantile grid, sampled by +inverse-CDF interpolation. No parametric family is assumed: production length +distributions are multi-modal (short chat turns and long document prompts in one +trace) and a lognormal fit would smear the modes together and quietly change the +prefill/decode ratio the whole exercise turns on. + +Arrival burstiness is generated by drawing **per-bin counts from a negative +binomial**, whose index of dispersion is ``1 + m/r`` and so can be set directly +to the target. A Poisson process is the ``D = 1`` special case and is used as +such. ``D < 1`` (more regular than Poisson — a paced or rate-limited client) is +clamped to Poisson and noted; ponytail: a binomial thinning would cover it, add +it when a real trace shows underdispersion. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Iterator + +import numpy as np +from pydantic import BaseModel, ConfigDict, Field + +from gitm.traffic.regime import DEFAULT_BIN_S, Regime, SourceKind, index_of_dispersion +from gitm.traffic.schema import CanonicalRequest, Trace, TraceMeta + +#: Quantile grid used for both length distributions. 101 points keeps the tails +#: (p99, p100) as real samples rather than as an extrapolated fit. +_QUANTILES = np.linspace(0.0, 100.0, 101) + + +class RegimeFit(BaseModel): + """One trace's envelope, as the axes the sampler moves along.""" + + model_config = ConfigDict(extra="forbid") + + trace: str # source identity the fit came from + source_sha256: str # the bytes it was fitted on + requests: int + span_s: float + rate_rps: float + burstiness: float + bin_s: float = DEFAULT_BIN_S + input_quantiles: list[float] = Field(default_factory=list) + output_quantiles: list[float] = Field(default_factory=list) + + def sample_lengths(self, rng: np.random.Generator, n: int, *, output: bool) -> np.ndarray: + """Inverse-CDF draw from the fitted empirical distribution.""" + q = np.asarray(self.output_quantiles if output else self.input_quantiles) + u = rng.uniform(0.0, 100.0, size=n) + return np.interp(u, _QUANTILES, q) + + +def fit(trace: Trace, *, bin_s: float = DEFAULT_BIN_S) -> RegimeFit: + """Fit the regime axes of a real trace.""" + if len(trace) < 2: + raise ValueError("need at least two requests to fit an envelope") + outs = trace.output_tokens + if not outs: + raise ValueError("cannot fit output lengths: the trace carries none") + return RegimeFit( + trace=trace.meta.source, + source_sha256=trace.meta.sha256, + requests=len(trace), + span_s=trace.meta.span_s, + rate_rps=trace.rate_rps(), + burstiness=index_of_dispersion( + trace.arrivals, bin_s=bin_s, span_s=trace.meta.span_s + ), + bin_s=bin_s, + input_quantiles=[float(x) for x in np.percentile(trace.input_tokens, _QUANTILES)], + output_quantiles=[float(x) for x in np.percentile(outs, _QUANTILES)], + ) + + +def _arrivals( + rng: np.random.Generator, *, rate_rps: float, burstiness: float, duration_s: float, bin_s: float +) -> np.ndarray: + """Arrival times with a controlled index of dispersion.""" + nbins = max(int(np.ceil(duration_s / bin_s)), 1) + m = rate_rps * bin_s + if m <= 0: + return np.zeros(0) + if burstiness <= 1.0: + counts = rng.poisson(m, size=nbins) + else: + var = burstiness * m + p = m / var # NB with mean m, variance var + r = m * p / (1.0 - p) + counts = rng.negative_binomial(r, p, size=nbins) + starts = np.repeat(np.arange(nbins) * bin_s, counts) + return np.sort(starts + rng.uniform(0.0, bin_s, size=starts.size)) + + +def sample_trace( + fit_: RegimeFit, + *, + rate_mult: float = 1.0, + burstiness: float | None = None, + input_scale: float = 1.0, + output_scale: float = 1.0, + duration_s: float | None = None, + seed: int = 0, +) -> tuple[Trace, Regime]: + """Draw one synthetic trace from the fitted envelope, plus its regime tag. + + Returns the trace and its :class:`~gitm.traffic.regime.Regime`. The regime's + ``in_envelope`` is False whenever any axis is pushed past what the source + trace actually showed — the margin the brief asks for, labelled as margin. + """ + burstiness = fit_.burstiness if burstiness is None else burstiness + duration_s = fit_.span_s if duration_s is None else duration_s + rate = fit_.rate_rps * rate_mult + rng = np.random.default_rng(seed) + + arrivals = _arrivals( + rng, rate_rps=rate, burstiness=burstiness, duration_s=duration_s, bin_s=fit_.bin_s + ) + n = arrivals.size + if n == 0: + raise ValueError("sampled zero requests — raise rate_mult or duration_s") + inp = np.maximum(np.rint(fit_.sample_lengths(rng, n, output=False) * input_scale), 1) + out = np.maximum(np.rint(fit_.sample_lengths(rng, n, output=True) * output_scale), 1) + + params = { + "fit": fit_.trace, + "source_sha256": fit_.source_sha256, + "rate_mult": rate_mult, + "burstiness": burstiness, + "input_scale": input_scale, + "output_scale": output_scale, + "duration_s": duration_s, + "bin_s": fit_.bin_s, + "seed": seed, + } + # A synthetic trace has no bytes on disk, so its identity is the digest of the + # parameters that reproduce it exactly. Same digest, same trace. + digest = hashlib.sha256(json.dumps(params, sort_keys=True).encode()).hexdigest() + + requests = [ + CanonicalRequest(arrival_s=float(a), input_tokens=int(i), output_tokens=int(o)) + for a, i, o in zip(arrivals, inp, out, strict=True) + ] + tol = 1e-9 + in_envelope = ( + rate_mult <= 1.0 + tol + and burstiness <= fit_.burstiness + tol + and input_scale <= 1.0 + tol + and output_scale <= 1.0 + tol + ) + notes = [f"sampled from {fit_.trace} envelope", json.dumps(params, sort_keys=True)] + if burstiness < 1.0: + notes.append("burstiness < 1 requested; clamped to a Poisson process") + meta = TraceMeta( + source="parameterized", + path=f"synthetic:{digest[:16]}", + sha256=digest, + rows_read=n, + rows_emitted=n, + span_s=float(arrivals[-1] - arrivals[0]) if n > 1 else 0.0, + raw_time_unit="s", + notes=notes, + ) + trace = Trace(meta=meta, requests=requests) + regime = Regime.from_trace( + trace, + source_kind=SourceKind.SYNTHETIC, + bin_s=fit_.bin_s, + in_envelope=in_envelope, + trace_id=f"{fit_.trace}~{digest[:8]}", + ) + return trace, regime + + +#: Default grid. Deliberately asymmetric: the interesting margin is *above* the +#: observed trace (more load, burstier, longer outputs), because that is where a +#: playbook row gets applied and nobody measured. +DEFAULT_RATE_MULTS = (0.5, 1.0, 2.0, 4.0) +DEFAULT_BURST_TARGETS = (1.0, 4.0, 16.0) +DEFAULT_OUTPUT_SCALES = (1.0, 2.0) + + +def grid( + fit_: RegimeFit, + *, + rate_mults: tuple[float, ...] = DEFAULT_RATE_MULTS, + burst_targets: tuple[float, ...] = DEFAULT_BURST_TARGETS, + output_scales: tuple[float, ...] = DEFAULT_OUTPUT_SCALES, + duration_s: float | None = None, + seed: int = 0, +) -> Iterator[tuple[Trace, Regime]]: + """Sample the whole grid. One seed per cell, derived, so cells are independent + but the sweep as a whole is reproducible from ``seed`` alone.""" + for i, rm in enumerate(rate_mults): + for j, b in enumerate(burst_targets): + for k, os_ in enumerate(output_scales): + yield sample_trace( + fit_, + rate_mult=rm, + burstiness=b, + output_scale=os_, + duration_s=duration_s, + seed=seed + 1000 * i + 100 * j + 10 * k, + ) diff --git a/gitm/traffic/regime.py b/gitm/traffic/regime.py new file mode 100644 index 0000000..fb6f63d --- /dev/null +++ b/gitm/traffic/regime.py @@ -0,0 +1,183 @@ +"""Workload regimes — the coordinates a playbook row is stated in. + +A :class:`Regime` is what turns "this knob won once" into "this knob wins in +decode-heavy bursty traffic". Every generated workload carries one, the harness +writes :meth:`Regime.label` into every result row, and deliverable 4 keys the +playbook on it. + +Two representations, deliberately both: + +* the **raw axes** (floats), which deliverable 4's match semantics measure a + distance on — live traffic never lands exactly on a measured point; +* the **label** (:meth:`Regime.label`), coarse and bucketed, which is what a + result row carries and a human reads. A label built from raw floats would + never match twice. + +The burstiness axis is the **index of dispersion** — variance over mean of the +arrival counts per bin — not the coefficient of variation of interarrival times. +Both are 1 for a Poisson process, but two traces with the same mean rate and +different bunching collapse to one CV under a fixed mean, and bunching is exactly +the axis the customer's traffic varies on. +""" + +from __future__ import annotations + +from enum import Enum + +import numpy as np +from pydantic import BaseModel, ConfigDict, Field + +from gitm.traffic.schema import Trace + +#: Default counting-bin width for the burstiness statistic, seconds. The index of +#: dispersion is only meaningful relative to a bin: 1 s reads sub-second bunching +#: as burst, 60 s reads it as smooth. Pinned here so every regime is comparable. +DEFAULT_BIN_S = 1.0 + + +class SourceKind(str, Enum): + """Where a workload came from — a schema field, not a naming convention. + + ``SCOREBOARD`` exists so Artificial Analysis's fixed-length benchmark can + never be read as production traffic. It is the public scoreboard condition; + a playbook row measured under it says something about the scoreboard, and the + distinction has to survive being copied into a spreadsheet. + """ + + PRODUCTION = "production" # replayed from a real production trace + SYNTHETIC = "synthetic" # sampled from a fitted envelope + SCOREBOARD = "scoreboard" # fixed-length public benchmark condition + + +def _bucket_tokens(n: float) -> str: + """Round a token count down to a power of two and render it compactly.""" + if n < 1: + return "0" + exp = int(np.floor(np.log2(n))) + v = 1 << exp + return f"{v // 1024}k" if v >= 1024 else str(v) + + +def _bucket_ratio(r: float) -> str: + if r <= 0: + return "io0" + exp = int(np.floor(np.log2(r))) + return f"io{2**exp}" if exp >= 0 else f"io1-{2 ** -exp}" + + +def _bucket_burst(d: float) -> str: + if d < 0.8: + return "burst-flat" # more regular than Poisson (paced or rate-limited) + if d < 1.5: + return "burst-poisson" + if d < 5.0: + return "burst-mod" + return "burst-hi" + + +def index_of_dispersion(arrivals: list[float], *, bin_s: float, span_s: float) -> float: + """Variance / mean of arrival counts per ``bin_s`` bin. 1.0 for Poisson. + + Returns 1.0 when the statistic is undefined — fewer than two bins, or a zero + mean — rather than raising, so a degenerate trace still produces a regime. + The caller learns it was degenerate from :attr:`Regime.burstiness_defined`. + """ + if span_s <= 0 or len(arrivals) < 2: + return 1.0 + nbins = max(int(np.ceil(span_s / bin_s)), 1) + if nbins < 2: + return 1.0 + counts = np.bincount( + np.minimum((np.asarray(arrivals) / bin_s).astype(np.int64), nbins - 1), + minlength=nbins, + ) + mean = counts.mean() + return float(counts.var() / mean) if mean > 0 else 1.0 + + +class Regime(BaseModel): + """One point in workload space, plus the identity of what produced it.""" + + model_config = ConfigDict(extra="forbid") + + source_kind: SourceKind + trace: str # trace identity — adapter name, or the synthetic generator's id + requests: int + rate_rps: float + io_ratio: float # total input tokens / total output tokens + input_p50: int + input_p95: int + output_p50: int + output_p95: int + burstiness: float # index of dispersion at bin_s + bin_s: float = DEFAULT_BIN_S + burstiness_defined: bool = True + concurrency: int | None = None # offered concurrency cap; None = open-loop + in_envelope: bool = True # False when sampled beyond any observed trace + notes: list[str] = Field(default_factory=list) + + @classmethod + def from_trace( + cls, + trace: Trace, + *, + source_kind: SourceKind = SourceKind.PRODUCTION, + bin_s: float = DEFAULT_BIN_S, + concurrency: int | None = None, + in_envelope: bool = True, + trace_id: str | None = None, + ) -> Regime: + if not trace.requests: + raise ValueError("cannot tag an empty trace with a regime") + inp = np.asarray(trace.input_tokens, dtype=float) + outs = trace.output_tokens + out = np.asarray(outs, dtype=float) if outs else np.zeros(1) + span = trace.meta.span_s + arrivals = trace.arrivals + return cls( + source_kind=source_kind, + trace=trace_id or trace.meta.source, + requests=len(trace.requests), + rate_rps=trace.rate_rps(), + io_ratio=float(inp.sum() / out.sum()) if out.sum() > 0 else 0.0, + input_p50=int(np.percentile(inp, 50)), + input_p95=int(np.percentile(inp, 95)), + output_p50=int(np.percentile(out, 50)), + output_p95=int(np.percentile(out, 95)), + burstiness=index_of_dispersion(arrivals, bin_s=bin_s, span_s=span), + bin_s=bin_s, + burstiness_defined=span > 0 and len(arrivals) >= 2, + concurrency=concurrency, + in_envelope=in_envelope, + notes=[] if outs else ["no output lengths in the source; io_ratio is 0"], + ) + + def label(self) -> str: + """The stable, coarse string the harness writes into every result row. + + Bucketed on purpose: two runs of the same workload must produce the same + label, and raw quantiles never repeat. An out-of-envelope point is + suffixed ``/xenv`` so a sampled extrapolation can never be mistaken for a + measured production point. + """ + parts = [ + {"production": "prod", "synthetic": "syn", "scoreboard": "board"}[ + self.source_kind.value + ], + _bucket_ratio(self.io_ratio), + f"in{_bucket_tokens(self.input_p50)}", + f"out{_bucket_tokens(self.output_p50)}", + _bucket_burst(self.burstiness), + f"c{self.concurrency}" if self.concurrency else "copen", + ] + if not self.in_envelope: + parts.append("xenv") + return "/".join(parts) + + def summary(self) -> str: + return ( + f"{self.label()} [{self.requests} req, {self.rate_rps:.3f} rps, " + f"in p50/p95 {self.input_p50}/{self.input_p95}, " + f"out p50/p95 {self.output_p50}/{self.output_p95}, " + f"D={self.burstiness:.2f}@{self.bin_s:g}s]" + ) diff --git a/gitm/traffic/replay.py b/gitm/traffic/replay.py new file mode 100644 index 0000000..cf358d7 --- /dev/null +++ b/gitm/traffic/replay.py @@ -0,0 +1,262 @@ +"""Replay mode — fire a canonical trace as-is, through a tool that already exists. + +vLLM's bench-serve has a native ``timed_trace`` dataset that consumes +``{"timestamp", "input_length", "output_length", "hash_ids"}`` JSONL and, under +``--self-timed``, schedules each request at **its own timestamp** rather than at +a synthesized rate. That is faithful replay, already written, already maintained. +So this module writes that file and builds that command line. There is no load +generator here and there should never be one. + +Two things this module exists to get right, both silent failures otherwise: + +**Block coverage.** ``timed_trace`` builds each prompt by expanding ``hash_ids`` +at ``--timed-trace-chunk-hash-size`` tokens per id and stops when the ids run +out. Pass 16 (the vLLM default) for a Mooncake trace whose blocks are 512 tokens +and every prompt comes out 32x short while every count in the output still looks +correct. :func:`write_timed_trace` checks ``len(blocks) * block_tokens >= +input_tokens`` for every request and refuses the whole file if it does not hold. + +**Sources with no prefix identity.** BurstGPT has no ``hash_ids``. Emitting an +empty list produces a zero-length prompt, not a 472-token one. So blocks are +*synthesized*: each request gets its own fresh run of ids, globally unique, so +lengths are honoured and **no prefix sharing is invented that the source never +had**. The plan records that this happened; a prefix-cache experiment reading a +plan with ``prefix_synthesized=True`` must reject it. +""" + +from __future__ import annotations + +import json +import math +from pathlib import Path + +from pydantic import BaseModel, ConfigDict, Field + +from gitm.traffic.schema import Trace, TraceMeta + +#: Block size used when synthesizing prefix ids for a source that has none. +#: Arbitrary — the ids are unique per request either way — but it sets how many +#: ids each row carries, so keep it large enough that the file stays small. +SYNTHETIC_BLOCK_TOKENS = 512 + +#: First vLLM release containing the ``timed_trace`` dataset. The feature landed +#: 2026-05-28 (``bfb9ebc21``), which missed v0.22.0 by one day: v0.22.1 does +#: **not** have it, v0.23.0 does — checked at both tags, not inferred from dates. +#: +#: This matters because the repo's own ``[vllm]`` extra says ``vllm>=0.6`` and +#: ChunkPrefill's Phase B says ``>=0.19.0``. Either would install a vLLM with no +#: ``timed_trace`` at all, and the failure is an argparse complaint about an +#: unknown dataset name — which reads like a typo in our command, not a missing +#: feature. The number lives here rather than in prose so the argv builder can +#: say it out loud. +VLLM_MIN_VERSION = "0.23.0" + + +class ReplayPlan(BaseModel): + """What was written, and the command that fires it. + + Carries the source :class:`~gitm.traffic.schema.TraceMeta` verbatim: the + promotion rule (deliverable 2) requires trace identity on every promoted row, + and a plan that has forgotten which bytes it came from cannot supply it. + """ + + model_config = ConfigDict(extra="forbid") + + path: str + requests: int + span_s: float + #: Token totals of what was written. Here so a *result* can be reconciled + #: against the trace after the trace object is gone: ``bench serve`` reports + #: ``total_input_tokens``, and comparing it to this is what catches a run + #: fired at the wrong ``--timed-trace-chunk-hash-size``. At vLLM's default of + #: 16 against Mooncake's 512-token blocks the run reports ~1/32 of this while + #: every other count still looks right. See :mod:`gitm.traffic.results`. + input_tokens_total: int = 0 + output_tokens_total: int = 0 + chunk_hash_size: int + sec_multiplier: float = 1.0 # we always emit seconds; vLLM's default is 1 + self_timed: bool = True + prefix_synthesized: bool = False + source: TraceMeta + notes: list[str] = Field(default_factory=list) + + def bench_serve_argv( + self, + *, + model: str, + base_url: str = "http://127.0.0.1:8000", + backend: str = "openai", + tokenizer: str | None = None, + max_concurrency: int | None = None, + result_filename: str | None = None, + seed: int = 0, + ) -> list[str]: + """The exact ``vllm bench serve`` invocation for this plan. + + **Requires vLLM >= :data:`VLLM_MIN_VERSION`.** Below that there is no + ``timed_trace`` dataset and the run dies on an argparse error that looks + like a typo in this command rather than a missing feature. + + ``backend`` must be a completions backend (``openai`` or ``vllm``): + ``timed_trace`` passes pre-tokenized prompts, which the chat endpoints + will not take. + + ``tokenizer`` is worth passing whenever the **served** model id is not an + id HuggingFace can resolve. ``bench serve`` builds a tokenizer from + ``--model`` when none is given, so a server started with + ``--served-model-name`` — or any stub — fails inside + ``AutoTokenizer.from_pretrained`` with *"X is not a local folder and is + not a valid model identifier"*, long after the endpoint answered fine. + Pre-tokenized prompts do not save you from this: the tokenizer is still + constructed for output accounting. + + ``result_filename`` is worth passing. What comes back does **not** + identify the workload: ``bench serve``'s result JSON carries + ``backend``, ``model_id``, ``num_prompts``, ``max_concurrency`` and the + metrics, but no trace identity and no regime. Worse, under + ``--self-timed`` it still records the CLI's ``request_rate`` and + ``burstiness`` defaults, which are meaningless — the real values came + from the trace and are on this plan's ``Regime``. Joining the two is the + open work; see the spec's "what the result JSON does not carry". + """ + argv = [ + "vllm", "bench", "serve", + "--backend", backend, + "--model", model, + "--base-url", base_url, + "--dataset-name", "timed_trace", + "--dataset-path", self.path, + "--num-prompts", str(self.requests), + "--timed-trace-sec-multiplier", f"{self.sec_multiplier:g}", + "--timed-trace-chunk-hash-size", str(self.chunk_hash_size), + "--seed", str(seed), + ] + if tokenizer is not None: + argv += ["--tokenizer", tokenizer] + argv.append("--self-timed" if self.self_timed else "--no-self-timed") + if max_concurrency is not None: + argv += ["--max-concurrency", str(max_concurrency)] + if result_filename is not None: + argv += ["--save-result", "--result-filename", result_filename] + return argv + + +def write_timed_trace( + trace: Trace, + out: str | Path, + *, + block_tokens: int | None = None, +) -> ReplayPlan: + """Write ``trace`` as a vLLM ``timed_trace`` JSONL and return the plan. + + ``block_tokens`` defaults to the trace's own + :attr:`TraceMeta.prefix_block_tokens` when it has prefix identity, and to + :data:`SYNTHETIC_BLOCK_TOKENS` when ids are being synthesized. + + Raises if any request lacks an output length (replay-as-is is not defined for + it — use the parameterized mode) or if block coverage would truncate a prompt. + """ + out = Path(out) + missing_out = sum(1 for r in trace.requests if r.output_tokens is None) + if missing_out: + raise ValueError( + f"{missing_out} of {len(trace.requests)} requests have no output length; " + "replay-as-is is undefined for them — use gitm.traffic.parameterize" + ) + + synthesize = not trace.meta.has_prefix_identity + if block_tokens is None: + block_tokens = ( + SYNTHETIC_BLOCK_TOKENS if synthesize else (trace.meta.prefix_block_tokens or 0) + ) + if block_tokens <= 0: + raise ValueError( + "block_tokens must be positive — a trace with prefix identity must " + "record how many tokens one block id stands for" + ) + + lines: list[str] = [] + next_block_id = 0 + for i, req in enumerate(trace.requests): + if synthesize or not req.prefix_blocks: + n = math.ceil(req.input_tokens / block_tokens) + blocks = list(range(next_block_id, next_block_id + n)) + next_block_id += n + else: + blocks = list(req.prefix_blocks) + if len(blocks) * block_tokens < req.input_tokens: + raise ValueError( + f"request {i}: {len(blocks)} blocks x {block_tokens} tokens cannot " + f"cover input_tokens={req.input_tokens}. The prompt would be " + f"silently truncated — check block_tokens against the source " + f"(Mooncake is 512, not vLLM's default 16)." + ) + lines.append( + json.dumps( + { + "timestamp": round(req.arrival_s, 6), + "input_length": req.input_tokens, + "output_length": req.output_tokens, + "hash_ids": blocks, + } + ) + ) + + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text("\n".join(lines) + "\n", encoding="utf-8", newline="\n") + + notes = ["timestamps emitted in seconds, so sec_multiplier is 1"] + if synthesize: + notes.append( + "prefix blocks SYNTHESIZED (source has no prefix identity): ids are unique " + "per request, so no prefix sharing is implied. Not valid for prefix-cache " + "experiments." + ) + if trace.meta.has_session_identity: + # BurstGPT_3 is exactly this case: it knows which requests are turns of + # one conversation, but not what they share. Later turns of a session do + # re-send the conversation so far, so unique-per-request blocks + # UNDERSTATE the real reuse. Understating is the safe direction — it + # never invents a cache hit — but a measured prefix-cache benefit on + # this trace is a floor, not an estimate, and that has to be said. + notes.append( + f"source has session identity ({trace.meta.session_rows} rows in " + f"{trace.meta.sessions} sessions) but no prefix identity: real " + "conversations share a prefix that these unique ids do not reproduce. " + "Prefix-cache reuse is UNDERSTATED, never overstated." + ) + if trace.meta.has_session_identity: + # Said on every session-carrying plan, not just the synthesized ones: the + # timed_trace format has no session field at all, so conversation identity + # stops at this boundary. Deriving prefix blocks from sessions would make + # it flow, but only by asserting how much each turn re-sends — an + # invented cache hit, which is the one thing this module will not do. + notes.append( + "session identity is NOT carried into the replay file: vLLM's timed_trace " + "format has no session field. Sessions are available for analysis and " + "regime characterization, not for session-aware firing." + ) + return ReplayPlan( + path=str(out), + requests=len(trace.requests), + span_s=trace.meta.span_s, + input_tokens_total=sum(r.input_tokens for r in trace.requests), + output_tokens_total=sum(r.output_tokens or 0 for r in trace.requests), + chunk_hash_size=block_tokens, + prefix_synthesized=synthesize, + source=trace.meta, + notes=notes, + ) + + +def read_timed_trace(path: str | Path, *, source: str = "timed_trace") -> Trace: + """Read a ``timed_trace`` JSONL back into canonical form. + + The inverse of :func:`write_timed_trace`, and the reason the replay claim is + evidence: :mod:`gitm.traffic.validate` compares this against the trace the + adapter produced, so "the pipeline preserves the trace" is a measured + statement about the file the benchmark will actually consume. + """ + from gitm.traffic.adapters import read_mooncake + + return read_mooncake(path, time_scale=1.0, source=source) diff --git a/gitm/traffic/results.py b/gitm/traffic/results.py new file mode 100644 index 0000000..a375484 --- /dev/null +++ b/gitm/traffic/results.py @@ -0,0 +1,308 @@ +"""Seam 3 — join what ``bench serve`` writes back to the workload that produced it. + +Deliverable 1's stated purpose includes *"the harness writes the regime label into +every result row"*. That was true of the tagger and false of the pipeline: the +result JSON comes from vLLM and nothing read it. This module reads it. + +**The result JSON is not merely incomplete — two of its fields are wrong.** +Measured, not assumed, against a real 0.28.0 run (all 33 keys are enumerated in +the standup's `verification.md`): + +``` +result.json request_rate "inf" burstiness 1.0 <- the CLI's defaults +the regime rate_rps 2.837 D 6.74 <- what actually happened +``` + +Under ``--self-timed`` the request schedule comes from the trace, so vLLM never +uses those two flags — but it records them anyway, untouched, beside real +metrics. They are exactly the two axes the playbook keys on. So they are +**dropped with a reason** rather than merged, renamed, or silently overwritten: +:attr:`BenchRun.dropped` shows a reader that they were considered and why they +did not survive. A field quietly removed looks like a field nobody thought about. + +**Reconciliation is the other half.** A result that cannot be tied back to its +trace is not evidence, and the tie has to be checked rather than assumed — the +run may have been fired at the wrong file, the wrong block size, or a server that +dropped half of it. :func:`join_result` runs the checks and records them; a +:class:`BenchRun` that does not reconcile still exists, and says so, because a +failed run is a finding. + +The one this exists for: **``total_input_tokens`` against the trace's own total.** +At vLLM's default ``--timed-trace-chunk-hash-size 16`` against Mooncake's +512-token blocks, every prompt is 32x short while completed, duration, throughput +and every percentile still read perfectly. The emitter refuses to write such a +file (:func:`gitm.traffic.replay.write_timed_trace`); this catches the case where +one was fired anyway, from a plan built elsewhere. + +**R1 is not solved here.** Knob and environment fields stay ``pending-adit``. +This module makes the *trace* half of a playbook row's provenance real; the +config-capture half arrives when Adit's types do, and the joiner is where they +land. +""" + +from __future__ import annotations + +import math +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + +from gitm.traffic.regime import Regime +from gitm.traffic.replay import ReplayPlan +from gitm.traffic.schema import TraceMeta + +#: Schema identity, in the style of ``gitm.bench.manifest.SCHEMA``. +SCHEMA = "gitm.traffic.benchrun/v1" + +#: Waiting on the shared config-capture schema (risk R1). Grep-able. +PENDING_ADIT = "pending-adit" + +#: Result fields that are **wrong** under ``--self-timed``, and why. Dropped, not +#: merged: the trace decided the schedule, so these are the CLI defaults vLLM +#: never consulted, sitting next to real metrics on the two axes a playbook row +#: is keyed on. +MISLEADING_UNDER_SELF_TIMED = { + "request_rate": "CLI default, not the trace's; the real value is regime.rate_rps", + "burstiness": "CLI default, not the trace's; the real value is regime.burstiness", +} + +#: Metrics kept from the result JSON. Deliberately a list rather than "everything +#: except the dropped ones": a future vLLM adding a field should not silently +#: enlarge what a playbook row claims to have measured. +KEPT_METRICS = ( + "duration", "completed", "failed", "num_prompts", + "total_input_tokens", "total_output_tokens", + "request_throughput", "output_throughput", "total_token_throughput", + "request_goodput", "max_output_tokens_per_s", "max_concurrent_requests", + "mean_ttft_ms", "median_ttft_ms", "std_ttft_ms", "p99_ttft_ms", + "mean_tpot_ms", "median_tpot_ms", "std_tpot_ms", "p99_tpot_ms", + "mean_itl_ms", "median_itl_ms", "std_itl_ms", "p99_itl_ms", + "model_id", "tokenizer_id", "backend", "endpoint_type", "date", "label", + "max_concurrency", + # Real-time factor. 0.0 on text serving — it is an ASR metric — but it is a + # field vLLM measured, and `unjoined_keys` exists precisely so a measured + # field cannot fall out of the record without someone deciding it should. + "rtfx", +) + +#: How far a paced run may overrun the trace's span before the schedule is +#: considered to have drifted. Startup and teardown cost a little; a saturating +#: server costs a lot. 5 % of the span plus one second, so short traces are not +#: judged by a percentage of nothing. +DRIFT_TOLERANCE = 0.05 +DRIFT_FLOOR_S = 1.0 + + +class Check(BaseModel): + """One reconciliation between the result and the trace it should describe.""" + + model_config = ConfigDict(extra="forbid") + + name: str + expected: float | int | str | None + actual: float | int | str | None + ok: bool + detail: str = "" + + +class BenchRun(BaseModel): + """One measured run, joined to the workload that produced it. + + This is what deliverable 2 promotes from and what deliverable 4's provenance + is built out of. It is deliberately *not* a playbook row: a row is a **delta + between two arms**, and this is one arm. + """ + + model_config = ConfigDict(extra="forbid") + + schema_id: str = SCHEMA + + # --- what workload this was, which the result JSON does not say ------------ + source: TraceMeta + regime: Regime + regime_label: str + + # --- the conditions it was fired under ------------------------------------- + chunk_hash_size: int + self_timed: bool + prefix_synthesized: bool + + # --- what came back --------------------------------------------------------- + metrics: dict[str, Any] = Field(default_factory=dict) + #: Fields removed from ``metrics``, mapped to why. Kept visible on purpose. + dropped: dict[str, str] = Field(default_factory=dict) + #: The values those fields held, so the record shows what was rejected. + dropped_values: dict[str, Any] = Field(default_factory=dict) + + # --- does it describe the trace it claims to? ------------------------------- + checks: list[Check] = Field(default_factory=list) + + # --- R1 --------------------------------------------------------------------- + knobs: dict[str, bool | int | float | str] = Field(default_factory=dict) + config_capture: str = PENDING_ADIT + + #: The result JSON exactly as vLLM wrote it. Nothing is lost by the join — + #: a dropped field is dropped from ``metrics``, not from the record. + raw: dict[str, Any] = Field(default_factory=dict) + + @property + def reconciled(self) -> bool: + """Whether this result actually describes the trace it is joined to.""" + return all(c.ok for c in self.checks) + + @property + def promotable(self) -> bool: + """Whether deliverable 2 may even look at this run. + + Necessary, never sufficient: D2 owns the promotion rule. What this says + is that the run reconciles with its trace and completed without failures + — below that bar there is nothing for a rule to judge. + """ + return self.reconciled and self.metrics.get("failed", 1) == 0 + + def failures(self) -> list[Check]: + return [c for c in self.checks if not c.ok] + + def summary(self) -> str: + head = "reconciled" if self.reconciled else f"NOT RECONCILED ({len(self.failures())})" + return ( + f"{self.source.source} / {self.regime_label} — {head}: " + f"{self.metrics.get('completed')} completed, " + f"{self.metrics.get('duration', 0.0):.2f}s, " + f"p99 TTFT {self.metrics.get('p99_ttft_ms', float('nan')):.1f}ms" + ) + + def render(self) -> str: + lines = [self.summary(), ""] + w = max((len(c.name) for c in self.checks), default=4) + for c in self.checks: + mark = "ok " if c.ok else "FAIL" + lines.append(f" {mark} {c.name.ljust(w)} expected {c.expected} actual {c.actual}" + + (f" — {c.detail}" if c.detail else "")) + if self.dropped: + lines.append("") + lines.append(" dropped from metrics (recorded, not merged):") + for k, why in sorted(self.dropped.items()): + lines.append(f" {k} = {self.dropped_values.get(k)!r}: {why}") + return "\n".join(lines) + + +def _num(v: Any) -> float | None: + """Coerce a result value to a float, or ``None``. + + ``request_rate`` arrives as the **string** ``"inf"`` — ``json.dumps`` cannot + write a bare ``Infinity``, so vLLM stringifies it. A joiner that assumed + float would raise on the one field it is trying to throw away. + """ + if isinstance(v, bool) or v is None: + return None + if isinstance(v, int | float): + return float(v) + try: + return float(v) + except (TypeError, ValueError): + return None + + +def join_result( + result: dict[str, Any], + plan: ReplayPlan, + regime: Regime, + *, + knobs: dict[str, bool | int | float | str] | None = None, + drift_tolerance: float = DRIFT_TOLERANCE, +) -> BenchRun: + """Join a ``bench serve`` result to the plan and regime that produced it. + + Keeps the metrics deliverable 2 needs, drops the two fields that are wrong + under ``--self-timed``, attaches the workload identity the result JSON has no + room for, and checks that the numbers actually describe this trace. + """ + metrics = {k: result[k] for k in KEPT_METRICS if k in result} + dropped, dropped_values = {}, {} + for k, why in MISLEADING_UNDER_SELF_TIMED.items(): + if k in result: + dropped_values[k] = result[k] + dropped[k] = why if plan.self_timed else f"{why} (run was NOT self-timed)" + + checks: list[Check] = [] + + def add(name, expected, actual, ok, detail=""): + checks.append(Check(name=name, expected=expected, actual=actual, ok=ok, detail=detail)) + + # 1. Every request the plan wrote should have been sent. + completed = result.get("completed") + add("requests_completed", plan.requests, completed, completed == plan.requests, + "" if completed == plan.requests else "the run did not replay the whole trace") + + # 2. Failures are disqualifying, not a footnote. + failed = result.get("failed") + add("no_failed_requests", 0, failed, failed == 0, + "" if failed == 0 else "a run with failures measures the failures too") + + # 3. THE one. Input tokens must match the trace, or the prompts were not the + # trace's prompts — the 32x truncation that leaves every other count right. + ti = result.get("total_input_tokens") + expected_in = plan.input_tokens_total + ok_tokens = ti == expected_in + detail = "" + if not ok_tokens and ti and expected_in: + ratio = expected_in / ti + detail = f"{ratio:.1f}x short — check --timed-trace-chunk-hash-size " \ + f"(the plan wrote {plan.chunk_hash_size}-token blocks)" if ratio > 1.5 else \ + "prompt lengths do not match the trace" + add("input_tokens_match_trace", expected_in, ti, ok_tokens, detail) + + # 4. Pacing, and the two directions mean different things. Only meaningful + # when the run was self-timed; a rate-driven run has no schedule to hold. + duration = _num(result.get("duration")) + if plan.self_timed and duration is not None and plan.span_s > 0: + budget = plan.span_s * (1 + drift_tolerance) + DRIFT_FLOOR_S + if duration < plan.span_s * (1 - drift_tolerance): + ok, why = False, ("finished FASTER than the trace's own span — the timestamps " + "were not honoured, so this is not a replay of this trace") + elif duration > budget: + ok, why = False, ("overran the trace's span — the schedule drifted, which usually " + "means the server saturated; the arrival pattern was not delivered") + else: + ok, why = True, "" + add("paced_to_trace_span", round(plan.span_s, 3), round(duration, 3), ok, why) + + # 5. A synthesized-prefix run cannot speak about cache reuse. Not a failure — + # a label, because the number is a floor and D4 has to know. + if plan.prefix_synthesized: + add("prefix_identity", "from source", "synthesized", True, + "source had no prefix identity; blocks were synthesized unique per request, so any " + "cache-reuse reading from this run is a FLOOR, never an estimate") + + return BenchRun( + source=plan.source, + regime=regime, + regime_label=regime.label(), + chunk_hash_size=plan.chunk_hash_size, + self_timed=plan.self_timed, + prefix_synthesized=plan.prefix_synthesized, + metrics=metrics, + dropped=dropped, + dropped_values=dropped_values, + checks=checks, + knobs=dict(knobs or {}), + raw=dict(result), + ) + + +def unjoined_keys(result: dict[str, Any]) -> list[str]: + """Result keys that are neither kept nor deliberately dropped. + + A new vLLM release adding a field should be a decision, not a silent + omission. `check_join_accounts_for_every_result_key` fails when this is + non-empty for the recorded real-run keys. + """ + known = set(KEPT_METRICS) | set(MISLEADING_UNDER_SELF_TIMED) + return sorted(k for k in result if k not in known) + + +def is_infinite(v: Any) -> bool: + """``request_rate`` arrives as ``"inf"``. Named so the string is not a surprise.""" + n = _num(v) + return n is not None and math.isinf(n) diff --git a/gitm/traffic/runner.py b/gitm/traffic/runner.py new file mode 100644 index 0000000..29a747c --- /dev/null +++ b/gitm/traffic/runner.py @@ -0,0 +1,325 @@ +"""Seam 2 — run the argv the replay module builds, and say what happened. + +:meth:`~gitm.traffic.replay.ReplayPlan.bench_serve_argv` builds the command; +nothing ran it. This module does, and it is deliberately thin — the interesting +work was already done, and everything hard about talking to a vLLM server is +already solved in :mod:`gitm.serve.vllm`, which this reuses rather than reimplements. + +Three things it does that a bare ``subprocess.run`` would not: + +**The version guard fires first.** Below :data:`~gitm.traffic.replay.VLLM_MIN_VERSION` +there is no ``timed_trace`` dataset, and vLLM's own failure is an argparse +complaint about an unknown dataset name — which reads like a typo in *our* +command rather than a missing feature, and costs an afternoon. Checking before +launching converts that into a sentence naming the version and the flag. + +**The model id comes from the server, not from the caller.** ``--served-model-name`` +can rename a model, and a completion request with the wrong id is a **404, not a +slow path** — a whole run that looks like it went badly rather than one that never +started. :func:`gitm.serve.vllm.served_model_name` already knows this; asking it +also proves the endpoint is answering before we fire a few thousand requests at it. + +**The result is joined, not just carried.** ``bench serve``'s JSON has the +metrics and **no trace identity, no regime, no config capture** — and under +``--self-timed`` it still records the CLI's ``request_rate`` and ``burstiness`` +defaults, which are wrong because the real values came from the trace. Pass +``regime=`` and :func:`run_replay` hands the result to +:func:`gitm.traffic.results.join_result` (seam 3), which attaches the workload +identity, drops those two fields with a reason, and reconciles the reported +totals against the trace. Without ``regime=`` the raw result is still carried and +a note says what is missing — joining is not silently skipped. + +Verified against real ``vllm bench serve`` 0.28.0: 40/40 requests, paced to +12.008 s against a 12.000 s trace span. The parts that still need a server — +output-length fidelity, and whether the schedule holds when a real server +saturates — are named in the standup's ``verification.md``. :func:`run_replay` +takes ``dry_run`` so the argv and the guard stay exercisable without one. +""" + +from __future__ import annotations + +import json +import re +import shutil +import subprocess +import sys +import time +from pathlib import Path + +from pydantic import BaseModel, ConfigDict, Field + +from gitm.traffic.regime import Regime +from gitm.traffic.replay import VLLM_MIN_VERSION, ReplayPlan +from gitm.traffic.results import BenchRun, join_result +from gitm.traffic.schema import TraceMeta + +#: How much of a failed run's output to keep. Enough to see the argparse line or +#: the traceback; not so much that a result row carries a log file. +TAIL_CHARS = 4000 + + +class VllmUnavailable(RuntimeError): + """vLLM is missing or too old. The message names the version and the flag.""" + + +def installed_vllm_version() -> str | None: + """The installed vLLM version, or ``None`` if it is not importable. + + Reads package metadata rather than importing ``vllm``: the import pulls in + torch and CUDA and takes tens of seconds, and this runs before every launch. + """ + try: + from importlib.metadata import PackageNotFoundError, version + + try: + return version("vllm") + except PackageNotFoundError: + return None + except ImportError: # pragma: no cover - importlib.metadata is stdlib + return None + + +def _release(v: str) -> tuple[int, ...]: + """The numeric release part of a version, for comparison. + + ``0.23.0rc1`` -> ``(0, 23, 0)``; ``0.23.0+cu128`` -> ``(0, 23, 0)``. Not a + full PEP 440 implementation on purpose — this compares a floor, and the only + thing that matters is that a release candidate does not read as newer than + its release, nor a local/build tag as older. + """ + head = re.split(r"[^0-9.]", v.strip(), maxsplit=1)[0].rstrip(".") + return tuple(int(p) for p in head.split(".") if p.isdigit()) or (0,) + + +def vllm_executable() -> str: + """The ``vllm`` console script **belonging to the running interpreter**. + + Not a bare ``"vllm"`` on ``PATH``, for two reasons and the second is the + serious one: + + * ``python -m gitm.traffic`` run with an absolute interpreter — a conda env + invoked without activation, which is the normal case in CI and under WSL — + has that env's ``bin/`` nowhere on ``PATH``, and the launch dies with + ``FileNotFoundError: 'vllm'``. + * **The guard and the run could disagree.** :func:`check_vllm` reads + ``importlib.metadata.version("vllm")``, which is the version installed + *for this interpreter*. Resolving the binary from ``PATH`` could then run a + different environment's vLLM — so the version that was validated and the + version that runs would not be the same install. Deriving both from + ``sys.executable`` makes that impossible. + + Falls back to ``"vllm"`` when no sibling script exists, so an unusual layout + still gets a ``PATH`` lookup rather than a hard failure. + """ + bindir = Path(sys.executable).parent + for name in ("vllm", "vllm.exe"): + cand = bindir / name + if cand.exists(): + return str(cand) + found = shutil.which("vllm") + return found or "vllm" + + +def check_vllm(min_version: str = VLLM_MIN_VERSION) -> str: + """Return the installed version, or raise with what to do about it. + + Called before launching, never after: the whole point is that the failure + arrives as a sentence about ``timed_trace`` rather than as vLLM's argparse + error about an unknown dataset name. + """ + found = installed_vllm_version() + if found is None: + raise VllmUnavailable( + f"vllm is not installed. Firing a replay needs vllm>={min_version} " + f"(the version that added `bench serve --dataset-name timed_trace`). " + f"Everything else in gitm.traffic is CPU-only and needs no vLLM. " + f"Install: pip install 'gitm-labs[vllm]'" + ) + if _release(found) < _release(min_version): + raise VllmUnavailable( + f"vllm {found} is too old: `--dataset-name timed_trace` needs " + f">={min_version}. Below it the run dies on an argparse complaint " + f"about an unknown dataset name, which reads like a typo in our " + f"command rather than a missing feature. Upgrade: " + f"pip install -U 'vllm>={min_version}'" + ) + return found + + +class RunResult(BaseModel): + """What the run did, with the plan's provenance and — given a regime — the join. + + Two levels on purpose: :attr:`result` is vLLM's JSON untouched, and + :attr:`joined` is that result tied to the workload, with the misleading fields + dropped and the reconciliation checked. Keeping the raw one means the join is + auditable rather than something you have to trust. + """ + + model_config = ConfigDict(extra="forbid") + + argv: list[str] + returncode: int + duration_s: float + vllm_version: str | None = None + resolved_model: str | None = None # what the server calls it, not what we asked for + stdout_tail: str = "" + stderr_tail: str = "" + result_path: str | None = None + #: ``bench serve``'s raw result JSON, unmodified. **Do not read + #: ``request_rate`` or ``burstiness`` out of this** — under ``--self-timed`` + #: they are the CLI's untouched defaults, and the true values are on + #: :attr:`source`'s regime. :attr:`joined` is that join already done + #: (:mod:`gitm.traffic.results`); this field is the unmodified original. + result: dict | None = None + #: The trace this came from, verbatim from the plan. A measured number whose + #: workload cannot be identified is not evidence, and this is the field that + #: stops the two being separated between here and a playbook row. + source: TraceMeta + #: The joined record — result metrics tied to the regime that produced them, + #: with vLLM's misleading ``request_rate`` / ``burstiness`` dropped and the + #: reconciliation checked. Present when a ``regime`` was supplied and a result + #: file came back; ``None`` otherwise. See :mod:`gitm.traffic.results`. + joined: BenchRun | None = None + notes: list[str] = Field(default_factory=list) + + @property + def ok(self) -> bool: + return self.returncode == 0 + + def summary(self) -> str: + head = "ok" if self.ok else f"FAILED rc={self.returncode}" + got = f", {len(self.result)} result keys" if self.result else "" + return ( + f"{head} in {self.duration_s:.1f}s — {self.source.source} " + f"({self.source.rows_emitted} requests){got}" + ) + + +def run_replay( + plan: ReplayPlan, + *, + model: str, + base_url: str = "http://127.0.0.1:8000", + backend: str = "openai", + tokenizer: str | None = None, + max_concurrency: int | None = None, + result_dir: str | Path | None = None, + seed: int = 0, + regime: Regime | None = None, + timeout_s: float = 3600.0, + resolve_model: bool = True, + dry_run: bool = False, +) -> RunResult: + """Fire ``plan`` at ``base_url`` with ``vllm bench serve``. + + Order of operations, and the first two are the reason this is not a bare + ``subprocess.run``: + + 1. **Version guard**, before anything is launched. + 2. **Resolve the served model id** off ``/v1/models``, which also proves the + endpoint is answering. ``resolve_model=False`` skips it and uses ``model`` + verbatim — for a server that does not expose the route. + 3. Run, capturing stdout, stderr and the exit code. + 4. Read back the result JSON if one was written, and — when a ``regime`` is + supplied — **join** it to the workload that produced it (seam 3). The + join is what makes the result evidence rather than a number: it attaches + the regime and trace identity the result JSON has no room for, drops the + two fields that are wrong under ``--self-timed``, and reconciles the + reported totals against the trace. + + ``dry_run`` builds and returns everything except the subprocess, so the argv + and the guard are exercisable with no vLLM and no server. + """ + notes: list[str] = [] + joined: BenchRun | None = None + found = None if dry_run else check_vllm() + + resolved = model + if resolve_model and not dry_run: + from gitm.serve.vllm import served_model_name + + resolved = served_model_name(base_url, model) + if resolved != model: + notes.append( + f"server calls the model {resolved!r}, not {model!r} " + "(--served-model-name); using the server's id — the wrong one is a 404" + ) + + result_path: Path | None = None + if result_dir is not None: + result_path = Path(result_dir) / f"benchserve_{plan.source.source}_{int(time.time())}.json" + result_path.parent.mkdir(parents=True, exist_ok=True) + + argv = plan.bench_serve_argv( + model=resolved, + base_url=base_url, + backend=backend, + tokenizer=tokenizer, + max_concurrency=max_concurrency, + result_filename=str(result_path) if result_path else None, + seed=seed, + ) + + if dry_run: + notes.append("dry run: nothing was launched") + return RunResult(argv=argv, returncode=0, duration_s=0.0, resolved_model=resolved, + result_path=str(result_path) if result_path else None, + source=plan.source, notes=notes) + + # argv[0] is the documented command name; run the console script that belongs + # to THIS interpreter, so the install check_vllm() validated is the one that + # runs. See vllm_executable(). + exe = vllm_executable() + if exe != argv[0]: + notes.append(f"running {exe} (the console script beside {sys.executable})") + argv = [exe, *argv[1:]] + + started = time.monotonic() + try: + proc = subprocess.run(argv, capture_output=True, text=True, timeout=timeout_s) + rc, out, err = proc.returncode, proc.stdout, proc.stderr + except subprocess.TimeoutExpired as exc: + # A timeout is a result, not an exception to propagate: the partial output + # is the only evidence of how far it got. + rc, out, err = 124, (exc.stdout or b"").decode(errors="replace") if isinstance( + exc.stdout, bytes) else (exc.stdout or ""), f"timed out after {timeout_s:.0f}s" + notes.append(f"killed at the {timeout_s:.0f}s timeout") + duration = time.monotonic() - started + + result = None + if result_path is not None and result_path.exists(): + try: + result = json.loads(result_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + notes.append(f"result file unreadable: {exc}") + else: + if regime is None: + notes.append( + "result JSON carries no trace identity, no regime and no config " + "capture, and under --self-timed its request_rate and burstiness are " + "the CLI defaults rather than the trace's. Pass `regime=` to join it." + ) + else: + joined = join_result(result, plan, regime) + notes.append( + f"joined to {joined.regime_label}; dropped {sorted(joined.dropped)} " + "as CLI defaults that --self-timed never consulted" + ) + if not joined.reconciled: + notes.append( + "RESULT DOES NOT RECONCILE WITH ITS TRACE: " + + "; ".join( + f"{c.name} expected {c.expected} got {c.actual}" + for c in joined.failures() + ) + ) + elif result_path is not None: + notes.append(f"no result file at {result_path} despite --save-result") + + return RunResult( + joined=joined, + argv=argv, returncode=rc, duration_s=duration, vllm_version=found, + resolved_model=resolved, stdout_tail=out[-TAIL_CHARS:], stderr_tail=err[-TAIL_CHARS:], + result_path=str(result_path) if result_path else None, result=result, + source=plan.source, notes=notes, + ) diff --git a/gitm/traffic/schema.py b/gitm/traffic/schema.py new file mode 100644 index 0000000..87f9280 --- /dev/null +++ b/gitm/traffic/schema.py @@ -0,0 +1,204 @@ +"""Canonical request-trace shapes — one format every load source normalizes into. + +Three contracts: + +* :class:`CanonicalRequest` — one request. The unit every adapter emits and every + output mode consumes. A frozen slotted dataclass, not a pydantic model: + a trace is millions of these and they are hot data, not configuration. The + *validation* that would justify pydantic already happens in the adapter, where + a rejected row can be attributed to a named :class:`DropReason` instead of + raising. +* :class:`TraceMeta` — provenance for a whole trace: where the bytes came from, + their sha256, how many rows were read, how many survived, and **what was + dropped and why**. This is the part that gets serialized into a result row, so + it is a pydantic model with ``extra="forbid"``. +* :class:`Trace` — meta + requests, and the reason the two cannot be separated. + Nothing downstream accepts a bare list of requests: a stream you cannot trace + back to bytes is not evidence, and the playbook (deliverable 4) keys on this + provenance. + +Units, stated once and never re-derived: **``arrival_s`` is seconds offset from +the trace's first row**, not an epoch and not the source's native unit. Adapters +convert; :attr:`TraceMeta.raw_time_unit` records what they converted from. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum + +from pydantic import BaseModel, ConfigDict, Field + +#: Schema identity, in the style of ``gitm.bench.manifest.SCHEMA``. Bump on any +#: field change that a consumer could misread as the old meaning. +SCHEMA = "gitm.traffic.trace/v1" + + +class DropReason(str, Enum): + """Why an adapter refused a raw row. + + Every rejection is one of these — an adapter may not drop a row silently, and + the counts travel with the trace in :attr:`TraceMeta.drops`. The first seven + are *defects* in the source; :attr:`FILTERED_OUT` is a selection the caller + asked for and is counted separately so a narrow filter never reads as dirty + data. + """ + + MALFORMED_ROW = "malformed_row" # unparseable line / wrong column count + MISSING_FIELD = "missing_field" # column present in the header, empty in the row + NON_NUMERIC = "non_numeric" # a length or timestamp that is not a number + NEGATIVE_VALUE = "negative_value" # negative tokens or a negative timestamp + ZERO_INPUT_TOKENS = "zero_input_tokens" # nothing to prefill + ZERO_OUTPUT_TOKENS = "zero_output_tokens" # nothing to decode + NON_MONOTONIC_ARRIVAL = "non_monotonic_arrival" # timestamp went backwards + FILTERED_OUT = "filtered_out" # excluded by a caller-supplied filter, not a defect + + +@dataclass(frozen=True, slots=True) +class CanonicalRequest: + """One request, in the only shape the harness fires. + + ============== ============================= ========================================= + field type / unit when the source lacks it + ============== ============================= ========================================= + arrival_s float, **seconds from trace fatal. No adapter may synthesize arrivals; + start** (not epoch) a source without timing is not a trace. + input_tokens int, tokens, ``> 0`` fatal — the row is dropped, never defaulted. + output_tokens int | None, tokens ``None`` = "generate to the regime's sampled + length". Recorded in ``TraceMeta.notes``; + a trace of ``None`` cannot be replayed + as-is, only parameterized. + session_id str | None ``None`` — no conversation identity. + ``TraceMeta.has_session_identity`` is False + and multi-turn experiments must not use it. + prefix_blocks tuple[int, ...], block ids ``()`` — no prefix identity. + in prompt order ``TraceMeta.has_prefix_identity`` is False + and prefix-cache experiments must not use it. + ============== ============================= ========================================= + + ``prefix_blocks`` is a *chain*, not a single hash, because partial sharing is + the whole point: two requests share a prefix exactly as far as their leading + block ids agree. One hash of the whole chain would only ever match identical + prompts, which is the case that does not need measuring. Each id stands for + :attr:`TraceMeta.prefix_block_tokens` tokens. + + ``source_e2e_latency_s`` is the **source system's** end-to-end + submission-to-final-response time, when the trace records one (BurstGPT_3's + ``Elapsed time``). It is carried so the adapter does not destroy real data at + the boundary, and it is named at length so it cannot be mistaken for + something it is not: + + * it is **end-to-end**, not TTFT, and not ITL; + * it was measured on **someone else's hardware, model and load**. + + **Never compare it against a measured TTFT/ITL, and never promote a playbook + row against it.** Its legitimate use is bounding think-time between turns of + a session, where only the source's own timeline matters. + """ + + arrival_s: float + input_tokens: int + output_tokens: int | None = None + session_id: str | None = None + prefix_blocks: tuple[int, ...] = () + source_e2e_latency_s: float | None = None + + +class TraceMeta(BaseModel): + """Provenance for one trace. Without it a trace cannot be replayed. + + ``rows_read`` counts raw records seen; ``rows_emitted`` counts survivors; + ``drops`` maps :class:`DropReason` values to counts. The three must reconcile + (:meth:`Trace.__post_init__` checks it), so "we dropped some bad rows" is + never a hand-wave. + """ + + model_config = ConfigDict(extra="forbid") + + schema_id: str = SCHEMA + source: str # adapter name, e.g. "burstgpt" + path: str # the file that was read + sha256: str # of the raw bytes — pins the trace to bytes, per gitm.bench.manifest + source_url: str | None = None # where the raw file came from, when known + + rows_read: int = 0 + rows_emitted: int = 0 + drops: dict[str, int] = Field(default_factory=dict) + + span_s: float = 0.0 # last arrival minus first, seconds + raw_time_unit: str = "s" # what the source's timestamps were before conversion + + prefix_block_tokens: int | None = None # tokens each prefix-block id stands for + has_prefix_identity: bool = False + has_session_identity: bool = False + #: How much session identity there actually is. ``has_session_identity`` only + #: says *some* row carried one; in BurstGPT_3 that is true while 90 % of rows + #: are single-shot API traffic with no conversation at all. A multi-turn + #: experiment needs the counts, not the flag, to decide whether the trace can + #: carry it. + session_rows: int = 0 + sessions: int = 0 + + notes: list[str] = Field(default_factory=list) + + @property + def dropped(self) -> int: + return sum(self.drops.values()) + + @property + def defects(self) -> int: + """Dropped rows that were *bad data*, excluding caller-requested filtering.""" + return sum(v for k, v in self.drops.items() if k != DropReason.FILTERED_OUT.value) + + def summary(self) -> str: + drops = ", ".join(f"{k}={v}" for k, v in sorted(self.drops.items())) or "none" + return ( + f"{self.source}: {self.rows_emitted}/{self.rows_read} rows over " + f"{self.span_s:.1f}s (drops: {drops})" + ) + + +@dataclass(frozen=True) +class Trace: + """A normalized trace: provenance plus requests, inseparable by construction. + + ``requests`` are held in arrival order. The whole trace is materialized — + ponytail: fine to a few million rows (BurstGPT_1 is ~1.4 M), and the memory + ceiling is `slots` dataclasses at roughly 100 B each. If a source arrives that + does not fit, make the adapters yield and give ``Trace`` a streaming sibling; + nothing above this line assumes random access except the quantile fits. + """ + + meta: TraceMeta + requests: list[CanonicalRequest] = field(default_factory=list) + + def __post_init__(self) -> None: + if self.meta.rows_emitted != len(self.requests): + raise ValueError( + f"TraceMeta.rows_emitted={self.meta.rows_emitted} disagrees with " + f"{len(self.requests)} requests — provenance must reconcile" + ) + if self.meta.rows_read != self.meta.rows_emitted + self.meta.dropped: + raise ValueError( + f"rows_read={self.meta.rows_read} != emitted={self.meta.rows_emitted} " + f"+ dropped={self.meta.dropped} — a row went missing unattributed" + ) + + def __len__(self) -> int: + return len(self.requests) + + @property + def arrivals(self) -> list[float]: + return [r.arrival_s for r in self.requests] + + @property + def input_tokens(self) -> list[int]: + return [r.input_tokens for r in self.requests] + + @property + def output_tokens(self) -> list[int]: + """Output lengths, with ``None`` excluded — callers must check the count.""" + return [r.output_tokens for r in self.requests if r.output_tokens is not None] + + def rate_rps(self) -> float: + return len(self.requests) / self.meta.span_s if self.meta.span_s > 0 else 0.0 diff --git a/gitm/traffic/validate.py b/gitm/traffic/validate.py new file mode 100644 index 0000000..fb8dbd1 --- /dev/null +++ b/gitm/traffic/validate.py @@ -0,0 +1,275 @@ +"""Validation — prove the pipeline preserves the trace, and show the proof. + +The brief asks for this as a deliverable, not a nicety: the replayed stream's +arrival-rate and length distributions compared against the source's, **shown**, +with any mismatch explained. Everything downstream — every regime label, every +promoted playbook row — inherits whatever distortion this step fails to catch. + +What gets compared is the file the benchmark will actually consume. The replay +emitter writes a vLLM ``timed_trace`` JSONL; +:func:`gitm.traffic.replay.read_timed_trace` reads that same file back into +canonical form, and :func:`compare` puts it beside the trace the adapter +produced. So "the pipeline preserves the trace" is a measurement of the artifact, +not an argument about the code. + +The same function serves the second, looser use: a parameterized sample against +the trace it was fitted on. There the thresholds are wider — a sample is drawn +from the envelope, not copied from it — and :data:`SAMPLED_THRESHOLDS` says so +explicitly rather than leaving a reader to wonder which standard was applied. +""" + +from __future__ import annotations + +import sys + +import numpy as np +from pydantic import BaseModel, ConfigDict, Field + +from gitm.traffic.regime import DEFAULT_BIN_S, index_of_dispersion +from gitm.traffic.schema import Trace + + +class Thresholds(BaseModel): + """What counts as preserved. Every number here is a decision, not a default.""" + + model_config = ConfigDict(extra="forbid") + + name: str + #: Two-sample KS statistic on arrival times, and on each length distribution. + ks: float = 0.02 + #: Relative error on mean request rate. + rate: float = 0.01 + #: Relative error on the burstiness axis (index of dispersion). + burstiness: float = 0.10 + #: Relative error on request count. 0 = exact. + count: float = 0.0 + #: Whether to compare the arrival *timeline* at all. True for a replay, which + #: must reproduce it. False for a parameterized sample, which reproduces the + #: rate and the dispersion by construction and the timeline by nothing — a KS + #: on arrival times there measures only that a sample is not a copy, which is + #: the point of sampling. + check_arrival_times: bool = True + + +#: A replay must reproduce the trace, not resemble it: the emitter is a format +#: change, so every statistic should come back identical and the tolerances are +#: there to absorb float rounding of the timestamps, nothing else. +REPLAY_THRESHOLDS = Thresholds(name="replay", ks=0.001, rate=1e-6, burstiness=1e-6, count=0.0) + +#: A parameterized sample is a *draw* from the fitted envelope. Sampling error at +#: a few hundred requests is real, so these are the finite-sample tolerances — +#: loose enough not to fire on noise, tight enough that a broken inverse-CDF or a +#: mis-set dispersion target does fire. +SAMPLED_THRESHOLDS = Thresholds( + name="sampled", ks=0.15, rate=0.25, burstiness=0.60, count=0.35, check_arrival_times=False +) + +#: Arrival times are compared at microsecond resolution. Two reasons, and both +#: are about not lying: a replay's timing fidelity below 1 us is meaningless next +#: to millisecond network jitter, and the emitted JSONL rounds to 6 decimals, so +#: an un-quantized KS reports the 6e-16 s residue of ``5999 / 1000`` as a real +#: distribution gap. It found exactly that on the Mooncake fixture. +ARRIVAL_RESOLUTION_S = 1e-6 + + +class Check(BaseModel): + model_config = ConfigDict(extra="forbid") + + name: str + statistic: float + threshold: float + passed: bool + detail: str = "" + + +class ValidationReport(BaseModel): + model_config = ConfigDict(extra="forbid") + + standard: str + source: str + replayed: str + checks: list[Check] = Field(default_factory=list) + source_hist: list[int] = Field(default_factory=list) + replayed_hist: list[int] = Field(default_factory=list) + hist_bin_s: float = 0.0 + + @property + def passed(self) -> bool: + return all(c.passed for c in self.checks) + + def render(self) -> str: + """The shown comparison: the check table plus both arrival-rate profiles. + + ASCII on purpose — this lands in a terminal, a markdown log and a commit + message, and none of those render a PNG. A plotting dependency would buy + prettier and lose all three. + """ + w = max((len(c.name) for c in self.checks), default=4) + lines = [ + f"trace validation [{self.standard}] {self.source} -> {self.replayed}", + f"{'check'.ljust(w)} {'value':>12} {'threshold':>12} result", + f"{'-' * w} {'-' * 12} {'-' * 12} ------", + ] + for c in self.checks: + mark = "pass" if c.passed else "FAIL" + lines.append( + f"{c.name.ljust(w)} {c.statistic:>12.6g} {c.threshold:>12.6g} {mark}" + ) + lines.append("") + lines.append(f"arrival rate, {self.hist_bin_s:g}s bins (S = source, R = replayed)") + lines.append(_sparkbars(self.source_hist, self.replayed_hist)) + lines.append("") + lines.append("PASS — the pipeline preserves the trace" if self.passed else self.explain()) + return "\n".join(lines) + + def explain(self) -> str: + """Prose for every failed check. A printed number is not an explanation.""" + bad = [c for c in self.checks if not c.passed] + if not bad: + return "no mismatch to explain" + out = ["FAIL — the replayed stream differs from the source:"] + for c in bad: + out.append(f" * {c.name}: {c.statistic:.6g} exceeds {c.threshold:.6g}. {c.detail}") + return "\n".join(out) + + +_BLOCKS = " ▁▂▃▄▅▆▇█" +_ASCII = " .:-=+*#@" + + +def _ramp() -> str: + """Block characters when the console can encode them, ASCII when it cannot. + + A Windows console on cp1252 raises ``UnicodeEncodeError`` on U+2588, which + would turn "show the comparison" into a crash at exactly the moment someone + is looking at a failure. + """ + enc = getattr(sys.stdout, "encoding", None) or "utf-8" + try: + _BLOCKS.encode(enc) + except (UnicodeEncodeError, LookupError): + return _ASCII + return _BLOCKS + + +def _spark(counts: list[int], peak: int) -> str: + ramp = _ramp() + if peak <= 0: + return " " * len(counts) + return "".join( + ramp[min(int(c / peak * (len(ramp) - 1) + 0.5), len(ramp) - 1)] for c in counts + ) + + +def _sparkbars(a: list[int], b: list[int]) -> str: + peak = max([*a, *b, 1]) + return f" S |{_spark(a, peak)}| peak {max(a, default=0)}\n R |{_spark(b, peak)}| peak {max(b, default=0)}" + + +def ks_statistic(a: list[float] | np.ndarray, b: list[float] | np.ndarray) -> float: + """Two-sample Kolmogorov-Smirnov statistic — the max CDF gap. + + Eight lines of numpy rather than a scipy dependency: only the statistic is + needed, never the p-value, because the thresholds here are operational + ("this replay is faithful enough to run on") and not a hypothesis test. + """ + a = np.sort(np.asarray(a, dtype=float)) + b = np.sort(np.asarray(b, dtype=float)) + if a.size == 0 or b.size == 0: + return 1.0 + grid = np.concatenate([a, b]) + ca = np.searchsorted(a, grid, side="right") / a.size + cb = np.searchsorted(b, grid, side="right") / b.size + return float(np.max(np.abs(ca - cb))) + + +def _quantize(xs: list[float]) -> np.ndarray: + """Snap arrival times to :data:`ARRIVAL_RESOLUTION_S` before comparing them.""" + return np.rint(np.asarray(xs, dtype=float) / ARRIVAL_RESOLUTION_S) + + +def _hist(trace: Trace, *, nbins: int, span: float) -> list[int]: + if span <= 0 or not trace.requests: + return [len(trace.requests)] + [0] * (nbins - 1) + idx = np.minimum((np.asarray(trace.arrivals) / span * nbins).astype(int), nbins - 1) + return [int(v) for v in np.bincount(idx, minlength=nbins)] + + +def _rel(a: float, b: float) -> float: + """Relative error of ``b`` against ``a``, with a zero-safe denominator.""" + return abs(a - b) / abs(a) if a else (0.0 if not b else 1.0) + + +def compare( + source: Trace, + replayed: Trace, + *, + thresholds: Thresholds = REPLAY_THRESHOLDS, + bin_s: float = DEFAULT_BIN_S, + hist_bins: int = 60, +) -> ValidationReport: + """Compare a replayed (or sampled) trace against its source.""" + checks: list[Check] = [] + + def add(name: str, stat: float, thr: float, detail: str) -> None: + checks.append( + Check(name=name, statistic=stat, threshold=thr, passed=stat <= thr, detail=detail) + ) + + add( + "request_count", + _rel(len(source), len(replayed)), + thresholds.count, + f"{len(source)} source vs {len(replayed)} replayed — requests were lost or " + "invented between the adapter and the emitted file.", + ) + if thresholds.check_arrival_times: + add( + "arrival_ks", + ks_statistic(_quantize(source.arrivals), _quantize(replayed.arrivals)), + thresholds.ks, + "the replayed arrival times do not follow the source's; timing fidelity " + "is lost, so every burstiness-conditioned result is suspect.", + ) + add( + "input_len_ks", + ks_statistic(source.input_tokens, replayed.input_tokens), + thresholds.ks, + "prompt lengths differ — check block coverage: vLLM expands hash_ids at " + "--timed-trace-chunk-hash-size tokens each and truncates silently when the " + "size is too small (Mooncake is 512, the vLLM default is 16).", + ) + add( + "output_len_ks", + ks_statistic(source.output_tokens, replayed.output_tokens), + thresholds.ks, + "decode lengths differ, which moves the prefill/decode ratio the regime " + "axis is defined on.", + ) + add( + "rate_rps", + _rel(source.rate_rps(), replayed.rate_rps()), + thresholds.rate, + "mean offered rate differs from the source's.", + ) + add( + "burstiness", + _rel( + index_of_dispersion(source.arrivals, bin_s=bin_s, span_s=source.meta.span_s), + index_of_dispersion(replayed.arrivals, bin_s=bin_s, span_s=replayed.meta.span_s), + ), + thresholds.burstiness, + "the index of dispersion moved, so the two traces sit in different regime " + "buckets even where their means agree.", + ) + + span = max(source.meta.span_s, replayed.meta.span_s) + return ValidationReport( + standard=thresholds.name, + source=f"{source.meta.source}({source.meta.sha256[:12]})", + replayed=f"{replayed.meta.source}({replayed.meta.sha256[:12]})", + checks=checks, + source_hist=_hist(source, nbins=hist_bins, span=span), + replayed_hist=_hist(replayed, nbins=hist_bins, span=span), + hist_bin_s=span / hist_bins if span > 0 else 0.0, + ) diff --git a/pyproject.toml b/pyproject.toml index 40bb220..69ec59f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,10 +29,7 @@ nvidia = ["pynvml>=11.5"] # fallback (pandas). On a GPU box the harness runs on cuDF/CuPy instead — those # install from the RAPIDS channel, not PyPI, so they are not listed here. bench = ["pyarrow>=15", "pandas>=2.0"] -# The vllm-decode workload. Deliberately NOT folded into [gpu]: vLLM is a heavy -# install that pins its own torch, and both must be built for a CUDA major the -# HOST driver supports — see gitm/cuda_env.py, which checks exactly that. -vllm = ["vllm>=0.6"] +vllm = ["vllm>=0.23.0"] # Full GPU stack for the real HFT path in two commands: cuDF/CuPy (RAPIDS) + # the CUPTI/runtime wheels the tracer shim links + NVML telemetry + the Parquet # generator deps. cuDF/CuPy resolve from NVIDIA's index, so install with: diff --git a/tests/test_playbook.py b/tests/test_playbook.py new file mode 100644 index 0000000..77d94d4 --- /dev/null +++ b/tests/test_playbook.py @@ -0,0 +1,29 @@ +"""Playbook schema — the pytest face of ``python -m gitm.playbook --selftest``. + +Same pattern as ``tests/test_traffic.py``: the assertions live in +:mod:`gitm.playbook._selftest` and are called from both places, so the runnable +check a reader is told about and the check CI runs cannot drift apart. +""" + +from __future__ import annotations + +import pytest + +from gitm.playbook import _selftest + + +@pytest.mark.parametrize("check", _selftest.CHECKS, ids=lambda f: f.__name__) +def test_playbook_check(check) -> None: + if not _selftest.EXAMPLES.exists(): + pytest.skip(f"examples not present at {_selftest.EXAMPLES}") + check() + + +def test_every_check_is_registered() -> None: + """A check that exists but is never run is worse than no check.""" + defined = { + name + for name in dir(_selftest) + if name.startswith("check_") and callable(getattr(_selftest, name)) + } + assert {f.__name__ for f in _selftest.CHECKS} == defined diff --git a/tests/test_traffic.py b/tests/test_traffic.py new file mode 100644 index 0000000..9d8e5b6 --- /dev/null +++ b/tests/test_traffic.py @@ -0,0 +1,30 @@ +"""Traffic replay library — the pytest face of ``python -m gitm.traffic --selftest``. + +The assertions live in :mod:`gitm.traffic._selftest` and are called from both +places, so the runnable check a reader is told about in the spec and the check CI +runs are the *same* check, not two that can drift apart. + +Each case is one function from ``_selftest.CHECKS``; a failure names the check +that broke rather than the whole library. +""" + +from __future__ import annotations + +import pytest + +from gitm.traffic import _selftest + + +@pytest.mark.parametrize("check", _selftest.CHECKS, ids=lambda f: f.__name__) +def test_traffic_check(check) -> None: + if not _selftest.FIXTURES.exists(): + pytest.skip(f"fixtures not present at {_selftest.FIXTURES}") + check() + + +def test_every_check_is_registered() -> None: + """A check that exists but is never run is worse than no check.""" + defined = { + name for name in dir(_selftest) if name.startswith("check_") and callable(getattr(_selftest, name)) + } + assert {f.__name__ for f in _selftest.CHECKS} == defined