From 1abe22b1bc8b72365cf64a16baefc5be72ae8bc7 Mon Sep 17 00:00:00 2001 From: Felipe705x Date: Fri, 24 Jul 2026 02:47:10 -0400 Subject: [PATCH 1/2] =?UTF-8?q?bench(typing):=20scenario=5Fbench=20?= =?UTF-8?q?=E2=80=94=20session-level=20cost/benefit=20of=20typechecking?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "We cannot improve what we cannot measure": the internal bench measures per-case pipeline cost and pattern_typecheck measures the checker in a hot loop; neither models a SESSION — amortization horizon, shape novelty, human think-time (hardware-cache eviction between queries), error mix, DML interleaving. This bin replays named session profiles end-to-end, tc-on vs tc-off, fresh store+caches per session, AB/BA config interleave, an LLC-sized eviction sweep as think-time, and the amortization/memory ledger (open/warm time, RSS) as first-class output. Profiles and first results (LDBC SF0.1, 3 reps, idle machine): - repl_explorer (typos/drift/DML — where mistakes happen naturally): tc-on 84–97 s vs tc-off 117–133 s per session (~30% saved) and ~50 MiB lower peak RSS; 6 mistakes rejected in microseconds. - sdk_app (reviewed templates × 300 requests — errors impossible by construction; measures the TAX of leaving typechecking on): ~10 us/request; session totals statistically identical. Answer: it's free, leave it on. - one_shot_reject_lazy (cost-to-first-verdict with the setup each path actually needs): tc-on 0.095 ms at 395 MiB vs tc-off 920 ms at 537 MiB — the runtime must build its ~140 MiB index to learn what the schema already knew. - one_shot_reject_eager: the "runtime rejects in 11 us" number from hot-loop benches requires ~2.2 s of session setup the typechecker never needed; per-session the two verdicts cost the same order. - write_heavy (DML→query cycles, lazy runtime): tc-on 8–10 ms vs tc-off 4.6–4.7 s (~500×) — each mutation invalidates the runtime's index (~0.9 s rebuild) while the typechecker's schema-side caches rebuild lazily in microseconds. Documented limitations: think-time eviction cannot model frequency scaling/predictor decay of real pauses; repl_explorer's drift/error mix is synthetic (no organic usage traces exist yet) and its knobs are meant to be swept. Co-Authored-By: Claude Fable 5 --- Cargo.toml | 5 + src/bin/scenario_bench.rs | 467 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 472 insertions(+) create mode 100644 src/bin/scenario_bench.rs diff --git a/Cargo.toml b/Cargo.toml index 1d63484..a9d1465 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -78,6 +78,11 @@ name = "ldbc_bench" path = "src/bin/ldbc_bench.rs" required-features = ["bench"] +[[bin]] +name = "scenario_bench" +path = "src/bin/scenario_bench.rs" +required-features = ["bench"] + [workspace] members = [".", "python", "node", "wasm"] # resolver = "2" computes features per-target instead of unifying them diff --git a/src/bin/scenario_bench.rs b/src/bin/scenario_bench.rs new file mode 100644 index 0000000..c833aab --- /dev/null +++ b/src/bin/scenario_bench.rs @@ -0,0 +1,467 @@ +//! scenario_bench — usage-scenario benchmark for the typechecker's +//! cost/benefit in the sessions users actually run, not in isolation. +//! +//! cargo run --release --features bench --bin scenario_bench -- \ +//! [path.gdb] [--scenario name] [--reps N] [--out file.csv] +//! +//! Motivation ("we cannot improve what we cannot measure"): the internal +//! bench measures the full pipeline per case, and `pattern_typecheck` +//! measures the checker in a hot loop — neither models a *session*: the +//! amortization horizon, shape-novelty distribution, human think-time +//! (which evicts hardware caches between queries), error mix, and DML +//! interleaving that decide what typechecking actually costs and saves +//! in consumption. This bench replays named session profiles end-to-end, +//! each in two configurations: +//! +//! * `tc-on` — compile with the typechecker against the active schema; +//! invalid queries are rejected, guaranteed-empty queries skip the +//! runtime (the REPL's behavior). +//! * `tc-off` — compile unchecked and always execute (the counterfactual). +//! +//! Profiles: +//! * `repl_explorer` — interactive session: shape drift, typos/errors, +//! a DML, one catastrophic-if-executed query; an LLC-sized eviction +//! sweep before every query stands in for human think-time. +//! (Limitation: the sweep cannot model frequency scaling or +//! predictor decay of real seconds-long pauses.) +//! * `sdk_app` — bindings/API usage: fixed reviewed templates executed +//! repeatedly with different literals (errors don't occur here by +//! construction; the question is the typechecker's TAX, not savings). +//! * `one_shot_*` — single-query sessions; cost-to-first-verdict with +//! the setup each path actually needs (eager vs lazy index warm). +//! * `write_heavy` — DML→query cycles under a lazy runtime: after every +//! mutation the runtime's TripleIndex is invalidated (~seconds to +//! rebuild at SF0.1) while the typechecker's schema-side caches +//! rebuild lazily in microseconds — the invalidation asymmetry. +//! +//! Fairness rules: fresh store + fresh caches per session (cold starts +//! are real); config order alternates AB/BA across repetitions so +//! machine drift cancels; setup time and RSS are first-class outputs +//! (the amortization/memory ledger), because per-query latency alone +//! hides what the runtime's fast paths cost up front. + +use std::fs::File; +use std::hint::black_box; +use std::io::Write; +use std::path::Path; +use std::time::Instant; + +use sysinfo::{Pid, ProcessRefreshKind, RefreshKind, System}; + +use frogql::parser; +use frogql::runtime::dm::run_dm; +use frogql::runtime::engine::Runtime; +use frogql::store::lazy::LazyGraphStore; +use frogql::syntax::statement::Statement; + +const DEFAULT_GDB: &str = "bench/data/ldbc-sf0.1.gdb"; +const EXEC_LIMIT: usize = 20; +const DEFAULT_REPS: usize = 3; +/// LLC on the dev box is 12 MiB; a 32 MiB sweep evicts it deterministically. +const EVICT_BYTES: usize = 32 * 1024 * 1024; + +#[derive(Clone, Copy, PartialEq)] +enum Cfg { + TcOn, + TcOff, +} + +impl Cfg { + fn name(self) -> &'static str { + match self { + Cfg::TcOn => "tc-on", + Cfg::TcOff => "tc-off", + } + } +} + +enum Step { + Query(String), + Dml(String), +} + +struct Scenario { + name: &'static str, + steps: Vec, + /// Evict hardware caches before every query (human think-time). + think: bool, + /// Warm the TripleIndex at open (as the REPL/bindings do today). + eager_warm: bool, +} + +fn repl_explorer() -> Scenario { + let q = |s: &str| Step::Query(s.to_string()); + Scenario { + name: "repl_explorer", + think: true, + eager_warm: true, + steps: vec![ + q("MATCH (p: Person) WHERE p.id = 933 RETURN p.firstName"), + // Typo'd label: schema-empty. tc-on rejects; tc-off scans. + q("MATCH (p: Persn) WHERE p.id = 933 RETURN p.firstName"), + q("MATCH (p: Person) WHERE p.id = 933 RETURN p.firstName"), + q("MATCH (p: Person)~[:knows]~(f: Person) WHERE p.id = 933 RETURN f.firstName"), + // Typo'd edge label. + q("MATCH (p: Person)~[:nows]~(f: Person) WHERE p.id = 933 RETURN f.firstName"), + q("MATCH (p: Person)~[:knows]~(f: Person) WHERE p.id = 933 RETURN f.firstName"), + // Unbound variable: invalid. tc-off silently returns 0 rows after work. + q("MATCH (p: Person)~[:knows]~(f: Person) WHERE q.id = 933 RETURN f.firstName"), + q("MATCH (p: Person)~[:knows]~(f: Person) WHERE p.id = 933 \ + RETURN f.firstName ORDER BY f.firstName LIMIT 10"), + // The guardrail case: schema-empty only at the chain's leaf — + // invisible to label-index pruning, catastrophic unchecked. + q( + "MATCH (a: Person)~[:knows]~(b: Person)~[:knows]~(c: Person)~[:knows]~(d: Wagumi) \ + RETURN a.id", + ), + q( + "MATCH (p: Person)~[:knows]~(f: Person)~[:knows]~(g: Person) \ + WHERE p.id = 933 RETURN g.firstName LIMIT 10", + ), + Step::Dml("INSERT (:Person {id: 999999901, firstName: 'Bench'})".to_string()), + // Post-DML: the runtime's TripleIndex was invalidated. + q("MATCH (p: Person) WHERE p.id = 933 RETURN p.firstName"), + q( + "MATCH (m: Comment)-[:hasCreator]->(p: Person) WHERE p.id = 933 \ + RETURN m.id LIMIT 10", + ), + q("MATCH (p: Person)~[:knows]~(f: Person) OPTIONAL MATCH \ + (f)<-[:hasCreator]-(c: Comment) RETURN p.id LIMIT 10"), + q( + "MATCH (p: Person)~[:knows]~(f: Person) WHERE EXISTS { MATCH \ + (f)<-[:hasCreator]-(c: Comment) } RETURN p.id LIMIT 10", + ), + // Type clash: firstName is a string. + q("MATCH (p: Person) WHERE p.firstName = 933 RETURN p.id"), + q( + "MATCH (m: Comment)-[:hasCreator]->(p: Person) WHERE p.id = 933 \ + RETURN m.id LIMIT 10", + ), + // Label-visible empty: the case the runtime rejects fast too. + q("MATCH (x: Wagumi) RETURN x.id"), + q("MATCH (p: Person)~[:knows]~(f: Person) WHERE p.id = 933 RETURN f.firstName"), + ], + } +} + +fn sdk_app() -> Scenario { + // Fixed, reviewed templates — errors cannot occur by construction. + // The app formats literals into the string per request, exactly like + // a REST handler would. + let templates = [ + "MATCH (p: Person)~[:knows]~(f: Person) WHERE p.id = {ID} RETURN f.firstName", + "MATCH (m: Comment)-[:hasCreator]->(p: Person) WHERE p.id = {ID} RETURN m.id LIMIT 10", + "MATCH (p: Person) WHERE p.id = {ID} RETURN p.firstName", + ]; + let ids = [933u64, 1129, 4194, 6597, 8333, 10995]; + let mut steps = Vec::new(); + for i in 0..100 { + for t in &templates { + let id = ids[i % ids.len()]; + steps.push(Step::Query(t.replace("{ID}", &id.to_string()))); + } + } + Scenario { + name: "sdk_app", + think: false, + eager_warm: true, + steps, + } +} + +fn one_shot(name: &'static str, query: &str, eager_warm: bool) -> Scenario { + Scenario { + name, + think: false, + eager_warm, + steps: vec![Step::Query(query.to_string())], + } +} + +fn write_heavy() -> Scenario { + // Lazy runtime: the index is built only if something executes. tc-on + // rejects every query in the cycle, so it never builds the index at + // all; tc-off rebuilds it after every DML just to discover 0 rows. + let mut steps = Vec::new(); + for i in 0..5 { + steps.push(Step::Dml(format!( + "INSERT (:Person {{id: 99999991{i}, firstName: 'Bench'}})" + ))); + steps.push(Step::Query( + "MATCH (a: Person)~[:knows]~(b: Person)-[:noSuchEdge]->(c: Person) RETURN a.id" + .to_string(), + )); + } + // One valid query at the end so both configs pay one real execution. + steps.push(Step::Query( + "MATCH (p: Person) WHERE p.id = 933 RETURN p.firstName".to_string(), + )); + Scenario { + name: "write_heavy", + think: false, + eager_warm: false, + steps, + } +} + +fn rss_mb(sys: &mut System) -> f64 { + let pid = Pid::from_u32(std::process::id()); + sys.refresh_process_specifics(pid, ProcessRefreshKind::new().with_memory()); + sys.process(pid) + .map(|p| p.memory() as f64 / (1024.0 * 1024.0)) + .unwrap_or(0.0) +} + +fn evict(buf: &mut [u8]) { + for i in (0..buf.len()).step_by(64) { + buf[i] = buf[i].wrapping_add(1); + } + black_box(&buf[buf.len() / 2]); +} + +struct SessionSummary { + open_ms: f64, + warm_ms: f64, + compile_ns_total: u128, + exec_ns_total: u128, + session_ns: u128, + rejected: usize, + executed: usize, + rss_after_mb: f64, +} + +#[allow(clippy::too_many_arguments)] +fn run_session( + gdb: &Path, + sc: &Scenario, + cfg: Cfg, + rep: usize, + sys: &mut System, + evict_buf: &mut [u8], + csv: &mut File, +) -> SessionSummary { + let t = Instant::now(); + let lazy = LazyGraphStore::open(gdb).unwrap_or_else(|e| panic!("open {gdb:?}: {e}")); + let open_ms = t.elapsed().as_secs_f64() * 1000.0; + let schema = lazy.catalog().active_schema(); + let rt = Runtime::new(&lazy); + let warm_ms = if sc.eager_warm { + let t = Instant::now(); + rt.warm_triple_index(); + t.elapsed().as_secs_f64() * 1000.0 + } else { + 0.0 + }; + + let mut sum = SessionSummary { + open_ms, + warm_ms, + compile_ns_total: 0, + exec_ns_total: 0, + session_ns: 0, + rejected: 0, + executed: 0, + rss_after_mb: 0.0, + }; + + let session_t = Instant::now(); + for (idx, step) in sc.steps.iter().enumerate() { + match step { + Step::Dml(stmt) => { + let t = Instant::now(); + match parser::parse_statement(stmt) { + Ok(Statement::DataModification(dm)) => { + // DEFAULT graph type is active → no G2000 validation. + run_dm(&lazy, &dm, None).unwrap_or_else(|e| panic!("dml failed: {e}")); + rt.invalidate_caches(); + } + other => panic!("expected DML statement, got {other:?}"), + } + let ns = t.elapsed().as_nanos(); + writeln!( + csv, + "{};{};{};{};dml;{ns};applied", + sc.name, + cfg.name(), + rep, + idx + ) + .unwrap(); + } + Step::Query(q) => { + if sc.think { + evict(evict_buf); + } + let (compile_ns, exec_ns, outcome) = match cfg { + Cfg::TcOn => { + let t = Instant::now(); + let compiled = frogql::compile_query_with_diagnostics_with(&schema, q); + let compile_ns = t.elapsed().as_nanos(); + match compiled { + Err(_) => (compile_ns, 0, "rejected_invalid"), + Ok(cr) if cr.guaranteed_empty => (compile_ns, 0, "rejected_empty"), + Ok(cr) => { + let t = Instant::now(); + black_box(rt.run_query(&cr.query, EXEC_LIMIT)); + (compile_ns, t.elapsed().as_nanos(), "executed") + } + } + } + Cfg::TcOff => { + let t = Instant::now(); + let compiled = frogql::compile_query_unchecked(q); + let compile_ns = t.elapsed().as_nanos(); + match compiled { + Err(_) => (compile_ns, 0, "parse_error"), + Ok(query) => { + let t = Instant::now(); + black_box(rt.run_query(&query, EXEC_LIMIT)); + (compile_ns, t.elapsed().as_nanos(), "executed") + } + } + } + }; + sum.compile_ns_total += compile_ns; + sum.exec_ns_total += exec_ns; + if outcome.starts_with("rejected") { + sum.rejected += 1; + } else if outcome == "executed" { + sum.executed += 1; + } + writeln!( + csv, + "{};{};{};{};compile;{compile_ns};{outcome}", + sc.name, + cfg.name(), + rep, + idx + ) + .unwrap(); + writeln!( + csv, + "{};{};{};{};exec;{exec_ns};{outcome}", + sc.name, + cfg.name(), + rep, + idx + ) + .unwrap(); + } + } + } + sum.session_ns = session_t.elapsed().as_nanos(); + sum.rss_after_mb = rss_mb(sys); + sum +} + +fn main() { + let mut gdb: Option = None; + let mut only: Option = None; + let mut reps = DEFAULT_REPS; + let mut out = "scenario_bench_results.csv".to_string(); + let mut args = std::env::args().skip(1); + while let Some(a) = args.next() { + match a.as_str() { + "--scenario" => only = args.next(), + "--reps" => reps = args.next().and_then(|v| v.parse().ok()).unwrap_or(reps), + "--out" => out = args.next().unwrap_or(out), + other if gdb.is_none() => gdb = Some(other.to_string()), + other => panic!("unexpected argument: {other}"), + } + } + let gdb = gdb.unwrap_or_else(|| DEFAULT_GDB.to_string()); + let gdb = Path::new(&gdb); + + let scenarios: Vec = vec![ + repl_explorer(), + sdk_app(), + one_shot( + "one_shot_valid_eager", + "MATCH (p: Person)~[:knows]~(f: Person) WHERE p.id = 933 RETURN f.firstName", + true, + ), + one_shot( + "one_shot_reject_eager", + "MATCH (a: Person)~[:knows]~(b: Person)-[:noSuchEdge]->(c: Person) RETURN a.id", + true, + ), + one_shot( + "one_shot_reject_lazy", + "MATCH (a: Person)~[:knows]~(b: Person)-[:noSuchEdge]->(c: Person) RETURN a.id", + false, + ), + write_heavy(), + ]; + + let mut sys = System::new_with_specifics( + RefreshKind::new().with_processes(ProcessRefreshKind::new().with_memory()), + ); + let rss_baseline = rss_mb(&mut sys); + let mut evict_buf = vec![0u8; EVICT_BYTES]; + let mut csv = File::create(&out).unwrap_or_else(|e| panic!("create {out}: {e}")); + writeln!(csv, "scenario;config;rep;step;phase;ns;outcome").unwrap(); + + eprintln!("scenario_bench: db={gdb:?} reps={reps} rss_baseline={rss_baseline:.1} MiB"); + eprintln!( + "\n{:<24}{:<7}{:>4}{:>10}{:>10}{:>12}{:>12}{:>12}{:>6}{:>6}{:>10}", + "scenario", + "config", + "rep", + "open_ms", + "warm_ms", + "compile_ms", + "exec_ms", + "session_ms", + "rej", + "exec", + "rss_MiB" + ); + eprintln!("{}", "-".repeat(113)); + + for sc in &scenarios { + if let Some(f) = &only { + if !sc.name.contains(f.as_str()) { + continue; + } + } + for rep in 0..reps { + // AB/BA interleave so machine drift cancels across configs. + let order = if rep % 2 == 0 { + [Cfg::TcOn, Cfg::TcOff] + } else { + [Cfg::TcOff, Cfg::TcOn] + }; + for cfg in order { + let s = run_session(gdb, sc, cfg, rep, &mut sys, &mut evict_buf, &mut csv); + eprintln!( + "{:<24}{:<7}{:>4}{:>10.1}{:>10.1}{:>12.3}{:>12.1}{:>12.1}{:>6}{:>6}{:>10.1}", + sc.name, + cfg.name(), + rep, + s.open_ms, + s.warm_ms, + s.compile_ns_total as f64 / 1e6, + s.exec_ns_total as f64 / 1e6, + s.session_ns as f64 / 1e6, + s.rejected, + s.executed, + s.rss_after_mb + ); + writeln!( + csv, + "{};{};{};-1;session_total;{};open_ms={:.1} warm_ms={:.1} rss_mb={:.1}", + sc.name, + cfg.name(), + rep, + s.session_ns, + s.open_ms, + s.warm_ms, + s.rss_after_mb + ) + .unwrap(); + } + } + } + eprintln!("{}", "-".repeat(113)); + eprintln!("wrote {out}"); +} From e1e7a4daac572f50227e3938eeeb6ab695a6bcac Mon Sep 17 00:00:00 2001 From: Felipe705x Date: Fri, 24 Jul 2026 02:51:01 -0400 Subject: [PATCH 2/2] bench(typing): full-surface case family for pattern_typecheck MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The legacy `bench` case set is patterns-only by design — it mirrors the Python reference checker, which supported only path patterns (right shape, not the queries users type). New `full` category exercises RETURN typing, ORDER BY (column and alias), aggregates, and the multi-MATCH collapse path; the legacy set stays intact for its comparison purpose. Verdicts all correct; full queries typecheck at 3.4–13 µs on the LDBC schema (1.1–4.1× parse cost). Co-Authored-By: Claude Fable 5 --- src/bin/pattern_typecheck.rs | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/bin/pattern_typecheck.rs b/src/bin/pattern_typecheck.rs index 0fc79bd..6b508d3 100644 --- a/src/bin/pattern_typecheck.rs +++ b/src/bin/pattern_typecheck.rs @@ -360,6 +360,38 @@ fn main() { "valid".into(), "(p: Person)~[:knows]~{2,4}(f: Person)".into(), )); + // Full-surface cases: the legacy `bench` set is patterns-only by + // design (it mirrors the Python reference checker, which supported + // only path patterns — right shape, but not the queries users type). + // These exercise RETURN typing, ORDER BY, aggregates, and the + // multi-MATCH collapse path. + cases.push(( + "full_return_orderby".into(), + "full".into(), + "valid".into(), + "MATCH (p: Person)~[:knows]~(f: Person) WHERE p.id = 933 \ + RETURN f.firstName ORDER BY f.firstName LIMIT 10" + .into(), + )); + cases.push(( + "full_count".into(), + "full".into(), + "valid".into(), + "MATCH (m: Comment)-[:hasCreator]->(p: Person) RETURN COUNT(m)".into(), + )); + cases.push(( + "full_multi_match".into(), + "full".into(), + "valid".into(), + "MATCH (p: Person)~[:knows]~(f: Person) MATCH (f)~[:knows]~(g: Person) RETURN g.id".into(), + )); + cases.push(( + "full_alias_orderby".into(), + "full".into(), + "valid".into(), + "MATCH (p: Person)~[:knows]~(f: Person) RETURN f.firstName AS name ORDER BY name LIMIT 5" + .into(), + )); cases.push(( "subq_exists".into(), "subq".into(),