From baf50a6876ce4d9028a3c58a8894b53fb771ecad Mon Sep 17 00:00:00 2001 From: rysweet Date: Mon, 27 Jul 2026 12:34:06 +0000 Subject: [PATCH] feat(cognitive-memory): durable fact_snapshot_dedup_ratio hygiene self-metric MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a durable, regressable graph-memory *hygiene* self-metric that turns the already-computed snapshot-revision dedup counts into a comparable metrics.jsonl series, mirroring the existing `fact_provenance_coverage` precedent. Snapshot facts (goal-board snapshots and other caller-key streams) are revisioned: each new revision SUPERSEDES the prior, and `prune_superseded` (controlled forgetting) reclaims the archived tail. `graph_stats()` already reports `snapshot_facts_total` (live + not-yet-pruned superseded revisions) and `distinct_snapshot_caller_keys` (distinct logical streams), but those raw counts were only visible via `simard memory stats` / OTel gauges — never as a durable series. Their ratio is the average *liveness* of the snapshot layer (distinct / total, in (0,1], higher is healthier); it falls when superseded revisions accumulate faster than pruning reclaims them — exactly the monotonic-growth failure controlled forgetting exists to prevent. - `cognitive_memory::metrics::snapshot_dedup_ratio` — pure ratio fn: None on an empty snapshot layer (skip, don't drag the series to a misleading 0.0), defensive clamp so a miscount can't exceed 1.0. - `record_snapshot_dedup_ratio_metric` — best-effort, cfg!(test)-skipped per-cycle emitter, sits beside `record_provenance_coverage_metric`. - Wired into the daemon per-cycle sweep from the SAME `graph_stats()` snapshot (no extra store read). - Unit tests (ratio math, None-for-empty, 0.0-when-keyless, clamp, no-op-under-test) + an end-to-end assertion in the seeded `bin_simard_memory_cli` integration test tying the metric to the operator-visible `memory stats --json` counts so they can't silently diverge. - Docs: telemetry-metrics + cognitive-memory-provenance observability sections. Pure observation: never changes memory state. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/reference/cognitive-memory-provenance.md | 52 ++++++++ docs/reference/telemetry-metrics.md | 9 ++ src/cognitive_memory/metrics.rs | 126 ++++++++++++++++++ src/operator_commands_ooda/daemon/mod.rs | 12 ++ tests/bin_simard_memory_cli.rs | 18 +++ 5 files changed, 217 insertions(+) diff --git a/docs/reference/cognitive-memory-provenance.md b/docs/reference/cognitive-memory-provenance.md index c5397c3a3..b2c4fff03 100644 --- a/docs/reference/cognitive-memory-provenance.md +++ b/docs/reference/cognitive-memory-provenance.md @@ -450,6 +450,56 @@ self-metrics they sit alongside. --- +## Observability: snapshot-dedup-hygiene self-metric + +Grounding coverage watches whether facts enter the graph *connected*; a sibling +self-metric watches whether the **snapshot layer stays lean**. Snapshot facts +(those written under a stable caller/dedup key — goal-board snapshots and the +like) are revisioned: each new revision `SUPERSEDES` the prior one, and +`prune_superseded` (controlled forgetting) reclaims the archived revisions over +time. `graph_stats()` already reports two raw counts for this layer: + +| Field | Meaning | +|---|---| +| `snapshot_facts_total` | every snapshot revision still held (live + not-yet-pruned superseded) | +| `distinct_snapshot_caller_keys` | distinct logical snapshot streams behind them | + +The durable `fact_snapshot_dedup_ratio` self-metric is the hygiene *health* +signal derived from them. Once per OODA cycle the daemon — from the **same** +`graph_stats()` snapshot it already collects for the OTel edge gauges and the +grounding-coverage metric — emits one sample to the `metrics.jsonl` series: + +``` +fact_snapshot_dedup_ratio = distinct_snapshot_caller_keys / snapshot_facts_total +``` + +- **What it measures.** The average *liveness* of the snapshot layer, in + `(0.0, 1.0]`. `1.0` means every stream holds a single live revision; the value + falls toward `0` as superseded revisions pile up (its inverse — total / + distinct — is the mean revisions retained per stream). +- **Why it matters.** That accumulation is exactly the monotonic-growth failure + controlled forgetting exists to prevent: if `prune_superseded` stops keeping + pace, archived revisions bloat semantic memory. Previously that was visible + only as the raw `graph_stats()` counts, never as a durable, comparable, + regressable series — so a *ratio* (store-size-independent) makes a pruning + regression raise the same gym-history signal every other cognition self-metric + (`fact_provenance_coverage`, `recall_precision_at_k`, `distill_fact_yield`) + does. +- **Undefined on an empty snapshot layer.** When the store holds zero snapshot + facts, the ratio is *undefined* and **no** sample is emitted (skip rather than + drag the series to a misleading `0.0`), mirroring the `fact_provenance_coverage` + convention. `distinct_snapshot_caller_keys` is clamped to `snapshot_facts_total` + defensively so a miscount can never yield a ratio above `1.0`. The emitter is + best-effort — a metrics-write failure is logged, never propagated — and pure + observation: it never changes memory state. + +The scoring is a pure function +(`cognitive_memory::metrics::snapshot_dedup_ratio`) with the per-cycle emitter +(`record_snapshot_dedup_ratio_metric`) beside `record_provenance_coverage_metric`, +so both graph-memory hygiene self-metrics sit together. + +--- + ## Testing The feature is covered by a TDD round-trip test (in @@ -488,6 +538,8 @@ cargo test memory_consolidation `facts_with_provenance` / `facts_total` snapshot the coverage metric reads. - `src/cognitive_memory/metrics.rs` — `provenance_coverage()` (pure ratio) and `record_provenance_coverage_metric()` (per-cycle `fact_provenance_coverage` + emitter), plus the sibling `snapshot_dedup_ratio()` / + `record_snapshot_dedup_ratio_metric()` (per-cycle `fact_snapshot_dedup_ratio` emitter), plus the `GraphStats` snapshot type in `src/memory_cognitive.rs`. - `src/operator_commands_ooda/daemon/mod.rs` — the per-cycle sweep that reads `graph_stats()` for the OTel edge gauges and emits the coverage self-metric diff --git a/docs/reference/telemetry-metrics.md b/docs/reference/telemetry-metrics.md index 6bf2f6da6..f96461b2a 100644 --- a/docs/reference/telemetry-metrics.md +++ b/docs/reference/telemetry-metrics.md @@ -187,6 +187,15 @@ gauges the section renders `absent`, never a fabricated zero. > comparable and regressable, not just a raw count. See > [Cognitive-memory provenance § Observability](./cognitive-memory-provenance.md#observability-grounding-coverage-self-metric). +> **Snapshot hygiene.** A sibling durable **`fact_snapshot_dedup_ratio`** +> self-metric emits, from the same per-cycle `graph_stats()` snapshot, the +> average *liveness* of the snapshot layer +> (`distinct_snapshot_caller_keys / snapshot_facts_total` ∈ `(0, 1]`, higher is +> healthier). It falls when superseded snapshot revisions accumulate faster than +> controlled forgetting (`prune_superseded`) reclaims them, turning a pruning +> regression into a regressable series rather than a raw count. See +> [Cognitive-memory provenance § Snapshot dedup hygiene](./cognitive-memory-provenance.md#observability-snapshot-dedup-hygiene-self-metric). + ### LLM usage — `simard.llm.*` Mirrored from `cost_tracking` (the ledger format is unchanged; these are diff --git a/src/cognitive_memory/metrics.rs b/src/cognitive_memory/metrics.rs index d2cf3c01c..4eb67d607 100644 --- a/src/cognitive_memory/metrics.rs +++ b/src/cognitive_memory/metrics.rs @@ -246,6 +246,92 @@ pub fn record_provenance_coverage_metric(facts_with_provenance: u64, facts_total } } +// ─────────────────────── graph-memory snapshot dedup hygiene ──────────────── +// +// A graph-memory *hygiene* signal complementary to grounding coverage above. +// Snapshot facts (those stored under a stable caller/dedup key — goal-board +// snapshots and the like) are revisioned: each new revision SUPERSEDES the +// prior one, and `prune_superseded` (controlled forgetting) reclaims the +// archived revisions over time. `snapshot_facts_total` counts every snapshot +// revision the store still holds (live + not-yet-pruned superseded); +// `distinct_snapshot_caller_keys` counts the distinct logical streams behind +// them. Their ratio is the average *liveness* of the snapshot layer: 1.0 when +// every stream holds exactly one revision, falling toward 0 as superseded +// revisions accumulate faster than pruning reclaims them. That accumulation is +// exactly the monotonic-growth failure controlled forgetting exists to prevent, +// and — like grounding coverage — it was previously visible only as raw +// `graph_stats()` counts, never as a durable, comparable, regressable +// `metrics.jsonl` series. Emitting the *ratio* makes a pruning/hygiene +// regression raise the same gym-history signal every other self-metric does. + +/// Durable self-metric name for snapshot-layer dedup hygiene, emitted to +/// `metrics.jsonl` once per OODA cycle by [`record_snapshot_dedup_ratio_metric`]. +pub const FACT_SNAPSHOT_DEDUP_RATIO_METRIC: &str = "fact_snapshot_dedup_ratio"; + +/// Average liveness of the snapshot layer (`distinct_snapshot_caller_keys / +/// snapshot_facts_total`), in `(0.0, 1.0]`. Higher is healthier: `1.0` means +/// every snapshot stream holds a single live revision; a value approaching `0` +/// means superseded revisions have piled up (the inverse — total / distinct — +/// is the mean revisions retained per stream). +/// +/// Returns `None` (undefined, **not** `0.0`) when the store holds no snapshot +/// facts, so a store with an empty snapshot layer contributes no misleading +/// `0.0` sample — the same "skip rather than drag the series to zero" +/// convention [`provenance_coverage`] and [`precision_at_k`] use. +/// `distinct_snapshot_caller_keys` is clamped to `snapshot_facts_total` +/// defensively (a stream always has ≥1 revision, so distinct ≤ total holds), so +/// a backend that miscounts can never yield a ratio above `1.0`. +pub fn snapshot_dedup_ratio( + distinct_snapshot_caller_keys: u64, + snapshot_facts_total: u64, +) -> Option { + if snapshot_facts_total == 0 { + return None; + } + let distinct = distinct_snapshot_caller_keys.min(snapshot_facts_total); + Some(distinct as f64 / snapshot_facts_total as f64) +} + +/// Emit ONE durable [`FACT_SNAPSHOT_DEDUP_RATIO_METRIC`] sample (the snapshot +/// liveness ratio over the current `graph_stats()` snapshot) to `metrics.jsonl`. +/// +/// Called once per OODA cycle by the daemon metric sweep from the same block +/// that already reads `graph_stats()` for the OpenTelemetry edge gauges and +/// [`record_provenance_coverage_metric`], so it adds no extra store read. A +/// snapshot-shaped metric (store state, not a per-cycle accumulator). +/// +/// No-op when the store holds no snapshot facts (undefined ratio — see +/// [`snapshot_dedup_ratio`]), so the series carries signal only. Best-effort: a +/// metrics-write failure is logged, never propagated. Skipped under +/// `cfg!(test)` so unit tests never append to the operator's real +/// `~/.simard/metrics/metrics.jsonl`. +pub fn record_snapshot_dedup_ratio_metric( + distinct_snapshot_caller_keys: u64, + snapshot_facts_total: u64, +) { + let Some(ratio) = snapshot_dedup_ratio(distinct_snapshot_caller_keys, snapshot_facts_total) + else { + return; + }; + if cfg!(test) { + return; + } + let context = serde_json::json!({ + "snapshot_facts_total": snapshot_facts_total, + "distinct_snapshot_caller_keys": distinct_snapshot_caller_keys, + }) + .to_string(); + if let Err(e) = + crate::self_metrics::record_metric(FACT_SNAPSHOT_DEDUP_RATIO_METRIC, ratio, &context) + { + tracing::warn!( + target: "simard::memory", + error = %e, + "failed to record fact_snapshot_dedup_ratio metric (memory unaffected)", + ); + } +} + #[cfg(test)] mod tests { use super::*; @@ -389,4 +475,44 @@ mod tests { record_provenance_coverage_metric(3, 4); record_provenance_coverage_metric(0, 0); } + + // ── graph-memory snapshot dedup hygiene: pure math ────────────────────── + + #[test] + fn snapshot_dedup_ratio_is_distinct_streams_over_total_revisions() { + // Two streams, four revisions retained → each stream averages two + // revisions → liveness 0.5. One-revision-per-stream is a healthy 1.0. + assert_eq!(snapshot_dedup_ratio(2, 4), Some(0.5)); + assert_eq!(snapshot_dedup_ratio(4, 4), Some(1.0)); + assert_eq!(snapshot_dedup_ratio(1, 8), Some(0.125)); + // Snapshot facts present but none carry a grouping key → distinct 0 over + // a nonzero total is a real, maximally-unhealthy 0.0 (emit it), NOT the + // undefined None reserved for an empty snapshot layer. + assert_eq!(snapshot_dedup_ratio(0, 4), Some(0.0)); + } + + #[test] + fn snapshot_dedup_ratio_is_none_for_an_empty_snapshot_layer() { + // No snapshot facts → undefined ratio (skip, do NOT emit a misleading + // 0.0), matching the provenance_coverage / precision@k convention. A + // nonzero distinct count with a zero denominator is still None. + assert_eq!(snapshot_dedup_ratio(0, 0), None); + assert_eq!(snapshot_dedup_ratio(3, 0), None); + } + + #[test] + fn snapshot_dedup_ratio_clamps_overcount_to_one() { + // distinct ≤ total always holds (a stream has ≥1 revision); a backend + // that miscounts must never yield a ratio above 1.0. + assert_eq!(snapshot_dedup_ratio(9, 4), Some(1.0)); + } + + #[test] + fn record_snapshot_dedup_ratio_metric_is_a_no_op_under_test() { + // Guards the operator's real metrics.jsonl: the emitter is cfg!(test)- + // skipped, and an empty snapshot layer is a no-op regardless. Neither + // call may panic or touch global state. + record_snapshot_dedup_ratio_metric(2, 4); + record_snapshot_dedup_ratio_metric(0, 0); + } } diff --git a/src/operator_commands_ooda/daemon/mod.rs b/src/operator_commands_ooda/daemon/mod.rs index 3772c95b0..aa43b0027 100644 --- a/src/operator_commands_ooda/daemon/mod.rs +++ b/src/operator_commands_ooda/daemon/mod.rs @@ -1687,6 +1687,18 @@ pub fn run_ooda_daemon( g.facts_with_provenance, g.facts_total, ); + // Emit the durable snapshot-layer dedup-hygiene + // self-metric from the SAME snapshot: average liveness + // of the snapshot layer (distinct streams / retained + // revisions). Turns a pruning/hygiene regression — + // superseded snapshot revisions accumulating faster than + // controlled forgetting reclaims them — into a + // comparable, regressable `metrics.jsonl` series. + // Best-effort; no-op on an empty snapshot layer. + crate::cognitive_memory::metrics::record_snapshot_dedup_ratio_metric( + g.distinct_snapshot_caller_keys, + g.snapshot_facts_total, + ); } // Flush the metrics snapshot with the per-cycle enrichment // rollup section attached (issue #2942) so the dashboard's diff --git a/tests/bin_simard_memory_cli.rs b/tests/bin_simard_memory_cli.rs index a195523bc..a6d9cd942 100644 --- a/tests/bin_simard_memory_cli.rs +++ b/tests/bin_simard_memory_cli.rs @@ -252,6 +252,24 @@ fn stats_shows_edges_and_dedup_section_via_direct_open() { Some(1), "the snapshot caller key must be grouped: {report}" ); + // The durable `fact_snapshot_dedup_ratio` self-metric (emitted per OODA + // cycle by the daemon) is defined over *exactly* these operator-visible + // counts, so its inputs can never silently diverge from what `memory stats` + // renders. One stream holding one revision is a healthy liveness of 1.0. + let snapshot_facts = report["snapshot_dedup"]["snapshot_facts"] + .as_u64() + .expect("snapshot_facts must be numeric"); + let distinct_caller_keys = report["snapshot_dedup"]["distinct_caller_keys"] + .as_u64() + .expect("distinct_caller_keys must be numeric"); + assert_eq!( + simard::cognitive_memory::metrics::snapshot_dedup_ratio( + distinct_caller_keys, + snapshot_facts, + ), + Some(1.0), + "fact_snapshot_dedup_ratio must derive from the operator-visible counts: {report}" + ); assert!( report.get("edges_note").is_none(), "direct open must compute the edges, not note them: {report}"