From 55f9dd286f353cc301aff4b985a08051d2bfcb6f Mon Sep 17 00:00:00 2001 From: Eric Moore Date: Fri, 28 Aug 2026 18:45:13 -0500 Subject: [PATCH 1/4] =?UTF-8?q?A=20node=20vouches=20about=20itself=20on=20?= =?UTF-8?q?infra:attest=20=E2=80=94=20config:load,=20self-attested,=20expi?= =?UTF-8?q?ring?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Third attempt at the same thing, and the first one that reads the constitution rather than reasoning around it. THE AUTHORITY WAS ALREADY CONFERRED. Every node holds `infra:attest` — "vouch as the delegator's infrastructure" — from its owner-binding, and the canonical holds it from the accord's charter (CC 4.4.3.8: "a root serves and vouches, or it is inert"). It is conferred SO THE NODE CAN VOUCH ABOUT THE DELEGATOR'S INFRASTRUCTURE, and "I am at capacity" is a fact about that infrastructure only the infrastructure can observe. This is not a node exceeding its grant; it is the grant working. Clause E says it directly: a node "may sign its own refusal without new authority." THE DIMENSION WAS ALREADY RESERVED. CC 3.1 `config:{scope}` — "a node's declared operating configuration, published as an auditable record rather than inferred from behaviour… only ever about the emitting node" — with open scope vocabulary. CC 3.4.5 gives its emitter rule, SELF-OR-OWNER: "a node's running configuration is a self-report; a third-party assertion of what you are running is a rumour." So `config:load` joins `admission` / `replication` / `moderation` / `transport`, and attesting == attested == subject by construction. WHAT THE TWO WRONG VERSIONS GOT WRONG. The first would have automated `OP_SELF_SHED`, an `admin_action:` — refused at the write door without a `delegation_id`, which a node does not have. The second authored a HardCaseEvent with an expiry, which is on NO replication plane, so "the artifact a peer reads to stop offering" could never reach a peer. I then over-corrected to "a node has no standing at all", which CC 4.2.1 does not say: that section governs `mesh_config`, the ROOT's plane, which changes what OTHER nodes carry. Two planes, two authors, one reason — the root relieves ACROSS nodes; a node vouches ABOUT itself. This module never touches the first. WHY IT IS AN ATTESTATION. Because attestations replicate — that is why they are the universal primitive. `expires_at` is a real signed instant persist enforces at read (`expires_at IS NULL OR expires_at > NOW()`), so a stale declaration is not merely ignorable, it is invisible. The lifetime encodes the authority; there is no lift because recovery is the lift. ONLY THIS NODE'S OWN STALL MAY RENEW. `/proc/pressure/*` describes the whole box, and a noisy neighbour there would make this node ATTEST, UNDER ITS OWN SIGNATURE, that it is shedding — a false self-report replicated to every peer. Only a `Cgroup`-scoped, `full`-line reading at `degradation`'s own degrade bound counts; host scope, a kernel with no `full` line, or PSI unavailable renews nothing. The loop acts on the FRESH probe result, never the registry, because the probe deliberately preserves a stale warning when PSI vanishes (no-evidence discipline) and a registry read would renew a "short-lived" row forever. Under the NODE key, not the engine's: on an agent-carrying node those differ, and this is a statement about the infrastructure. A gap named, not closed: persist gates `mesh_config:` at admission but does NOT enforce CC 3.4.5 self-or-owner on `config:*`. This producer honours it by construction; the substrate check is asked for upstream. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01BSa6L2MEDAXmoJuBVUZNVg --- src/compose.rs | 11 ++ src/lib.rs | 2 + src/load_shed.rs | 301 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 314 insertions(+) create mode 100644 src/load_shed.rs diff --git a/src/compose.rs b/src/compose.rs index a6e7c7f2..ae2246ef 100644 --- a/src/compose.rs +++ b/src/compose.rs @@ -1545,6 +1545,17 @@ pub async fn serve_with_adapter(cfg: ServerConfig, adapter: Arc) -> // the previous node's observer id and counts (Codex, PR #502). crate::mesh_status::invalidate(); + // The node vouches about ITSELF on `infra:attest` — a `config:load` + // self-attestation (CC 3.1, CC 3.4.5 self-or-owner), short-lived, replicated. + // Under the NODE key, not the engine's: on an agent-carrying node those differ + // and this is a statement about the infrastructure. It declares; enforcement + // stays cooperative. It is not `mesh_config` — that plane is the root's. + crate::compose_status::phase("load_observer"); + let load_observer_key = crate::node_key::wire_identity() + .map(str::to_owned) + .unwrap_or_else(|| cfg.key_id.clone()); + let _load_observer_join = crate::load_shed::spawn(Arc::clone(&engine), load_observer_key); + crate::compose_status::phase("retention_loop"); let (retention_sd_tx, retention_sd_rx) = watch::channel(false); let retention_join = { diff --git a/src/lib.rs b/src/lib.rs index 49576064..fb0a1ad6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -293,6 +293,8 @@ pub mod key_standing; /// bidirectional replication with an out-of-group peer (Node B / `ciris-status`). /// Public so the integration test (`tests/peer_replication.rs`) can drive the /// admission + consent-emit logic directly. +/// A node's own `config:load` self-attestation, short-lived and replicated. +pub mod load_shed; pub mod location; /// Repetition-collapsing log layer — "event X occurred Y times in past Z". diff --git a/src/load_shed.rs b/src/load_shed.rs new file mode 100644 index 00000000..eced33eb --- /dev/null +++ b/src/load_shed.rs @@ -0,0 +1,301 @@ +//! **A node's own `config:load` attestation — a self-report, short-lived, replicated.** +//! +//! `degradation` measures contention and raises `resource.cpu_stall`; nothing +//! consumed it. CIRISServer#501 was a node that measured its own starvation, raised +//! the warning correctly, and went dark anyway — detection with no consequence. +//! +//! # The authority, read from the constitution +//! +//! Every node holds `infra:attest` — *"vouch as the delegator's infrastructure"* — +//! from its owner-binding, and the canonical holds it from the accord's charter +//! (`[infra:attest, infra:serve, infra:store, infra:transport]`; CC 4.4.3.8: *"a +//! root serves and vouches, or it is inert"*). It is conferred **so the node can +//! vouch about the delegator's infrastructure**, and "I am at capacity" is a fact +//! about that infrastructure only the infrastructure can observe. +//! +//! CC 3.1 already reserves the dimension: **`config:{scope}`** — *"a node's +//! declared operating configuration, published as an auditable record rather than +//! inferred from behaviour… only ever about the emitting node."* And CC 3.4.5 gives +//! its emitter rule: **self-or-owner** — `attesting_key_id` MUST be +//! `attested_key_id` or its live owner. *"A node's running configuration is a +//! self-report; a third-party assertion of what you are running is a rumour."* +//! +//! So this is not a node exceeding its grant. It is the grant working. +//! +//! # What this is NOT — and what it was, wrongly, twice +//! +//! It is **not a `mesh_config` value.** That plane (CC 4.2.1) is the trust root's, +//! on the delegation plane, and it governs what OTHER nodes carry. A node MUST NOT +//! author there, and this module never does. Two planes, two authors, one reason: +//! the root relieves ACROSS nodes; a node vouches ABOUT itself. +//! +//! An earlier version authored a `HardCaseEvent` instead. Hard-case events are on +//! no replication plane, so *"the artifact a peer reads to stop offering"* could +//! never reach a peer. An attestation replicates; that is why it is the primitive. +//! +//! # Why short-lived is what makes it safe +//! +//! The objection to a node speaking for itself was never that it speaks; it is +//! that it would accumulate STANDING nobody granted and keep it after the +//! condition passed. `expires_at` — a real, signed, admission-enforced instant +//! (persist#598) — makes the lifetime encode the authority: the row cannot outlive +//! the observation that produced it. Renewal is re-observation, not inheritance. +//! +//! There is deliberately no lift. A durable self-declaration would need a person to +//! clear it, and that person may never come. Here **recovery is the lift**: a node +//! that recovers stops renewing, a node that dies stops renewing, a node that was +//! wrong stops renewing. Nothing has to notice. +//! +//! # A gap this module does not close +//! +//! persist gates `mesh_config:` at admission but does NOT enforce CC 3.4.5's +//! self-or-owner rule on `config:*` — the emitter rule is producer obligation only +//! today. This producer honours it by construction (attesting == attested), and the +//! substrate-side check is asked for upstream so a third party's `config:load` +//! about this node is refused rather than merely unfashionable. + +use std::sync::Arc; +use std::time::Duration; + +use ciris_persist::federation::envelope::paths; +use ciris_persist::federation::types::{attestation_type, cohort_scope}; +use ciris_persist::prelude::Engine; + +/// The `config:{scope}` leaf for carried load. Open vocabulary per CC 4.5.1.1; +/// `admission` / `replication` / `moderation` / `transport` are the canonical +/// scopes and this joins them as the node's self-report of what it is carrying. +pub const DIMENSION: &str = "config:load"; + +/// The one value this attestation carries: what the node is doing about it. +pub const STATE_SHEDDING: &str = "shedding"; + +/// How often the node measures itself. +pub const OBSERVE_INTERVAL: Duration = Duration::from_secs(60); + +/// How long one attestation stands. +/// +/// Three observation intervals: long enough that a single slow tick does not drop +/// a live row, short enough that a node which stops observing stops declaring +/// within minutes. Too short and the node flaps; too long and a recovered node +/// keeps telling the mesh it is shedding — the failure only a person can clear. +pub const TTL_SECS: i64 = 180; + +/// Build the self-report the node will sign. +/// +/// `attesting == attested == subject`: CC 3.4.5 self-or-owner, satisfied on the +/// producer side by construction. `witness_relation: self` is the CC 2.1 marker +/// consumers weight by — this is a self-attestation and says so on the wire. +/// +/// No `asserted_at` here (CIRISServer#402 / persist#598): the stamp writes the +/// signed instants once, truncated to the substrate's resolution. +#[must_use] +pub fn spec(node_key_id: &str, expires_at: chrono::DateTime) -> crate::attest::Spec { + let envelope = serde_json::json!({ + (paths::DIMENSION): DIMENSION, + "attesting_key_id": node_key_id, + "subject_key_ids": [node_key_id], + "score": 1.0, + "cohort_scope": cohort_scope::FEDERATION, + "witness_relation": "self", + "state": STATE_SHEDDING, + "reason": "measured sustained CPU or IO contention", + }); + crate::attest::Spec::new(attestation_type::SCORES, cohort_scope::FEDERATION, envelope) + .about(node_key_id) + .expiring(Some(expires_at)) + .weighing(Some(1.0)) +} + +/// Emit one `config:load` self-attestation, signed by the engine. +/// +/// # Errors +/// Stamp, sign, or put failure. +pub async fn emit(engine: &Engine, node_key_id: &str) -> anyhow::Result { + let now = chrono::Utc::now(); + let expires_at = now + chrono::Duration::seconds(TTL_SECS); + let row = crate::attest::Emit::stamp(node_key_id, spec(node_key_id, expires_at)) + .map_err(|e| anyhow::anyhow!("stamp config:load for {node_key_id}: {e}"))? + .sign_and_assemble(crate::attest::KeySigner::Engine(engine)) + .await + .map_err(|e| anyhow::anyhow!("sign config:load as {node_key_id}: {e}"))?; + crate::attest::put(engine, row) + .await + .map_err(|e| anyhow::anyhow!("put config:load for {node_key_id}: {e}")) +} + +/// Spawn the observe-and-attest loop. +pub fn spawn(engine: Arc, node_key_id: String) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + let mut interval = tokio::time::interval(OBSERVE_INTERVAL); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + tracing::info!( + dimension = DIMENSION, + observe_secs = OBSERVE_INTERVAL.as_secs(), + ttl_secs = TTL_SECS, + "config:load observer started (self-attested on infra:attest, expires on its own)" + ); + loop { + interval.tick().await; + + // PROBE, and act on the FRESH reading — not the registry. `probe_contention` + // is otherwise only called from `/v1/health`, so a node with no health + // traffic would observe nothing. And when PSI becomes unavailable + // mid-run, the probe deliberately preserves the old warning rather than + // clearing it (no-evidence discipline), so reading the registry here + // would renew a "short-lived" row forever off a stale entry (Codex, + // PR #503). A fresh `Measured` stall is the only thing that renews. + let (cpu, io) = crate::degradation::probe_contention(); + if !stalled(&cpu) && !stalled(&io) { + continue; + } + + match emit(&engine, &node_key_id).await { + Ok(id) => tracing::warn!( + attestation_id = %id, + ttl_secs = TTL_SECS, + "config:load = shedding — self-attested; expires on its own; peers may \ + read it to stop offering" + ), + Err(e) => tracing::warn!(error = %e, "config:load attestation failed — continuing"), + } + } + }) +} + +/// A FRESH, CGROUP-SCOPED, degrading stall — never a stale registry entry, and +/// never a host reading. +/// +/// Scope is the load-bearing check. `/proc/pressure/*` describes the whole box, +/// and a noisy neighbour there would make this node **attest, under its own +/// signature, that it is shedding** — a self-report about a condition it is not +/// suffering, replicated to every peer (PR #483 review: host readings are never +/// escalated to a statement about this node). Only this process's own cgroup may +/// speak for this process. +/// +/// The threshold is `degradation`'s own `full` bound, not a restated literal, so +/// this attests exactly when the node would report itself degraded and never on a +/// looser standard of its own. +fn stalled(p: &crate::degradation::Pressure) -> bool { + use crate::degradation::{Pressure, PressureScope, PRESSURE_DEGRADE_FULL_PCT}; + match p { + Pressure::Measured { + scope: PressureScope::Cgroup, + full_avg10: Some(full), + .. + } => *full >= PRESSURE_DEGRADE_FULL_PCT, + // A host reading, a kernel with no `full` line, or no PSI at all: none of + // these is evidence about THIS node's own stall, so none may renew. + _ => false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const NODE: &str = "node-under-test"; + + fn at(secs: i64) -> chrono::DateTime { + chrono::DateTime::from_timestamp(1_700_000_000 + secs, 0).expect("ts") + } + + /// CC 3.4.5 self-or-owner, satisfied by construction: attesting, attested and + /// subject are all the node. A third party's `config:load` about this node is + /// a rumour, and this producer can never write one. + #[test] + fn it_is_a_self_report_on_every_axis() { + let s = spec(NODE, at(180)); + assert_eq!(s.attested_key_id.as_deref(), Some(NODE)); + assert_eq!(s.subject_key_ids, vec![NODE.to_string()]); + assert_eq!(s.envelope["attesting_key_id"], NODE); + assert_eq!(s.envelope["witness_relation"], "self"); + } + + /// The row carries its own expiry — a real signed instant persist enforces at + /// read time (`expires_at IS NULL OR expires_at > NOW()`), so a stale + /// declaration is not merely ignorable, it is invisible. + #[test] + fn it_expires() { + let s = spec(NODE, at(180)); + assert_eq!(s.expires_at, Some(at(180))); + } + + /// It is `config:{scope}` — the CC 3.1 dimension reserved for exactly this, not + /// a new family and not `mesh_config:` (the root's plane, which a node MUST NOT + /// author). + #[test] + fn it_is_on_the_config_dimension_and_never_mesh_config() { + let s = spec(NODE, at(180)); + assert_eq!(s.envelope[paths::DIMENSION], DIMENSION); + assert!(DIMENSION.starts_with("config:")); + assert!(!DIMENSION.starts_with("mesh_config:")); + } + + /// Federation scope, so it actually replicates — the reason an attestation and + /// not a hard-case event. + #[test] + fn it_replicates_at_federation_scope() { + let s = spec(NODE, at(180)); + assert_eq!(s.cohort_scope, cohort_scope::FEDERATION); + } + + /// No `asserted_at` in the envelope: the stamp owns the signed instants + /// (persist#598), and a hand-written one lands with nanoseconds postgres cannot + /// store, refusing the put. Every write on this path failed on v31 for that. + #[test] + fn it_leaves_the_signed_instants_to_the_stamp() { + let s = spec(NODE, at(180)); + assert!(s.envelope.get(paths::ASSERTED_AT).is_none()); + assert!(s.envelope.get(paths::EXPIRES_AT).is_none()); + } + + /// Only a fresh, cgroup-scoped, degrading reading renews. Everything else — + /// host scope, sub-threshold, no `full` line, PSI unavailable — is not evidence + /// about THIS node and must not put this node's signature on "shedding". + #[test] + fn only_this_nodes_own_degrading_stall_counts() { + use crate::degradation::{Pressure, PressureScope, PRESSURE_DEGRADE_FULL_PCT}; + let own = |full: f64| Pressure::Measured { + scope: PressureScope::Cgroup, + some_avg10: 99.0, + full_avg10: Some(full), + }; + assert!( + stalled(&own(PRESSURE_DEGRADE_FULL_PCT)), + "at the degrade bound" + ); + assert!(stalled(&own(99.0))); + assert!(!stalled(&own(PRESSURE_DEGRADE_FULL_PCT - 0.01)), "under it"); + + // The noisy-neighbour case: the BOX is thrashing, this node is fine. A + // self-attestation here would be a false statement under our signature. + assert!(!stalled(&Pressure::Measured { + scope: PressureScope::Host, + some_avg10: 99.0, + full_avg10: Some(99.0), + })); + // No `full` line (CPU on many kernels): absent is not zero and not evidence. + assert!(!stalled(&Pressure::Measured { + scope: PressureScope::Cgroup, + some_avg10: 99.0, + full_avg10: None, + })); + assert!(!stalled(&Pressure::Unavailable { + reason: "no PSI".into() + })); + } + + /// TTL outlasts a missed tick but not a recovery. + #[test] + fn the_ttl_is_a_small_multiple_of_the_observation_interval() { + let interval = OBSERVE_INTERVAL.as_secs() as i64; + assert!( + TTL_SECS > interval, + "one slow tick must not drop a live row" + ); + assert!( + TTL_SECS <= interval * 5, + "a recovered node must stop declaring in minutes" + ); + } +} From 675d448879882b4f4b1e7b1b0961f9662495ef67 Mon Sep 17 00:00:00 2001 From: Eric Moore Date: Fri, 28 Aug 2026 18:50:02 -0500 Subject: [PATCH 2/4] 0.5.195: a node vouches about itself on infra:attest Substrate unchanged: persist v38.6.0 / edge v18.11.0 / verify v13.6.1. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01BSa6L2MEDAXmoJuBVUZNVg --- Cargo.lock | 2 +- Cargo.toml | 2 +- evidence/CIRISServer.cc_impl.tsv | 16 ++++++++-------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index df4be004..7e7b9ad4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1102,7 +1102,7 @@ dependencies = [ [[package]] name = "ciris-server" -version = "0.5.194" +version = "0.5.195" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 6c2cc54a..33139c4b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ciris-server" -version = "0.5.194" # 0.5.194 = THE READ API STOPS STARVING ON A SMALL HOST. CIRISServer#501: the canonical binds :4243 and never accepts — confirmed live INSIDE the container netns (`LISTEN Recv-Q 3`, curl timing out at 0 bytes, ONE tokio worker at 99.9% while every other thread idles). NOT a .193 regression: 0.5.192 reproduces it, so the rollback did not restore service. TWO causes, and either alone leaves it broken. (1) `prime_trusted_peers` was awaited INLINE at compose.rs:711 while `read_api_bind` sits ~180 lines below, so TIME-TO-SERVING SCALED WITH THE DIRECTORY — it walks `list_all_transport_destinations()`, 11,034 rows on the canonical against 748 keys, because a row is written per binding and pruned NEVER. 33s of boot with the socket bound and nothing accepted. Now spawned (announces root peers progressively anyway, so no later stage needs every peer rooted), yielding per batch (tokio reschedules only at an await that PENDS, and every await in that loop can resolve ready, so it owned its worker start to finish), and no longer paying two `knows_peer` probes plus an INFO line per peer purely to fill log fields. (2) A bare `#[tokio::main]` sizes to core count = TWO workers on 2 vCPU; one blocking task and HTTP is unschedulable. `MIN_WORKER_THREADS = 4` as a FLOOR, never a cap — a 32-core canonical still gets 32. THE FLOOR ALONE WOULD NOT HAVE CLOSED IT: more workers shortens the window, but the read API still could not bind until the prime returned. BACKPRESSURE ON THE ONE TABLE THAT IS ACTUALLY AN OBSERVATION LOG: persist v38.6.0 ships `prune_announced_peers_not_seen_since` and it had no caller here; now wired into the hourly retention pass at 30 days (vs the corpus's 90 — losing an announcement costs a round-trip, losing a trace costs the trace), capped at 2,000 rows/pass so hygiene cannot become the long task that starves accept. `transport_destinations` is deliberately NOT pruned even though it is the bigger table and the one boot prime walks: the measurement killed the premise (Codex, PR #502). ALL 11,034 rows have `last_seen_at IS NULL` — there are no observations in it, only assertions — and a 30-day prune would delete 5,408 rows, 100% of them never-observed asserted routes. Its keep-predicate spares SIGNED rows, but `TransportDestination` has no signature field at all, so every locally-asserted row is unsigned and prunable — including the accord-quorum-authorized canonical address update, which an explicit-hash canonical cannot announce back. Bounding it needs a predicate persist cannot express (stale OBSERVATION vs old assertion), asked for upstream; the outage is closed by taking the prime OFF the critical path, which makes table size cost background work rather than availability. KEYS ARE NOT PRUNED EITHER — a federation_keys row is an identity ANCHOR and deleting it makes every row it authored unverifiable; keys need EXPIRY, unreachable until CIRISVerify#267 lands. ALSO FIXED: .193's mesh-status refresh loop ran `storage_summary()` every 30s, whose SQLite path is `SUM(pgsize) FROM dbstat` — a walk of EVERY PAGE of a 1.3 GiB database, inline on a worker. A module whose own doc called an uncached read an amplification vector then scheduled it on a timer. Now lazy, single-flight (one walk in flight ever; the flag clears on Drop so a panic cannot wedge the surface stale), off the runtime via spawn_blocking, TTL 60s→900s pinned by test. Substrate unchanged: persist v38.6.0 / edge v18.11.0 / verify v13.6.1. +version = "0.5.195" # 0.5.195 = A NODE VOUCHES ABOUT ITSELF ON infra:attest. Third attempt at one thing, the first that reads the constitution instead of reasoning around it. THE AUTHORITY WAS ALREADY CONFERRED: every node holds `infra:attest` ("vouch as the delegator's infrastructure") from its owner-binding, the canonical from the accord's charter, conferred SO THE NODE CAN VOUCH ABOUT THE DELEGATOR'S INFRASTRUCTURE — and "I am at capacity" is the fact only the infrastructure can observe. Clause E: a node "may sign its own refusal without new authority". THE DIMENSION WAS ALREADY RESERVED: CC 3.1 `config:{scope}` ("a node's declared operating configuration… only ever about the emitting node"), with CC 3.4.5's emitter rule SELF-OR-OWNER ("a self-report; a third-party assertion of what you are running is a rumour"). So `config:load` joins admission/replication/moderation/transport, attesting == attested == subject by construction, `witness_relation: self`, federation scope so it REPLICATES, `expires_at` a real signed instant persist enforces at read. THE TWO WRONG VERSIONS: automating `OP_SELF_SHED` (an admin_action, refused at the write door without a delegation_id a node does not have); then a HardCaseEvent with an expiry (on NO replication plane, so "the artifact a peer reads" could never reach a peer); then an over-correction to "a node has no standing at all", which CC 4.2.1 does not say — that section governs `mesh_config`, the ROOT's plane, which changes what OTHER nodes carry. TWO PLANES, TWO AUTHORS, ONE REASON: the root relieves ACROSS nodes; a node vouches ABOUT itself. ONLY THIS NODE'S OWN STALL MAY RENEW: `/proc/pressure` describes the whole box, and a noisy neighbour would make this node ATTEST UNDER ITS OWN SIGNATURE that it is shedding — a false self-report to every peer. Only a Cgroup-scoped `full`-line reading at degradation's own bound counts; host scope, no `full` line, or PSI unavailable renews nothing, and the loop acts on the FRESH probe rather than the registry (which deliberately preserves a stale warning when PSI vanishes). TTL = 3 × 60s; recovery is the lift. Under the NODE key, not the engine's. Filed: CIRISConstitution#96 (RC4: two planes, `config:load` as a named scope), CIRISPersist#777 (the root's `load.ceiling` key), CIRISPersist#778 (CC 3.4.5 self-or-owner is NOT enforced at admission for config:* — anyone can attest what a node is running). Substrate unchanged: persist v38.6.0 / edge v18.11.0 / verify v13.6.1. edition = "2021" # MSRV floor is set by the substrate: ciris-verify v5.2.0 requires 1.86 # (persist v6.0.1 is 1.83). Build with the higher of the two. diff --git a/evidence/CIRISServer.cc_impl.tsv b/evidence/CIRISServer.cc_impl.tsv index 8af81a2b..3ebe1ab6 100644 --- a/evidence/CIRISServer.cc_impl.tsv +++ b/evidence/CIRISServer.cc_impl.tsv @@ -10,14 +10,14 @@ # # ─── RESOLVED ────────────────────────────────────────────────────────────────── decimal_id claim_id repo path#symbol crate@version -3.1 CLM-nsproc-dimension CIRISServer src/compose_policy.rs#polarity_for ciris-server@v0.5.194 -4.4.2 CLM-nsproc-aggregation-policy CIRISServer src/compose_policy.rs#polarity_for ciris-server@v0.5.194 -4.4.3.4.3 CLM-nsproc-cohort-scope CIRISServer src/graph_config.rs#set_config ciris-server@v0.5.194 -4.2.2.1 CLM-nsproc-attestation-evidence CIRISServer src/hardware_attestation.rs#admit_hardware_class_against_root ciris-server@v0.5.194 -2.6.6.1 CLM-location CIRISServer src/location.rs#mint_location_proof ciris-server@v0.5.194 -2.6.6 CLM-canonicalization-cell CIRISServer src/location.rs#mint_location_proof ciris-server@v0.5.194 -3.4.5 CLM-capacity-score CIRISServer src/scorer.rs#score_and_emit ciris-server@v0.5.194 -3.3.7 CLM-consent-directed CIRISServer src/peer.rs#emit_replication_consent ciris-server@v0.5.194 +3.1 CLM-nsproc-dimension CIRISServer src/compose_policy.rs#polarity_for ciris-server@v0.5.195 +4.4.2 CLM-nsproc-aggregation-policy CIRISServer src/compose_policy.rs#polarity_for ciris-server@v0.5.195 +4.4.3.4.3 CLM-nsproc-cohort-scope CIRISServer src/graph_config.rs#set_config ciris-server@v0.5.195 +4.2.2.1 CLM-nsproc-attestation-evidence CIRISServer src/hardware_attestation.rs#admit_hardware_class_against_root ciris-server@v0.5.195 +2.6.6.1 CLM-location CIRISServer src/location.rs#mint_location_proof ciris-server@v0.5.195 +2.6.6 CLM-canonicalization-cell CIRISServer src/location.rs#mint_location_proof ciris-server@v0.5.195 +3.4.5 CLM-capacity-score CIRISServer src/scorer.rs#score_and_emit ciris-server@v0.5.195 +3.3.7 CLM-consent-directed CIRISServer src/peer.rs#emit_replication_consent ciris-server@v0.5.195 # # ─── DECLARED GAPS ───────────────────────────────────────────────────────────── # Controls the Constitution attributes to this repo that are NOT enforced here. From 165ec59d8cb819968b0c9ed55d6df56e0d31a6bb Mon Sep 17 00:00:00 2001 From: Eric Moore Date: Fri, 28 Aug 2026 19:02:56 -0500 Subject: [PATCH 3/4] Codex on #504: two of five would have refused every row at admission MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five findings, all correct. Two are the kind where the feature does not work AT ALL and every local test passes, because the test never reaches the put door. THE DIMENSION HAD NO VERSION SEGMENT. persist's `DimensionAdmissionPolicy` refuses any `scores` dimension without `:vN` (`MissingVersionSegment`, T3), and every existing producer carries one — `config:v1`, `consent:replication:v1`. So `config:load` would have been rejected on every emission while the loop logged "attestation failed" once a minute and nothing ever reached a peer. Now `config:load:v1`, and a test pins the segment. THE SIGNER WAS THE ACTOR'S ON A SPLIT NODE. `wire_identity()` returns the separately-minted node key on an agent-carrying node, but `KeySigner::Engine` signs with the actor's engine key. A row stamped as the node and signed by the actor fails signature verification against the node's registered public key — every emission rejected at admission on exactly the topology this exists for. `NodeIdentityResolution.signer` was built for precisely this ("author rows AS the node while the engine signs as the actor") and was being dropped after boot. It is now carried into the observer as `NodePen::Node`; unsplit nodes — the binary and the wheel's bare-agent path, where the engine key IS the node key — use `NodePen::Engine`. THE OBSERVER NEVER STARTED ON THE BARE-AGENT PATH. The wheel topology enters through `federation_delivery::start_and_hold` and never runs `serve_with_adapter`, so agent-carrying wheel nodes would never self-report under contention. It now starts from both entry points, and on the delivery path it is OWNED by `DeliveryController` — shutdown wired into its `Drop` beside the reconcile loop's — rather than spawned and detached. THE OBSERVER OUTLIVED THE NODE. Dropping a `JoinHandle` only detaches; on the in-process `shutdown_node()` restart the old observer kept its `Arc`, kept probing, kept renewing rows against the old node, and every restart added one more. It now has a shutdown channel and is joined at teardown beside the retention loop. THE PROBE RACED HEALTH COLLECTION. `/v1/health` holds `COLLECT_LOCK` across its probes and final `verdict()` precisely so one response cannot pair resource readings from one instant with warning state from another. A background probe without the lock could land between those two steps and reintroduce that. The observer now probes under the same lock; `probe_contention` is synchronous file IO, so the std mutex is held for microseconds and never across an await. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01BSa6L2MEDAXmoJuBVUZNVg --- src/compose.rs | 16 +++- src/federation_delivery.rs | 31 ++++++- src/load_shed.rs | 168 ++++++++++++++++++++++++++++++++----- 3 files changed, 189 insertions(+), 26 deletions(-) diff --git a/src/compose.rs b/src/compose.rs index ae2246ef..dc6080e2 100644 --- a/src/compose.rs +++ b/src/compose.rs @@ -1551,10 +1551,14 @@ pub async fn serve_with_adapter(cfg: ServerConfig, adapter: Arc) -> // and this is a statement about the infrastructure. It declares; enforcement // stays cooperative. It is not `mesh_config` — that plane is the root's. crate::compose_status::phase("load_observer"); - let load_observer_key = crate::node_key::wire_identity() - .map(str::to_owned) - .unwrap_or_else(|| cfg.key_id.clone()); - let _load_observer_join = crate::load_shed::spawn(Arc::clone(&engine), load_observer_key); + let (load_observer_sd, load_observer_join) = crate::load_shed::spawn( + Arc::clone(&engine), + // The NODE's pen. On a split node the engine signs as the actor, and a row + // stamped as the node but signed by the actor fails verification at + // admission — `node_resolution.signer` exists for exactly this. + crate::load_shed::NodePen::from_resolution(&node_resolution), + node_resolution.node_key_id.clone(), + ); crate::compose_status::phase("retention_loop"); let (retention_sd_tx, retention_sd_rx) = watch::channel(false); @@ -1644,6 +1648,10 @@ pub async fn serve_with_adapter(cfg: ServerConfig, adapter: Arc) -> // first would race its shutdown branch against a `changed()` error break. let _ = retention_sd_tx.send(true); let _ = retention_join.await; + // The observer holds an Arc; on the in-process restart path a detached + // one would keep renewing attestations against the OLD node. + let _ = load_observer_sd.send(true); + let _ = load_observer_join.await; // Tear down the mesh-config consumer refresh loop (CIRISServer#365). Its // readers (the read API, the ingest router) are already gone by here. let _ = mesh_config_sd_tx.send(true); diff --git a/src/federation_delivery.rs b/src/federation_delivery.rs index 63bf440b..c8787c25 100644 --- a/src/federation_delivery.rs +++ b/src/federation_delivery.rs @@ -89,11 +89,19 @@ pub struct DeliveryController { reconcile_shutdown: watch::Sender, /// The reconcile-loop task handle (held so the task is not detached-and-lost). _reconcile_join: tokio::task::JoinHandle<()>, + /// Shutdown signal for the config:load observer, owned here so it lives and + /// dies with delivery rather than being detached at spawn. + load_observer_shutdown: Option>, + _load_observer_join: Option>, } impl Drop for DeliveryController { fn drop(&mut self) { let _ = self.reconcile_shutdown.send(true); + // The observer dies with delivery, never detached (Codex, PR #504). + if let Some(tx) = &self.load_observer_shutdown { + let _ = tx.send(true); + } } } @@ -258,12 +266,31 @@ pub fn start_and_hold(cadence_seconds: Option, announce_logger: bool) -> Re // absent there). Before `engine`/`edge` are moved into the controller below. #[cfg(feature = "test-anchor")] rt.block_on(crate::test_bless::maybe_test_bless_delivery_self(&engine))?; - let controller = rt.block_on(run_federation_delivery( + + // The bare-agent topology never runs `serve_with_adapter`, so the config:load + // observer must start HERE too or agent-carrying wheel nodes never self-report + // under contention (Codex, PR #504). No identity split on this path — the + // engine key IS the node key — so the engine is the right pen. Held on the + // delivery runtime; it lives exactly as long as delivery does. + let load_observer_key = crate::node_key::wire_identity() + .map(str::to_owned) + .unwrap_or_else(|| edge.signer_key_id().to_string()); + let (load_observer_sd, load_observer_join) = { + let _g = rt.enter(); + crate::load_shed::spawn( + Arc::clone(&engine), + crate::load_shed::NodePen::Engine, + load_observer_key, + ) + }; + let mut controller = rt.block_on(run_federation_delivery( engine, edge, cadence_seconds, announce_logger, ))?; + controller.load_observer_shutdown = Some(load_observer_sd); + controller._load_observer_join = Some(load_observer_join); let controller = Arc::new(controller); Ok(hold(rt, controller)) } @@ -1504,6 +1531,8 @@ pub async fn run_federation_delivery( canonical_targets: admitted_targets, reconcile_shutdown, _reconcile_join: reconcile_join, + load_observer_shutdown: None, + _load_observer_join: None, }) } diff --git a/src/load_shed.rs b/src/load_shed.rs index eced33eb..943f9489 100644 --- a/src/load_shed.rs +++ b/src/load_shed.rs @@ -61,10 +61,18 @@ use ciris_persist::federation::envelope::paths; use ciris_persist::federation::types::{attestation_type, cohort_scope}; use ciris_persist::prelude::Engine; -/// The `config:{scope}` leaf for carried load. Open vocabulary per CC 4.5.1.1; -/// `admission` / `replication` / `moderation` / `transport` are the canonical -/// scopes and this joins them as the node's self-report of what it is carrying. -pub const DIMENSION: &str = "config:load"; +/// The `config:{scope}` leaf for carried load, VERSIONED. Open vocabulary per CC +/// 4.5.1.1; `admission` / `replication` / `moderation` / `transport` are the +/// canonical scopes and this joins them as the node's self-report of what it is +/// carrying. +/// +/// The `:v1` is not decoration: persist's `DimensionAdmissionPolicy` refuses any +/// `scores` dimension without a `:vN` segment (`MissingVersionSegment`, T3). The +/// first cut of this module omitted it, and every emission would have been +/// rejected at the put door while the loop logged "attestation failed" once a +/// minute — the row could never have reached a peer (Codex, PR #504). Existing +/// producers all carry one (`config:v1`, `consent:replication:v1`). +pub const DIMENSION: &str = "config:load:v1"; /// The one value this attestation carries: what the node is doing about it. pub const STATE_SHEDDING: &str = "shedding"; @@ -106,16 +114,55 @@ pub fn spec(node_key_id: &str, expires_at: chrono::DateTime) -> cra .weighing(Some(1.0)) } -/// Emit one `config:load` self-attestation, signed by the engine. +/// Who signs the node's self-report. +/// +/// On a split node (an agent-carrying node whose configured key was an actor, +/// `node_key::resolve_node_identity`) the wire identity is a separately minted +/// node key and the Engine signs as the ACTOR. A row stamped with the node key as +/// `attesting_key_id` but signed by the actor's pen fails signature verification +/// against the node's registered public key — every emission rejected at +/// admission (Codex, PR #504). `NodeIdentityResolution.signer` exists for exactly +/// this ("author rows AS the node while the engine signs as the actor") and is +/// carried here rather than dropped after boot. +/// +/// On an unsplit node — the standalone binary, and the wheel's bare-agent path — +/// the engine key IS the node key, and the engine is the right pen. +#[derive(Clone)] +pub enum NodePen { + /// Engine key == node key. Sign with the engine. + Engine, + /// A split happened; sign as the node with its own signer. + Node(Arc), +} + +impl NodePen { + /// Pick the pen from a boot's identity resolution. + #[must_use] + pub fn from_resolution(r: &crate::node_key::NodeIdentityResolution) -> Self { + match &r.signer { + Some(s) => Self::Node(Arc::clone(s)), + None => Self::Engine, + } + } + + fn signer<'a>(&'a self, engine: &'a Engine) -> crate::attest::KeySigner<'a> { + match self { + Self::Engine => crate::attest::KeySigner::Engine(engine), + Self::Node(s) => crate::attest::KeySigner::Local(s), + } + } +} + +/// Emit one `config:load` self-attestation, signed with the node's own pen. /// /// # Errors /// Stamp, sign, or put failure. -pub async fn emit(engine: &Engine, node_key_id: &str) -> anyhow::Result { +pub async fn emit(engine: &Engine, pen: &NodePen, node_key_id: &str) -> anyhow::Result { let now = chrono::Utc::now(); let expires_at = now + chrono::Duration::seconds(TTL_SECS); let row = crate::attest::Emit::stamp(node_key_id, spec(node_key_id, expires_at)) .map_err(|e| anyhow::anyhow!("stamp config:load for {node_key_id}: {e}"))? - .sign_and_assemble(crate::attest::KeySigner::Engine(engine)) + .sign_and_assemble(pen.signer(engine)) .await .map_err(|e| anyhow::anyhow!("sign config:load as {node_key_id}: {e}"))?; crate::attest::put(engine, row) @@ -124,32 +171,52 @@ pub async fn emit(engine: &Engine, node_key_id: &str) -> anyhow::Result } /// Spawn the observe-and-attest loop. -pub fn spawn(engine: Arc, node_key_id: String) -> tokio::task::JoinHandle<()> { - tokio::spawn(async move { +/// +/// Returns the shutdown sender and the join handle, so composition can stop it +/// the way it stops the retention and config loops. Dropping the handle would +/// only DETACH the task: on the supported in-process `shutdown_node()` restart the +/// old observer would keep its `Arc`, keep probing, keep renewing +/// attestations against the old node, and every restart would add another one +/// (Codex, PR #504). +pub fn spawn( + engine: Arc, + pen: NodePen, + node_key_id: String, +) -> ( + tokio::sync::watch::Sender, + tokio::task::JoinHandle<()>, +) { + let (shutdown_tx, mut shutdown_rx) = tokio::sync::watch::channel(false); + let join = tokio::spawn(async move { let mut interval = tokio::time::interval(OBSERVE_INTERVAL); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); tracing::info!( dimension = DIMENSION, observe_secs = OBSERVE_INTERVAL.as_secs(), ttl_secs = TTL_SECS, + pen = match pen { + NodePen::Engine => "engine", + NodePen::Node(_) => "node (split)", + }, "config:load observer started (self-attested on infra:attest, expires on its own)" ); loop { - interval.tick().await; + tokio::select! { + _ = interval.tick() => {} + changed = shutdown_rx.changed() => { + if changed.is_err() || *shutdown_rx.borrow() { + tracing::info!("config:load observer shutting down"); + return; + } + continue; + } + } - // PROBE, and act on the FRESH reading — not the registry. `probe_contention` - // is otherwise only called from `/v1/health`, so a node with no health - // traffic would observe nothing. And when PSI becomes unavailable - // mid-run, the probe deliberately preserves the old warning rather than - // clearing it (no-evidence discipline), so reading the registry here - // would renew a "short-lived" row forever off a stale entry (Codex, - // PR #503). A fresh `Measured` stall is the only thing that renews. - let (cpu, io) = crate::degradation::probe_contention(); - if !stalled(&cpu) && !stalled(&io) { + if !own_stall_now() { continue; } - match emit(&engine, &node_key_id).await { + match emit(&engine, &pen, &node_key_id).await { Ok(id) => tracing::warn!( attestation_id = %id, ttl_secs = TTL_SECS, @@ -159,7 +226,30 @@ pub fn spawn(engine: Arc, node_key_id: String) -> tokio::task::JoinHandl Err(e) => tracing::warn!(error = %e, "config:load attestation failed — continuing"), } } - }) + }); + (shutdown_tx, join) +} + +/// Probe, under the same lock `/v1/health` holds, and judge the FRESH reading. +/// +/// PROBE rather than read: `probe_contention` is otherwise only called from +/// `/v1/health`, so a node with no health traffic would observe nothing — and when +/// PSI vanishes mid-run the probe deliberately preserves the old warning rather +/// than clearing it (no-evidence discipline), so a registry read would renew a +/// "short-lived" row forever off a stale entry. +/// +/// UNDER `COLLECT_LOCK`: the health route holds it across its probes and its final +/// `verdict()` precisely so one response cannot pair resource readings from one +/// instant with warning state from another. A probe here without the lock could +/// land between those two steps and put that inconsistency back (Codex, PR #504). +/// `probe_contention` is synchronous file IO, so the `std` mutex is held for +/// microseconds and never across an await. +fn own_stall_now() -> bool { + let _collect = crate::degradation::COLLECT_LOCK + .lock() + .unwrap_or_else(|e| e.into_inner()); + let (cpu, io) = crate::degradation::probe_contention(); + stalled(&cpu) || stalled(&io) } /// A FRESH, CGROUP-SCOPED, degrading stall — never a stale registry entry, and @@ -231,6 +321,42 @@ mod tests { assert!(!DIMENSION.starts_with("mesh_config:")); } + /// The dimension carries a `:vN` segment. Without one persist refuses the row + /// at admission (`MissingVersionSegment`) and the observer logs a failure once + /// a minute while nothing ever reaches a peer — the first cut of this module. + #[test] + fn the_dimension_is_versioned_or_persist_refuses_it() { + let last = DIMENSION.rsplit(':').next().unwrap_or(""); + assert!( + last.len() >= 2 + && last.starts_with('v') + && last[1..].chars().all(|c| c.is_ascii_digit()), + "{DIMENSION} must end in :vN" + ); + } + + /// On a split node the row must be signed by the NODE's pen, not the engine's: + /// stamped as the node and signed by the actor fails verification at admission. + /// On an unsplit node the engine is the node, and the engine is right. + #[test] + fn the_pen_follows_the_split() { + use crate::node_key::{IdentityVerdict, NodeIdentityResolution}; + let unsplit = NodeIdentityResolution { + node_key_id: NODE.into(), + split_from: None, + verdict: IdentityVerdict::SubstrateOnly, + signer: None, + }; + assert!(matches!( + NodePen::from_resolution(&unsplit), + NodePen::Engine + )); + // A split resolution carries a signer; the pen must be that signer. (A + // real LocalSigner needs key material, so this asserts the mapping's shape + // on the `None` half and leaves the `Some` half to the compose path.) + assert!(unsplit.signer.is_none()); + } + /// Federation scope, so it actually replicates — the reason an attestation and /// not a hard-case event. #[test] From 7de999a46c30249d6edfa59c7c2989e79a1f925e Mon Sep 17 00:00:00 2001 From: Eric Moore Date: Fri, 28 Aug 2026 19:23:01 -0500 Subject: [PATCH 4/4] =?UTF-8?q?The=20put=20door=20in=20a=20test,=20and=20t?= =?UTF-8?q?hree=20more=20Codex=20P1s=20=E2=80=94=20one=20correcting=20me?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit THE TEST FIRST, because it is why the rest were findable at all. `tests/config_load_reaches_the_put_door.rs` runs the producer's row through a REAL `put_attestation` on a real engine, in both topologies, and asserts a peer can screen what it receives. Every unit test in `load_shed` was green while the row was unadmittable in production; the rule this encodes is that A PRODUCER IS NOT TESTED UNTIL ITS ROW HAS BEEN THROUGH THE PUT DOOR. One case deliberately emits the unversioned dimension the first cut shipped and asserts persist REFUSES it, so the defect that passed every local test now fails loudly. THE WHEEL PATH SPLITS TOO — I told the operator it did not, and that was wrong. `provision_node_identity` mints a node key and sets BOTH `set_actor_identity` and `set_wire_identity`, leaving the engine signing as the ACTOR. So the embedded observer was stamping rows as the node and signing them with the actor's pen: rejected at admission, once a minute, on exactly the topology this feature exists for. The signer was already being created and thrown away (`let (_signer, ...)`); it is now RETAINED at provisioning and read back by the delivery path. CONFIDENCE WAS MISSING FROM THE ENVELOPE. `compose_policy::Composer::screen` refuses an envelope without it (`MalformedEnvelope("confidence")`) and deliberately does not default to 1.0 — so the row would have been admitted, replicated, and contributed NOTHING to the verdict a peer uses to decide whether to stop offering work. Admission is not the last gate, and the whole point of replicating this is that a peer can act on it. Now asserted against the consumer, not just the producer. A STALLED NODE WAS MINTING A PERMANENT ROW EVERY MINUTE. Expiry makes a row inactive; it does not remove it, and the retention loop has deletion levers for traces and audit entries only — an attestation-heavy store is a documented unenforceable disk case. One continuously stalled node would add 1,440 permanent rows a day to its own store AND every peer's, worsening the storage pressure that can trigger this loop in the first place. Renewal is now spaced to just inside the TTL: a live declaration continuously, at ~1/3 the rows. Detection stays at 60s because noticing quickly still matters — it is the WRITING that is throttled. Only a successful emission moves the renewal clock, or a transient refusal would suppress renewal for a whole gap and the node would go quiet while still struggling. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01BSa6L2MEDAXmoJuBVUZNVg --- src/federation_delivery.rs | 19 +- src/load_shed.rs | 74 +++++- src/node_key.rs | 25 +- tests/config_load_reaches_the_put_door.rs | 267 ++++++++++++++++++++++ 4 files changed, 372 insertions(+), 13 deletions(-) create mode 100644 tests/config_load_reaches_the_put_door.rs diff --git a/src/federation_delivery.rs b/src/federation_delivery.rs index c8787c25..47413a7c 100644 --- a/src/federation_delivery.rs +++ b/src/federation_delivery.rs @@ -272,16 +272,23 @@ pub fn start_and_hold(cadence_seconds: Option, announce_logger: bool) -> Re // under contention (Codex, PR #504). No identity split on this path — the // engine key IS the node key — so the engine is the right pen. Held on the // delivery runtime; it lives exactly as long as delivery does. + // THE WHEEL PATH SPLITS TOO. `provision_node_identity` mints a node key and + // sets both `set_actor_identity` and `set_wire_identity`, leaving the engine + // signing as the ACTOR — so a row stamped as the node and signed by the engine + // is rejected at admission (Codex, PR #504; the earlier claim that this path + // does not split was wrong). Provisioning RETAINS the node's pen for exactly + // this; when there is none, the engine is the node and is the right pen. + let engine_key = edge.signer_key_id().to_string(); let load_observer_key = crate::node_key::wire_identity() .map(str::to_owned) - .unwrap_or_else(|| edge.signer_key_id().to_string()); + .unwrap_or_else(|| engine_key.clone()); + let load_pen = match crate::node_key::node_pen_signer() { + Some(signer) => crate::load_shed::NodePen::Node(signer), + None => crate::load_shed::NodePen::Engine, + }; let (load_observer_sd, load_observer_join) = { let _g = rt.enter(); - crate::load_shed::spawn( - Arc::clone(&engine), - crate::load_shed::NodePen::Engine, - load_observer_key, - ) + crate::load_shed::spawn(Arc::clone(&engine), load_pen, load_observer_key) }; let mut controller = rt.block_on(run_federation_delivery( engine, diff --git a/src/load_shed.rs b/src/load_shed.rs index 943f9489..28c2b9a7 100644 --- a/src/load_shed.rs +++ b/src/load_shed.rs @@ -80,6 +80,32 @@ pub const STATE_SHEDDING: &str = "shedding"; /// How often the node measures itself. pub const OBSERVE_INTERVAL: Duration = Duration::from_secs(60); +/// Minimum gap between two emissions. +/// +/// Expiry makes a row INACTIVE; it does not remove it, and this repo's retention +/// loop has deletion levers for traces and audit entries only — an +/// attestation-heavy store is a documented unenforceable disk case. A row per +/// minute is 1,440 permanent rows a day, in this node's store AND every peer's, +/// which worsens the very storage pressure that can trigger this loop (Codex, +/// PR #504). +/// +/// So renewal is spaced to just inside the TTL rather than run at the observation +/// cadence: the node keeps a live declaration continuously, at ~1/3 the rows. The +/// observation interval stays short because DETECTING quickly still matters — it +/// is the *writing* that is throttled, not the looking. +pub const MIN_EMIT_GAP_SECS: i64 = TTL_SECS - OBSERVE_INTERVAL.as_secs() as i64; + +// Pinned at COMPILE time — these are constants, so a test could only restate what +// the compiler already knows (clippy `assertions_on_constants`). As const +// assertions they fail the BUILD the moment someone retunes a value past its bound. +// +// Spacing must exceed the observation tick or nothing is saved, and must stay +// under the TTL or the declaration lapses while the node is still stalled. +const _: () = assert!( + MIN_EMIT_GAP_SECS > 0, + "renewal spacing must be positive or every tick mints a permanent row" +); + /// How long one attestation stands. /// /// Three observation intervals: long enough that a single slow tick does not drop @@ -88,6 +114,11 @@ pub const OBSERVE_INTERVAL: Duration = Duration::from_secs(60); /// keeps telling the mesh it is shedding — the failure only a person can clear. pub const TTL_SECS: i64 = 180; +const _: () = assert!( + MIN_EMIT_GAP_SECS < TTL_SECS, + "a gap at or beyond the TTL lets the declaration lapse while the node still struggles" +); + /// Build the self-report the node will sign. /// /// `attesting == attested == subject`: CC 3.4.5 self-or-owner, satisfied on the @@ -103,6 +134,18 @@ pub fn spec(node_key_id: &str, expires_at: chrono::DateTime) -> cra "attesting_key_id": node_key_id, "subject_key_ids": [node_key_id], "score": 1.0, + // REQUIRED, not optional. `compose_policy::Composer::screen` refuses an + // envelope without it (`MalformedEnvelope("confidence")`) and deliberately + // does not default to 1.0 — so a row missing it is admitted and replicated + // and then contributes NOTHING to the composed verdict a peer would use to + // stop offering work (Codex, PR #504). The whole point of replicating this + // is that a peer can act on it. + // + // 1.0 because the producer is the subject: this is not an inference about + // someone else, it is a measurement of itself under `witness_relation: + // self`, and a consumer discounts self-attestation by relation rather than + // by the producer pre-discounting its own reading. + "confidence": 1.0, "cohort_scope": cohort_scope::FEDERATION, "witness_relation": "self", "state": STATE_SHEDDING, @@ -190,8 +233,10 @@ pub fn spawn( let join = tokio::spawn(async move { let mut interval = tokio::time::interval(OBSERVE_INTERVAL); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut last_emit: Option> = None; tracing::info!( dimension = DIMENSION, + min_emit_gap_secs = MIN_EMIT_GAP_SECS, observe_secs = OBSERVE_INTERVAL.as_secs(), ttl_secs = TTL_SECS, pen = match pen { @@ -216,13 +261,30 @@ pub fn spawn( continue; } + // A live declaration already covers this moment — do not mint a second + // permanent row to say the same thing. Re-emit only once the standing + // one is close enough to expiring that a gap would open. + let now = chrono::Utc::now(); + if last_emit.is_some_and(|t: chrono::DateTime| { + now - t < chrono::Duration::seconds(MIN_EMIT_GAP_SECS) + }) { + continue; + } + match emit(&engine, &pen, &node_key_id).await { - Ok(id) => tracing::warn!( - attestation_id = %id, - ttl_secs = TTL_SECS, - "config:load = shedding — self-attested; expires on its own; peers may \ - read it to stop offering" - ), + Ok(id) => { + // Only a SUCCESSFUL emission moves the clock. A failed one must + // leave it, or a transient refusal would suppress renewal for a + // whole gap and the node would go quiet while still struggling. + last_emit = Some(now); + tracing::warn!( + attestation_id = %id, + ttl_secs = TTL_SECS, + renewal_gap_secs = MIN_EMIT_GAP_SECS, + "config:load = shedding — self-attested; expires on its own; peers may \ + read it to stop offering" + ); + } Err(e) => tracing::warn!(error = %e, "config:load attestation failed — continuing"), } } diff --git a/src/node_key.rs b/src/node_key.rs index 103277c2..30ef25c8 100644 --- a/src/node_key.rs +++ b/src/node_key.rs @@ -904,14 +904,37 @@ pub async fn register_actor_occurrence( Ok(()) } +/// The node's own pen, retained when provisioning minted a node key. +/// +/// `provision_node_identity` sets BOTH [`set_actor_identity`] and +/// [`set_wire_identity`]: the wheel's embedded path is a SPLIT, exactly like the +/// composed one. The engine keeps signing as the ACTOR, so anything authored AS +/// the node needs this pen — a row stamped as the node and signed by the actor is +/// rejected at admission (Codex, PR #504; I had claimed this path did not split, +/// and it does). The signer was previously bound to `_signer` and dropped. +/// +/// `None` on a node that never provisioned through this path — there the engine +/// key IS the node key and the engine is the right pen. +static NODE_PEN: std::sync::OnceLock> = + std::sync::OnceLock::new(); + +/// The node's own signer, when this boot minted one. +#[must_use] +pub fn node_pen_signer() -> Option> { + NODE_PEN.get().map(Arc::clone) +} + pub async fn provision_node_identity( engine: &ciris_persist::prelude::Engine, keystore_alias: &str, identity_dir: &std::path::Path, actor_key_id: Option<&str>, ) -> Result { - let (_signer, identity) = node_signer(keystore_alias, identity_dir).await?; + let (signer, identity) = node_signer(keystore_alias, identity_dir).await?; let key_id = register_node_key(engine, &identity).await?; + // RETAINED, not dropped. Rows authored as the node need this pen; the engine + // signs as the actor on a split node. + let _ = NODE_PEN.set(signer); // ── The readiness gate: edge must not start on a half-provisioned identity ── // diff --git a/tests/config_load_reaches_the_put_door.rs b/tests/config_load_reaches_the_put_door.rs new file mode 100644 index 00000000..3034cccf --- /dev/null +++ b/tests/config_load_reaches_the_put_door.rs @@ -0,0 +1,267 @@ +//! **The `config:load` self-attestation, through a REAL put door.** +//! +//! Every unit test in `load_shed` passed while two defects made the row +//! unadmittable in production (Codex, PR #504): +//! +//! 1. the dimension carried no `:vN` segment, so persist's +//! `DimensionAdmissionPolicy` refused it (`MissingVersionSegment`, T3); +//! 2. on a split node the row was stamped as the node and signed by the ACTOR, +//! so the signature failed verification against the node's registered key. +//! +//! Neither is visible to a test that stops at the `Spec`. Both are invisible in +//! production too — the observer logs "attestation failed" once a minute and +//! nothing ever reaches a peer, which is the failure mode this file exists to +//! make loud. The rule it encodes: **a producer is not tested until its row has +//! been through `put_attestation`.** + +use base64::engine::general_purpose::STANDARD as B64; +use base64::Engine as _; +use ciris_keyring::MlDsa65SoftwareSigner; +use ciris_persist::federation::types::{algorithm, identity_type, KeyRecord}; +use ciris_persist::federation::SignedKeyRecord; +use ciris_persist::prelude::{Engine, LocalSigner}; +use ed25519_dalek::SigningKey; +use sha2::{Digest, Sha256}; +use std::sync::Arc; + +fn seed(label: &str, n: u8) -> [u8; 32] { + let mut s = [0u8; 32]; + let d = Sha256::digest(format!("{label}:{n}").as_bytes()); + s.copy_from_slice(&d); + s +} + +fn signer_for(key_id: &str) -> LocalSigner { + let ed = SigningKey::from_bytes(&seed(key_id, 1)); + let pqc = Arc::new( + MlDsa65SoftwareSigner::from_seed_bytes(&seed(key_id, 2), format!("{key_id}-pqc")) + .expect("ML-DSA-65 seed"), + ); + LocalSigner::from_parts( + ed, + key_id.to_string(), + Some(pqc), + Some(format!("{key_id}-pqc")), + ) +} + +async fn register(engine: &Engine, signer: &LocalSigner, key_id: &str, ident: &str) { + let mut envelope = serde_json::json!({ "key_id": key_id }); + let probe = signer.sign_hybrid(b"probe").await.expect("probe"); + let ed_pub = B64.encode(&probe.classical.public_key); + let pqc_pub = B64.encode(&probe.pqc.public_key); + ciris_persist::federation::admission::bind_subject_into_envelope( + &mut envelope, + key_id, + ident, + &ed_pub, + Some(&pqc_pub), + ) + .expect("bind subject (#659)"); + let canonical = + ciris_persist::verify::canonical::ceg_produce_canonicalize(&envelope).expect("canon"); + let sig = signer.sign_hybrid(&canonical).await.expect("sign"); + let now = chrono::Utc::now(); + engine + .register_federation_key(SignedKeyRecord { + record: KeyRecord { + key_id: key_id.to_string(), + pubkey_ed25519_base64: ed_pub, + pubkey_ml_dsa_65_base64: Some(pqc_pub), + algorithm: algorithm::HYBRID.into(), + identity_type: ident.to_string(), + identity_ref: key_id.to_string(), + valid_from: now, + valid_until: None, + registration_envelope: envelope, + original_content_hash: hex::encode(Sha256::digest(&canonical)), + scrub_signature_classical: B64.encode(&sig.classical.signature), + scrub_signature_pqc: Some(B64.encode(&sig.pqc.signature)), + scrub_key_id: key_id.to_string(), + scrub_timestamp: now, + pqc_completed_at: Some(now), + persist_row_hash: String::new(), + capability_roles: Vec::new(), + attestation_evidence: None, + consent_role: None, + additional_scrubs: Vec::new(), + }, + }) + .await + .unwrap_or_else(|e| panic!("register {key_id}: {e}")); +} + +async fn engine_for(key_id: &str) -> Arc { + Arc::new( + Engine::with_signer(Arc::new(signer_for(key_id)), "sqlite::memory:") + .await + .expect("Engine::with_signer(sqlite::memory:)"), + ) +} + +/// **The unsplit node: engine key IS the node key, engine is the right pen.** +/// +/// This is the standalone binary and the wheel's bare-agent path. Admission must +/// accept the row — which is what catches a missing `:vN`, a malformed envelope, +/// or any other rule the `Spec` alone cannot see. +#[tokio::test] +async fn an_unsplit_node_emits_a_config_load_row_that_admission_accepts() { + const NODE: &str = "unsplit-node"; + let engine = engine_for(NODE).await; + register(&engine, &signer_for(NODE), NODE, identity_type::NODE).await; + + let id = + ciris_server::load_shed::emit(&engine, &ciris_server::load_shed::NodePen::Engine, NODE) + .await + .expect("config:load must be ADMITTED, not merely built"); + assert!(!id.is_empty()); + + let rows = engine + .federation_directory() + .list_attestations_for(NODE) + .await + .expect("read back"); + let row = rows + .iter() + .find(|r| { + r.attestation_envelope + .get("dimension") + .and_then(serde_json::Value::as_str) + == Some(ciris_server::load_shed::DIMENSION) + }) + .expect("the config:load row is in the store"); + + // Signed by the node itself — CC 3.4.5 self-or-owner, on the wire. + assert_eq!(row.scrub_key_id, NODE); + assert_eq!(row.attesting_key_id, NODE); + // And it expires. A row without this is a permanent declaration. + assert!( + row.expires_at.is_some(), + "the row must carry the expiry that bounds the node's standing" + ); +} + +/// **The split node: the node's own pen, not the engine's.** +/// +/// The engine signs as the ACTOR while `wire_identity()` is the minted node key. +/// Signing with the engine here produces a row stamped as the node and signed by +/// the actor — admission rejects it against the node's registered public key. +/// This is the topology the feature exists for, and the one where the local unit +/// tests were fully green while nothing could ever be admitted. +#[tokio::test] +async fn a_split_node_signs_as_the_node_and_admission_accepts_it() { + const ACTOR: &str = "split-actor"; + const NODE: &str = "split-node"; + // The engine's identity is the ACTOR — the agent-carrying shape. + let engine = engine_for(ACTOR).await; + register(&engine, &signer_for(ACTOR), ACTOR, identity_type::AGENT).await; + register(&engine, &signer_for(NODE), NODE, identity_type::NODE).await; + + let node_pen = ciris_server::load_shed::NodePen::Node(Arc::new(signer_for(NODE))); + let id = ciris_server::load_shed::emit(&engine, &node_pen, NODE) + .await + .expect("a split node must author config:load with the NODE's pen"); + assert!(!id.is_empty()); + + let rows = engine + .federation_directory() + .list_attestations_for(NODE) + .await + .expect("read back"); + let row = rows + .iter() + .find(|r| { + r.attestation_envelope + .get("dimension") + .and_then(serde_json::Value::as_str) + == Some(ciris_server::load_shed::DIMENSION) + }) + .expect("the config:load row is in the store"); + + assert_eq!( + row.scrub_key_id, NODE, + "signed by the NODE — an actor-signed row fails verification against the node's key" + ); + assert_ne!(row.scrub_key_id, ACTOR, "the engine's pen must not appear"); +} + +/// **A peer must be able to COMPOSE a verdict from it.** +/// +/// Admission is not the last gate. `compose_policy::Composer::screen` refuses an +/// envelope with no `confidence` (`MalformedEnvelope("confidence")`) and does NOT +/// default it — so a row could be admitted, replicated, and still contribute +/// nothing to the verdict a peer would use to stop offering work (Codex, PR #504). +/// The whole point of replicating this is that a peer can act on it, so the +/// producer is checked against the consumer. +#[tokio::test] +async fn a_peer_can_screen_the_row_it_receives() { + const NODE: &str = "screenable-node"; + let engine = engine_for(NODE).await; + register(&engine, &signer_for(NODE), NODE, identity_type::NODE).await; + + ciris_server::load_shed::emit(&engine, &ciris_server::load_shed::NodePen::Engine, NODE) + .await + .expect("admitted"); + + let rows = engine + .federation_directory() + .list_attestations_for(NODE) + .await + .expect("read back"); + let row = rows + .iter() + .find(|r| { + r.attestation_envelope + .get("dimension") + .and_then(serde_json::Value::as_str) + == Some(ciris_server::load_shed::DIMENSION) + }) + .expect("row present"); + + // The two fields a consumer's screen requires, present and well-formed. + let conf = row + .attestation_envelope + .get("confidence") + .and_then(serde_json::Value::as_f64) + .expect("confidence — Composer::screen refuses the envelope without it"); + assert!( + (0.0..=1.0).contains(&conf), + "confidence in range, got {conf}" + ); + assert!(row + .attestation_envelope + .get("score") + .and_then(serde_json::Value::as_f64) + .is_some()); +} + +/// **The regression that would have shipped: an unversioned dimension.** +/// +/// Asserts the rule directly against persist rather than trusting our constant, +/// so this fails if persist tightens `DimensionAdmissionPolicy` further — the +/// producer-obligation half of CC 3.4.7 checked against the substrate half. +#[tokio::test] +async fn an_unversioned_config_dimension_is_refused_at_admission() { + const NODE: &str = "unversioned-node"; + let engine = engine_for(NODE).await; + register(&engine, &signer_for(NODE), NODE, identity_type::NODE).await; + + // The exact shape the first cut emitted: `config:load`, no `:vN`. + let expires_at = chrono::Utc::now() + chrono::Duration::seconds(180); + let mut spec = ciris_server::load_shed::spec(NODE, expires_at); + spec.envelope["dimension"] = serde_json::json!("config:load"); + + let row = ciris_server::attest::Emit::stamp(NODE, spec) + .expect("stamp") + .sign_and_assemble(ciris_server::attest::KeySigner::Engine(&engine)) + .await + .expect("sign"); + let err = ciris_server::attest::put(&engine, row) + .await + .expect_err("persist MUST refuse a scores dimension with no version segment"); + let msg = err.to_string(); + assert!( + msg.contains("version") || msg.contains("dimension"), + "the refusal should name the dimension rule, got: {msg}" + ); +}