diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d319294..b17150d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2063,10 +2063,32 @@ jobs: fail(f"unparseable consensus_protocol {proto!r}") else: fail(f"unrecognised consensus_protocol {proto!r} — refusing to guess a quorum") + # CIRISEdge#575 — REPORT, not gate, while the trust root moves. + # + # This was a hard failure, matching what the guard is for. It is + # downgraded to match CIRISPersist#809, whose reasoning applies + # here unchanged: CIRISRegistry v3.0.0 went live 2026-09-04 and + # holders and authorizations move during that migration, so a tag + # run blocked on a mid-migration count is a release blocked on + # somebody else's deploy window. Edge has already lost one release + # (v20.2.0, no artifacts at all) to a registry state it does not + # control, and that is the risk being weighed against. + # + # Say plainly what it costs, because a check that cannot fail is a + # report: for the length of the transition this leg CANNOT catch a + # registry serving a bundle whose quorum has collapsed. What still + # fails in that case is the registration step below — this leg is + # early warning, and this half of it is warning only. + # + # Restore it when the root settles; CIRISEdge#575 tracks that, and + # the two repos should restore together rather than drift again. if len(signed_by) < need: - fail( - f"accord bundle has {len(signed_by)} authorization(s) " - f"under {proto} — quorum not met" + print( + f"::warning::accord bundle has {len(signed_by)} " + f"authorization(s) under {proto} — quorum NOT met. Not " + f"gating while the trust root transitions (CIRISEdge#575 / " + f"CIRISPersist#809)", + file=sys.stderr, ) root = data.get("charter_root_key_id") if not root: diff --git a/Cargo.lock b/Cargo.lock index c5b7f1b..5fbad9c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -722,6 +722,35 @@ dependencies = [ "piper", ] +[[package]] +name = "bluer" +version = "0.17.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af68112f5c60196495c8b0eea68349817855f565df5b04b2477916d09fb1a901" +dependencies = [ + "custom_debug", + "dbus", + "dbus-crossroads", + "dbus-tokio", + "displaydoc", + "futures", + "hex", + "lazy_static", + "libc", + "log", + "macaddr", + "nix 0.29.0", + "num-derive", + "num-traits", + "pin-project", + "serde", + "serde_json", + "strum", + "tokio", + "tokio-stream", + "uuid", +] + [[package]] name = "borrow-or-share" version = "0.2.4" @@ -967,7 +996,7 @@ dependencies = [ [[package]] name = "ciris-edge" -version = "20.2.1" +version = "20.3.0" dependencies = [ "async-trait", "axum", @@ -1066,8 +1095,8 @@ dependencies = [ [[package]] name = "ciris-persist" -version = "41.0.0" -source = "git+https://github.com/CIRISAI/CIRISPersist?tag=v41.0.0#20af0a8fca64014fb560ec0f4f14d57782cf0a43" +version = "41.2.0" +source = "git+https://github.com/CIRISAI/CIRISPersist?tag=v41.2.0#52f659257a480f3956824f3241b60fca4a5ece39" dependencies = [ "async-trait", "base64 0.22.1", @@ -1484,6 +1513,63 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "custom_debug" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2da7d1ad9567b3e11e877f1d7a0fa0360f04162f94965fc4448fbed41a65298e" +dependencies = [ + "custom_debug_derive", +] + +[[package]] +name = "custom_debug_derive" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a707ceda8652f6c7624f2be725652e9524c815bf3b9d55a0b2320be2303f9c11" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.119", +] + [[package]] name = "data-encoding" version = "2.11.0" @@ -1512,6 +1598,39 @@ dependencies = [ "system-deps", ] +[[package]] +name = "dbus" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ab69f03cc8c4340c9c8e315114e1658e6775a9b16a04357973aa21cec22b32e" +dependencies = [ + "futures-channel", + "futures-util", + "libc", + "libdbus-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "dbus-crossroads" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64bff0bd181fba667660276c6b7ebdc50cff37ce593e7adf9e734f89c8f444e8" +dependencies = [ + "dbus", +] + +[[package]] +name = "dbus-tokio" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "007688d459bc677131c063a3a77fb899526e17b7980f390b69644bdbc41fad13" +dependencies = [ + "dbus", + "libc", + "tokio", +] + [[package]] name = "deadpool" version = "0.12.3" @@ -1764,6 +1883,35 @@ dependencies = [ "serde", ] +[[package]] +name = "embassy-sync" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73974a3edbd0bd286759b3d483540f0ebef705919a5f56f4fc7709066f71689b" +dependencies = [ + "cfg-if", + "critical-section", + "embedded-io-async", + "futures-core", + "futures-sink", + "heapless", +] + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + +[[package]] +name = "embedded-io-async" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ff09972d4073aa8c299395be75161d582e7629cd663171d62af73c8d50dba3f" +dependencies = [ + "embedded-io", +] + [[package]] name = "enum-as-inner" version = "0.6.1" @@ -2803,6 +2951,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "1.1.0" @@ -3136,10 +3290,19 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "leviculum-ble-tx" +version = "0.1.0" +source = "git+https://github.com/CIRISAI/leviculum?tag=v0.25.0+ciris.1#381bc31d2b2c33e35bf8f2a7880aaf0803c11ca1" +dependencies = [ + "embassy-sync", + "leviculum-core", +] + [[package]] name = "leviculum-core" -version = "0.24.0+ciris.1" -source = "git+https://github.com/CIRISAI/leviculum?tag=v0.24.0+ciris.1#553bb43f4283b26bc714f6fb42a88082022305d6" +version = "0.25.0+ciris.1" +source = "git+https://github.com/CIRISAI/leviculum?tag=v0.25.0+ciris.1#381bc31d2b2c33e35bf8f2a7880aaf0803c11ca1" dependencies = [ "aes", "cbc", @@ -3156,8 +3319,8 @@ dependencies = [ [[package]] name = "leviculum-lxmf" -version = "0.24.0+ciris.1" -source = "git+https://github.com/CIRISAI/leviculum?tag=v0.24.0+ciris.1#553bb43f4283b26bc714f6fb42a88082022305d6" +version = "0.25.0+ciris.1" +source = "git+https://github.com/CIRISAI/leviculum?tag=v0.25.0+ciris.1#381bc31d2b2c33e35bf8f2a7880aaf0803c11ca1" dependencies = [ "leviculum-core", "rand_core 0.6.4", @@ -3167,13 +3330,17 @@ dependencies = [ [[package]] name = "leviculum-std" -version = "0.24.0+ciris.1" -source = "git+https://github.com/CIRISAI/leviculum?tag=v0.24.0+ciris.1#553bb43f4283b26bc714f6fb42a88082022305d6" +version = "0.25.0+ciris.1" +source = "git+https://github.com/CIRISAI/leviculum?tag=v0.25.0+ciris.1#381bc31d2b2c33e35bf8f2a7880aaf0803c11ca1" dependencies = [ + "bluer", "clap", + "critical-section", + "dbus", "futures", "hmac 0.12.1", "if-addrs", + "leviculum-ble-tx", "leviculum-core", "leviculum-lxmf", "libc", @@ -3469,6 +3636,16 @@ dependencies = [ "rand 0.9.5", ] +[[package]] +name = "libdbus-sys" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "328c4789d42200f1eeec05bd86c9c13c7f091d2ba9a6ea35acdf51f31bc0f043" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "libfuzzer-sys" version = "0.4.13" @@ -3560,6 +3737,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "macaddr" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baee0bbc17ce759db233beb01648088061bf678383130602a298e6998eedb2d8" + [[package]] name = "mach2" version = "0.4.3" @@ -3767,6 +3950,18 @@ dependencies = [ "memoffset 0.7.1", ] +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags 2.13.1", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nix" version = "0.31.3" @@ -4250,6 +4445,26 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -5755,6 +5970,28 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.119", +] + [[package]] name = "subtle" version = "2.6.1" diff --git a/Cargo.toml b/Cargo.toml index 5f2e90a..1002e8d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ciris-edge" -version = "20.2.1" +version = "20.3.0" edition = "2021" rust-version = "1.75" authors = ["Eric Moore "] @@ -441,7 +441,7 @@ publish = false # registers a SINGLE-role `identity_type` — no fixture encoded the # repealed loophole. # * 2566bc54 still pins CIRISVerify v13.6.1 — one `ciris-verify-core`. -ciris-persist = { git = "https://github.com/CIRISAI/CIRISPersist", tag = "v41.0.0", version = "41", features = ["sqlite", "encrypted-kv"] } +ciris-persist = { git = "https://github.com/CIRISAI/CIRISPersist", tag = "v41.2.0", version = "41", features = ["sqlite", "encrypted-kv"] } # Keyring — Ed25519 + ML-DSA-65 hardware/software signers used by # Edge::send and Edge::send_durable to sign outbound envelopes. # v0.13.0 — bumped to v4.0.0 in lockstep with persist v3.0.0. Both @@ -884,8 +884,8 @@ tempfile = { version = "3", optional = true } # transfer. What that completion carries for a SPLIT transfer is the FINAL # segment's `resource_hash`, not the one `send_resource_awaited` returned — see # `ship_resource_on_link`. -leviculum-core = { git = "https://github.com/CIRISAI/leviculum", tag = "v0.24.0+ciris.1", version = "0.24", optional = true } -leviculum-std = { git = "https://github.com/CIRISAI/leviculum", tag = "v0.24.0+ciris.1", version = "0.24", optional = true } +leviculum-core = { git = "https://github.com/CIRISAI/leviculum", tag = "v0.25.0+ciris.1", version = "0.25", optional = true } +leviculum-std = { git = "https://github.com/CIRISAI/leviculum", tag = "v0.25.0+ciris.1", version = "0.25", optional = true } # v15.20.0 (CIRISEdge#169) — LXMF store-and-forward propagation. The # `leviculum-lxmf` workspace member (same pinned tag) ships the whole # LXMF wire protocol: propagation-node discovery/upload/`/get` codecs, @@ -895,7 +895,7 @@ leviculum-std = { git = "https://github.com/CIRISAI/leviculum", tag = "v0.24.0+ # features keep `pow` on (the stamp/ticket cost we validate at # admission). Gated behind the `lxmf` feature — see the `[features]` # block. Edge INTEGRATES this crate's protocol; it never reimplements it. -leviculum-lxmf = { git = "https://github.com/CIRISAI/leviculum", tag = "v0.24.0+ciris.1", version = "0.24", optional = true } +leviculum-lxmf = { git = "https://github.com/CIRISAI/leviculum", tag = "v0.25.0+ciris.1", version = "0.25", optional = true } # v0.11.0 (CIRISEdge#31) — Identity FFI QR codec. `qrcodegen` is the # canonical pure-Rust QR encoder (Apache-2.0, no_std, zero unsafe in @@ -1540,7 +1540,7 @@ async-trait = "0.1" # two `ciris-verify-core` cdylibs — the empty-stdout SIGSEGV class). # v38.6.0 (CIRISPersist#774) — held in lockstep with the runtime pin above # through the RC-adopt and back onto `tag`. These two move together, always. -ciris-persist = { git = "https://github.com/CIRISAI/CIRISPersist", tag = "v41.0.0", version = "41", features = ["sqlite", "cirisnode", "classify", "scrub", "encrypted-kv"] } +ciris-persist = { git = "https://github.com/CIRISAI/CIRISPersist", tag = "v41.2.0", version = "41", features = ["sqlite", "cirisnode", "classify", "scrub", "encrypted-kv"] } # CIRISEdge#23 / #49 — `tests/transport_http_hardening.rs` + # `tests/https_per_messagetype_roundtrip.rs` + `tests/https_pyedge_init.rs` # (v0.19.3) mint self-signed Ed25519 certs on the fly. v0.19.3 diff --git a/docs/RELEASE_NOTES.md b/docs/RELEASE_NOTES.md index 1413053..b0e855d 100644 --- a/docs/RELEASE_NOTES.md +++ b/docs/RELEASE_NOTES.md @@ -1,5 +1,181 @@ # CIRISEdge Release Notes +# v20.3.0 — adopt persist v41.2.0 + leviculum v0.25.0; the dial outlives its round (#568); the pyo3 envelope helper signs hybrid (#573); the rotation seal names its hazard (leviculum#52) + +**2026-09-05** — Additive at every public surface. Two substrate adopts, one +transport fix, one FFI fix. + +## Adopts + +**CIRISPersist v41.0.0 → v41.2.0.** Two minors, both currency for edge. All +four ABI constants unchanged across both, verify stays v14.2.0, so the +`ciris-persist>=41,<42` wheel floor holds. v41.1.0 (#807) fixes +`list_widening_candidates` offering an announced node's owner-binding as a +widening candidate forever. v41.2.0 (#810) adds `rejected` to the mirrored +CIRISLens `TaskStatus` vocabulary with a SQLite V136 rebuild. **Edge drives +none of the affected APIs** — no `TaskStatus`, no lens tables, none of the +lens features enabled. + +**leviculum v0.24.0+ciris.1 → v0.25.0+ciris.1.** Three things matter here: + +- **leviculum#64 unbreaks our own wheels.** The upstream catch-up (+92) brought + a BLE interface declaring `bluer`/`dbus` unconditionally; both pull + `libdbus-sys`, which builds on neither macOS nor Windows. leviculum's own + changelog names the consumer: *"CIRISEdge's darwin and win_amd64 wheels + resolve this crate as a git dependency and would have failed at build + time."* The deps are now target-gated. +- **leviculum#63 gives us the instrument for #568** (below). +- **leviculum#52** is addressed below. + +## CIRISEdge#568 — a round no longer destroys the link it paid for + +The first fix for #568 (v20.2.0) addressed the missing-signer race. Re-measured +on the two-node ladder, the number did not move: **181 s and 210 s**. The +owner's key was on the peer at 31 s, so the key was never the bottleneck. This +is a second, independent mechanism. + +**What it is.** The scheduler abandons a round at `DEFAULT_ROUND_TIMEOUT` +(10 s). A peer with a known path is allowed `LINK_ESTABLISH_TIMEOUT` (30 s) to +establish. The dial ran **inline inside the round's future**, so a cold dial +could never finish inside the round that started it — that is arithmetic, not +load — and abandoning the round **dropped the half-established link**. Every +retry began cold; the peer converged only when it dialled us. The observed +shape matches exactly: first Attestation round to a fresh peer times out at ++10 s, next completed round lands ~150 s later — five 30 s cadence ticks. + +This is the #532 establish:identify gap one layer up. #532 stopped N +coordinators discarding *each other's* links; nothing stopped a round +discarding *its own*. + +**The fix.** The dial runs in a spawned task holding a cloneable `DialCtx`, so +abandoning a round abandons the **wait**, not the **link**. The task runs to +completion and publishes into `reusable_dialed_link` — `dial_and_identify`'s +existing "PUBLISH LAST" step — so the next round takes the fast reuse path +instead of dialling cold again. + +`DialCtx` is a context struct rather than a `Weak` because +`ReticulumTransport::new` returns `Self`: a self-handle would need installing +at every construction site, and a site that forgot would silently lose the +detach. The dial-gate permit is acquired **inside** the task, since a permit +held by the caller would be released on cancellation while the dial it gates +still ran — leaving #532's single-flight property true only for callers that +survive. + +**Measurement, not inference.** `ladder.owner_binding_converged` now records +leviculum#63's `retry_queued` / `retry_queue_cap` / `retry_dropped_total` +beside `elapsed_ms`. A slow convergence with a flat retry queue and one with +`dropped_total` climbing are different bugs that read identically as a +stopwatch value — leviculum reported that exact pattern from the live canonical +(*"the retry queue climbed past its warning threshold and began discarding +traffic while the log read as quiet"*). The next run distinguishes them. + +**What is not claimed:** that the 181/210 s numbers are now fixed. The unit +tests pin the mechanism; only a mesh run measures the outcome. One run each was +not a regression call in either direction, and it is not a fix call either. + +## leviculum#52 — the rotation seal docstring named the wrong hazard + +Audited against leviculum's finding that `seal` retires the old IFAC key for +inbound, so sealing with no dwell after `activate` strands in-flight traffic +(~50% packet loss on a zero-dwell rotation upstream, `drops_ifac` incrementing +exactly once each time). + +**Edge is exposed as the API surface, not as a driver.** It wraps and +re-exports the three phases (`ifac_install_next` / `ifac_activate_next` / +`ifac_seal_rotation`), and **nothing in edge sequences them** — no +`activate → seal` pair in `src/`, `tests/` or `src/bin/`. Edge cannot commit +the defect itself; it hands it to the operator. Two things it told that +operator were wrong. + +**The docstring named the wrong hazard.** Both seal wrappers said "call after +the convergence window". That window is about *membership* — every member +holding the new key. Sealing also retires the old key for *inbound*, so it +rejects packets already on the wire under the old mask, sent by the members +that **did** re-key. An operator who followed the instruction correctly still +stranded traffic. Worse than no guidance, because it read as complete. Both +docstrings now name the drain hazard as distinct from membership, state that +install/activate are make-before-break and only seal breaks, and quote the +upstream evidence. + +**Seal breaks inbound only, so the gauge points the other way.** leviculum +consults the alternate key on the inbound path alone (`verify_ifac`); +`apply_ifac` never reads it. A node's seal rejects old-masked traffic arriving +*from peers* — upstream's failing run showed `drops_ifac` at the relay, +rejecting a member's in-flight output. `retry_queued` counts *this* node's +**outbound** backlog, so reaching zero licenses this node's **peers** to seal, +not this node. The true precondition — every peer's `retry_queued` is zero — +is not observable from any one node. Under lockstep rotation (the only mode +edge supports, since nothing sequences the phases) each node polling its own +gauge to zero approximates it. The gauge is now on `PyEdge` and documented as +a **fleet proxy, never a measurement of what this node's own seal will +reject**. The retry queue is the long pole: packets are masked before +enqueueing, re-sent verbatim, count-capped (1024) with no time bound. + +**A default — a conservative choice, not a derivation.** +`DEFAULT_IFAC_ROTATION_DWELL` = `RESOURCE_TRANSFER_TIMEOUT` (120 s), exported +as `default_ifac_rotation_dwell_ms()`. The drain window is bounded by peers' +retry-queue residency, which has no time bound, so there is no principled +number to derive; 120 s reuses edge's committed answer to how long traffic can +legitimately remain in flight. (An earlier draft justified it by transfer +duration and leviculum#62 — wrong: old-masked bytes stop being generated at +`activate`, and #62 is receiver-side reassembly.) 10 s was proposed; 10 s is +shorter than every in-flight window edge already asserts. It is the *safe* +default: ejecting a compromised member is a reason to go lower on purpose and +accept the drops. + +**Two clocks, not to be harmonized.** `install → activate` waits on key +*distribution* — `DEFAULT_CONVERGENCE_WINDOW` (300 s); activating early only +degrades stragglers, since upstream keeps the old key accept-only. +`activate → seal` waits on packet *drain* — `DEFAULT_IFAC_ROTATION_DWELL` +(120 s), the only one whose failure is silent. The pre-fix prose collapsed both +into "the convergence window"; a later cleanup equalising the two constants +would reintroduce the hazard. Retry-queue residency, the drain's long pole, is +bounded by neither a clock nor any deterministic quantity: no TTL, head-of-line, +airtime-paced, and the 1024 cap bounds *depth* — a packet at the front of a +quiet link leaves only once 1024 arrive behind it. + +**No `seal_after_drain` helper, deliberately.** It would have to invent a +policy for the queue never draining, and both answers belong to the operator: +block forever and the member being excluded stays admitted indefinitely; time +out and seal anyway and the hazard returns, buried in a function whose name +promises it cannot happen. + +**No test, stated plainly.** Nothing in edge sequences the IFAC trio, so there +is zero coverage and nothing to regress from. `scope.rotation` and +`conformance.rotation_frame_loss` drive the MLS epoch advance and the +scope-address table, not IFAC. Per upstream the natural test is a ~50% flake; a +test worth having is a soak. + +Upstream Reticulum's IFAC is static config with no rotation ceremony, so there +is no prior art and no ecosystem number to borrow — leviculum's 500 ms is a +loopback floor it disclaims as a deployment value. + +## CIRISEdge#573 — the pyo3 envelope helper signs hybrid + +`Edge.build_signed_inbound_envelope` hard-coded `None` for the PQC half, so +since **v19.0.0** — when every signature became the full hybrid — it could not +build a single envelope on the wheel that ships it. Its doc justified the +omission with two claims that were both false on this wheel. + +Nothing here caught it because the Rust twin the doc names as its counterpart +(`tests/trust_short_circuit.rs::FedKey::local_signer`) *had* been moved to +hybrid. The two codepaths the doc calls identical had silently diverged, and +the only caller of the broken one is Python. + +`pqc_seed_bytes` (optional, 32 bytes) now builds the PQC half. The seed is +**taken, not derived** from `seed_bytes`: a derivation convention has to be +known identically by whoever REGISTERS the ML-DSA pubkey and whoever SIGNS with +it, and when they disagree the only symptom is a verify refusal at the far end +with nothing pointing at the cause. `Edge.derive_ml_dsa_65_pubkey_base64` gives +the matching `federation_keys` pubkey from the same code that signs. + +The argument stays optional so the classical-only refusal remains reachable and +loud. Surfaced by CIRISConformance#91; unblocks `test_230_intake_gate` and +`test_520_wire_vocabulary::test_tier1_and_opaque_variants_accepted`, whose +xfails are keyed on the exact refusal token rather than blanket markers. + +--- + # v20.2.1 — republish v20.2.0's artifacts past a stale registry guard **2026-09-04** — No code change. v20.2.0 is byte-identical in behaviour; this diff --git a/evidence/CIRISEdge.cc_impl.tsv b/evidence/CIRISEdge.cc_impl.tsv index 9a7c443..1e83e12 100644 --- a/evidence/CIRISEdge.cc_impl.tsv +++ b/evidence/CIRISEdge.cc_impl.tsv @@ -7,10 +7,10 @@ # failure. Vendored by CIRISConstitution/tools/check_evidence.py. # Columns: cc_section clm repo path#symbol crate@version decimal_id claim_id repo path#symbol crate@version -5.3.3 CLM-nsproc-delivery-mode CIRISEdge src/delivery_mode.rs#decide ciris-edge@v20.2.1 -3.3.6 CLM-nsproc-cohort-scope CIRISEdge src/replication/bridge.rs#attestation_is_advertised ciris-edge@v20.2.1 -3.1 CLM-nsproc-dimension CIRISEdge src/replication/bridge.rs#attestation_is_advertised ciris-edge@v20.2.1 -3.4 CLM-nsproc-key-boundary-scope CIRISEdge src/key_boundary.rs#KeyBoundaryScope ciris-edge@v20.2.1 -5.3.3.5 CLM-nsproc-recipient-serve-capability CIRISEdge src/replication/bridge.rs#peer_has_serve_capability ciris-edge@v20.2.1 -5.3.2.4 CLM-nsproc-recipient-capability CIRISEdge src/replication/bridge.rs#recipient_capability_withholds ciris-edge@v20.2.1 -5.3.2.4 CLM-nsproc-attestation-prefixes CIRISEdge src/replication/bridge.rs#recipient_capability_withholds ciris-edge@v20.2.1 +5.3.3 CLM-nsproc-delivery-mode CIRISEdge src/delivery_mode.rs#decide ciris-edge@v20.3.0 +3.3.6 CLM-nsproc-cohort-scope CIRISEdge src/replication/bridge.rs#attestation_is_advertised ciris-edge@v20.3.0 +3.1 CLM-nsproc-dimension CIRISEdge src/replication/bridge.rs#attestation_is_advertised ciris-edge@v20.3.0 +3.4 CLM-nsproc-key-boundary-scope CIRISEdge src/key_boundary.rs#KeyBoundaryScope ciris-edge@v20.3.0 +5.3.3.5 CLM-nsproc-recipient-serve-capability CIRISEdge src/replication/bridge.rs#peer_has_serve_capability ciris-edge@v20.3.0 +5.3.2.4 CLM-nsproc-recipient-capability CIRISEdge src/replication/bridge.rs#recipient_capability_withholds ciris-edge@v20.3.0 +5.3.2.4 CLM-nsproc-attestation-prefixes CIRISEdge src/replication/bridge.rs#recipient_capability_withholds ciris-edge@v20.3.0 diff --git a/src/bin/edge_node.rs b/src/bin/edge_node.rs index 14016d6..f4b8592 100644 --- a/src/bin/edge_node.rs +++ b/src/bin/edge_node.rs @@ -2585,6 +2585,13 @@ async fn run_chat_legs(occ: &Occurrence) { peer stops at owner_of => None (CIRISEdge#568)" ); } + // CIRISEdge#568 — the retry-queue gauges alongside the stopwatch + // (leviculum#63). A slow convergence with a flat queue and one with + // `dropped_total` climbing are different bugs that read identically + // as elapsed_ms; recording both is what makes the next run a + // measurement instead of another inference. + let (retry_queued, retry_queue_cap, retry_dropped_total) = + occ.transport.retry_queue_gauges(); rep.ran( "ladder.owner_binding_converged", ok, @@ -2595,6 +2602,9 @@ async fn run_chat_legs(occ: &Occurrence) { "expected_owner": peer_owner, "resolved_owner": resolved, "elapsed_ms": elapsed_ms, + "retry_queued": retry_queued, + "retry_queue_cap": retry_queue_cap, + "retry_dropped_total": retry_dropped_total, }), ); } diff --git a/src/ffi/pyo3.rs b/src/ffi/pyo3.rs index c98021a..09ee943 100644 --- a/src/ffi/pyo3.rs +++ b/src/ffi/pyo3.rs @@ -1122,14 +1122,36 @@ impl PyEdge { /// e.g. `"inline_text"`, `"accord_events_batch"`, /// `"contribution_submit"`). /// + /// `pqc_seed_bytes` is the 32-byte ML-DSA-65 seed for the SAME + /// `signing_key_id`. **Required in practice** since v19.0.0 + /// (CIRISEdge#573): every signature is the full Ed25519 + ML-DSA-65 + /// hybrid and `sign_envelope` refuses a signer without its PQC half, + /// so omitting it produces an envelope that cannot be built at all + /// rather than one with an optional field left empty. + /// + /// The seed is taken rather than derived from `seed_bytes` on + /// purpose. A convention ("same seed", "seed with byte 0 flipped") + /// has to be known identically by whoever REGISTERS the ML-DSA + /// pubkey and whoever SIGNS with it, and when the two disagree the + /// only symptom is a verify refusal at the far end — the failure this + /// codebase has already paid for twice. Passing the seed makes the + /// harness the single source of it. Use + /// [`Self::derive_ml_dsa_65_pubkey_base64`] to get the matching + /// pubkey for the `federation_keys` row. + /// + /// It is left OPTIONAL so the classical-only refusal stays reachable + /// and loud: `None` fails at `sign_envelope` naming the missing half, + /// which is the correct answer and not a silent downgrade. + /// /// Returns the byte-exact `EdgeEnvelope` JSON ready to feed into - /// [`Self::dispatch_inbound_bytes`]. The classical signature is - /// always Ed25519 over the persist-canonicalized envelope; PQC - /// (ML-DSA-65) is OMITTED at this surface — software-only signers - /// don't carry a PQC half, and the conformance intake-gate test runs - /// under the verify pipeline's `Ed25519Fallback` policy where the - /// PQC field is optional. - #[pyo3(signature = (signing_key_id, seed_bytes, destination_key_id, message_type, body_json))] + /// [`Self::dispatch_inbound_bytes`]. + #[pyo3(signature = (signing_key_id, seed_bytes, destination_key_id, message_type, body_json, pqc_seed_bytes=None))] + // Eight, one over clippy's seven. This is a Python-facing signature whose + // arguments are each a distinct wire field the harness supplies; folding + // them into a struct would mean the caller building a dict to describe an + // envelope in order to build an envelope. Kept flat, and kept ADJACENT to + // its target — an `allow` separated from its fn silently moves. + #[allow(clippy::too_many_arguments)] fn build_signed_inbound_envelope( &self, py: Python<'_>, @@ -1138,6 +1160,7 @@ impl PyEdge { destination_key_id: &str, message_type: &str, body_json: &str, + pqc_seed_bytes: Option<&[u8]>, ) -> PyResult> { if seed_bytes.len() != 32 { return Err(PyValueError::new_err(format!( @@ -1164,6 +1187,18 @@ impl PyEdge { let seed_owned: [u8; 32] = seed_bytes.try_into().map_err(|_| { PyValueError::new_err("build_signed_inbound_envelope: seed_bytes must be 32 bytes") })?; + // Checked HERE, not inside the async block: a wrong-length PQC seed is + // a caller error and should read like one, next to the Ed25519 check. + let pqc_seed_owned: Option<[u8; 32]> = match pqc_seed_bytes { + None => None, + Some(b) => Some(b.try_into().map_err(|_| { + PyValueError::new_err(format!( + "build_signed_inbound_envelope: pqc_seed_bytes must be 32 bytes \ + (ML-DSA-65); got {}", + b.len() + )) + })?), + }; let envelope_bytes: Vec = py.detach(|| { run_async(&self.executor, async move { // Build a fresh software signer from the seed. The seed @@ -1172,17 +1207,15 @@ impl PyEdge { // one the persist OS-keyring fallback uses + matches the // `tests/trust_short_circuit.rs::FedKey::local_signer` // pattern byte-for-byte. - let mut sw = ciris_keyring::Ed25519SoftwareSigner::new(&signing_key_id); - sw.import_key(&seed_owned).map_err(|e| { - PyValueError::new_err(format!( - "build_signed_inbound_envelope: import_key failed: {e}" - )) + // CIRISEdge#573 — the PQC half. v19.0.0 made every signature + // the full hybrid and this surface was not moved with it, so + // the helper could not build ANY envelope on the wheel that + // ships it. Construction lives in `software_hybrid_signer` so + // a Rust test drives the same code rather than a copy. + let signer = software_hybrid_signer(&signing_key_id, &seed_owned, pqc_seed_owned) + .map_err(|e| { + PyValueError::new_err(format!("build_signed_inbound_envelope: {e}")) })?; - let signer = crate::identity::LocalSigner::new( - signing_key_id.clone(), - Arc::new(sw) as Arc, - None, - ); let mut env = crate::identity::build_envelope( mt, @@ -1216,6 +1249,40 @@ impl PyEdge { }) } + /// CIRISEdge#573 — the ML-DSA-65 public key a given 32-byte seed yields, + /// base64 (standard), for the `pubkey_ml_dsa_65_base64` column of the + /// `federation_keys` row. + /// + /// The companion to [`Self::build_signed_inbound_envelope`]'s + /// `pqc_seed_bytes`. A harness must REGISTER the same PQC key it later + /// SIGNS with, and deriving that pubkey independently means reimplementing + /// the alias-and-seed convention in Python — which is exactly how the two + /// sides drift apart and produce a verify refusal with no clue in it. One + /// seed in, one pubkey out, from the same code that does the signing. + #[staticmethod] + fn derive_ml_dsa_65_pubkey_base64(pqc_seed_bytes: &[u8]) -> PyResult { + use base64::Engine as _; + use ciris_keyring::PqcSigner as _; + let seed: [u8; 32] = pqc_seed_bytes.try_into().map_err(|_| { + PyValueError::new_err(format!( + "derive_ml_dsa_65_pubkey_base64: pqc_seed_bytes must be 32 bytes (ML-DSA-65); got {}", + pqc_seed_bytes.len() + )) + })?; + let signer = ciris_keyring::MlDsa65SoftwareSigner::from_seed_bytes(&seed, "derive-pqc") + .map_err(|e| { + PyValueError::new_err(format!( + "derive_ml_dsa_65_pubkey_base64: ml_dsa_65 from seed failed: {e}" + )) + })?; + let pk = futures::executor::block_on(signer.public_key()).map_err(|e| { + PyRuntimeError::new_err(format!( + "derive_ml_dsa_65_pubkey_base64: public_key failed: {e}" + )) + })?; + Ok(base64::engine::general_purpose::STANDARD.encode(pk)) + } + /// v7.2.0 (CIRISEdge#219) — runtime hot-plug a phone-attached LoRa /// (RNode-firmware) radio over a host-driven byte channel. The /// caller supplies a Python `(read_cb, write_cb)` pair that bridges @@ -1399,8 +1466,96 @@ impl PyEdge { /// CIRISEdge#492 — IFAC rotation phase 3 (seal): drop the OLD code. A member /// that never re-keyed is now excluded (readmission = fresh grant + re-key). - /// Call after the convergence window. Returns the number of interfaces - /// affected. + /// Returns the number of interfaces affected. + /// + /// **Leave a dwell after `ifac_activate_next` (leviculum#52).** This used to + /// say "call after the convergence window", which named the wrong hazard: + /// that window is about MEMBERSHIP, and sealing also retires the old key for + /// INBOUND — rejecting packets already on the wire under the old mask, sent + /// by the members that DID re-key. Waiting for membership convergence and + /// then sealing still strands traffic. Upstream measured a zero-dwell + /// rotation losing its packet in ~50% of runs. + /// + /// `install` and `activate` are make-before-break and cannot lose a packet; + /// only sealing breaks. Nothing imposes the dwell — the cutover moment is + /// deliberately yours. + /// + /// **Seal breaks INBOUND only.** leviculum consults the alternate key on + /// the inbound path alone; outbound never reads it. What your seal rejects + /// is old-masked traffic arriving FROM PEERS — upstream's failing run + /// showed `drops_ifac` at the relay, rejecting a member's in-flight output. + /// + /// **So [`Self::retry_queue_gauges`] points the other way.** It counts THIS + /// node's OUTBOUND backlog. Reaching 0 licenses your PEERS to seal, not + /// you. The true precondition for you sealing is "every peer's + /// `retry_queued` is 0", which you cannot observe from here. Under + /// lockstep rotation, every node polling its own gauge to 0 approximates + /// that; treat your local reading as a fleet proxy, never as a measurement + /// of what your seal will reject. A local 0 while one congested peer still + /// holds 800 old-masked packets is a green light that means nothing. + /// + /// The retry queue is the long pole: packets are masked BEFORE enqueueing, + /// re-sent verbatim, and the queue is capped by count (1024) with no time + /// bound — it drains as the interface unblocks and no faster. + /// + /// **Two clocks.** `install → activate` waits on key DISTRIBUTION (the + /// 300 s convergence window; activating early only degrades stragglers). + /// `activate → seal` waits on packet DRAIN (this 120 s dwell; failure is + /// silent). They measure different things — do not set them equal. + /// + /// The dwell is PER-NODE, from that node's own activate; a fleet need not + /// seal in lockstep. With no better number, wait + /// [`Self::default_ifac_rotation_dwell_ms`] (120 000) after every node's + /// `retry_queued` has hit 0. leviculum's 500 ms is a loopback floor. + /// CIRISEdge#568 / leviculum#63 — `(retry_queued, retry_queue_cap, + /// retry_dropped_total)` for this node's transport. + /// + /// Exposed because [`Self::ifac_seal_rotation`] tells the operator to wait + /// for the retry queue to drain, and until now this number existed only + /// inside Rust. **Direction matters:** this is THIS node's OUTBOUND + /// backlog. Zero here is what this node's PEERS need before they seal; it + /// is a fleet proxy, not a measurement of what this node's own seal will + /// reject. See `ifac_seal_rotation`. + /// + /// Also the attribution signal for a slow replication round: a round that + /// took 150 s with a flat queue and one that took 150 s while + /// `retry_dropped_total` climbed are different bugs with the same stopwatch + /// reading. + fn retry_queue_gauges(&self) -> PyResult<(usize, usize, u64)> { + #[cfg(feature = "_reticulum-module")] + { + let transport = self.inner.reticulum_transport().ok_or_else(|| { + PyRuntimeError::new_err("retry_queue_gauges: edge has no Reticulum transport") + })?; + Ok(transport.retry_queue_gauges()) + } + #[cfg(not(feature = "_reticulum-module"))] + { + Err(PyRuntimeError::new_err( + "retry_queue_gauges: requires the _reticulum-module feature", + )) + } + } + + /// leviculum#52 — the default dwell between `ifac_activate_next` and + /// `ifac_seal_rotation`, in milliseconds. A conservative choice, not a + /// derivation: the drain window is bounded by peers' retry-queue + /// residency, which has no time bound (count-capped, drains as the + /// interface unblocks). It reuses edge's resource-transfer timeout (120 s) + /// because that is already edge's committed answer to how long traffic can + /// legitimately remain in flight — not because a transfer's duration is the + /// drain window; old-masked bytes stop being generated at `activate`. + /// + /// The SAFE default, not the right number for every rotation — ejecting a + /// compromised member is a reason to go lower on purpose and accept the + /// drops. See `ifac_seal_rotation` for what must drain and what edge can + /// observe of it. + #[staticmethod] + fn default_ifac_rotation_dwell_ms() -> u64 { + u64::try_from(crate::transport::reticulum::DEFAULT_IFAC_ROTATION_DWELL.as_millis()) + .unwrap_or(u64::MAX) + } + fn ifac_seal_rotation(&self) -> PyResult { #[cfg(feature = "_reticulum-module")] { @@ -9043,6 +9198,40 @@ fn ciris_edge(m: &Bound<'_, PyModule>) -> PyResult<()> { register(m) } +/// CIRISEdge#573 — the signer `build_signed_inbound_envelope` signs with. +/// +/// Extracted from the pyo3 method so a Rust test can drive the SAME +/// construction the wheel uses, rather than a copy of it that can drift. The +/// helper shipped a hard-coded classical-only signer from v19.0.0 — when every +/// signature became the full hybrid — until #573, so it could not build any +/// envelope at all. Nothing in this crate caught it, because the only caller is +/// Python. +/// +/// `pqc_seed` is `None`-able ON PURPOSE: the classical-only refusal must stay +/// reachable and loud. That path gets `sign_envelope`'s own "no ML-DSA-65 (PQC) +/// half" error, which is the correct answer rather than a silent downgrade. +fn software_hybrid_signer( + key_id: &str, + seed: &[u8; 32], + pqc_seed: Option<[u8; 32]>, +) -> Result { + let mut sw = ciris_keyring::Ed25519SoftwareSigner::new(key_id); + sw.import_key(seed) + .map_err(|e| format!("import_key failed: {e}"))?; + let pqc: Option> = match pqc_seed { + None => None, + Some(ps) => Some(Arc::new( + ciris_keyring::MlDsa65SoftwareSigner::from_seed_bytes(&ps, format!("{key_id}-pqc")) + .map_err(|e| format!("ml_dsa_65 from seed failed: {e}"))?, + ) as Arc), + }; + Ok(crate::identity::LocalSigner::new( + key_id.to_owned(), + Arc::new(sw) as Arc, + pqc, + )) +} + #[cfg(test)] #[cfg(feature = "transport-reticulum")] mod tests { @@ -9063,6 +9252,101 @@ mod tests { use super::*; use crate::transport::{InboundFrame, TransportId, TransportSendOutcome}; + // ── CIRISEdge#573 — the pyo3 envelope helper signs HYBRID ──────── + // + // The bug: v19.0.0 made every signature the full Ed25519 + ML-DSA-65 + // hybrid, and `build_signed_inbound_envelope` was not moved with it — it + // built a classical-only signer, so it could not produce ANY envelope on + // the wheel that ships it. Its Rust twin + // (`tests/trust_short_circuit.rs::FedKey::local_signer`) HAD been moved, + // which is exactly why nothing here noticed: the two codepaths the doc + // calls identical had silently stopped being so, and the only caller of + // this one is Python. + // + // These drive `software_hybrid_signer` — the function the pyo3 method + // itself calls — rather than a reconstruction of it. + + #[tokio::test] + async fn the_envelope_helper_signs_with_the_pqc_half() { + let seed = [7u8; 32]; + let pqc_seed = [9u8; 32]; + let signer = super::software_hybrid_signer("node-573", &seed, Some(pqc_seed)) + .expect("build the hybrid signer"); + + let mut env = crate::identity::build_envelope( + crate::messages::MessageType::OpaqueEvent, + "node-573", + "node-dest", + &serde_json::json!({ "kind": 7, "payload": "eA==" }), + None, + ) + .expect("build_envelope"); + + crate::identity::sign_envelope(&signer, &mut env) + .await + .expect("a hybrid signer signs — this is the #573 regression"); + + assert!( + !env.signature.is_empty(), + "the classical half must still be present" + ); + assert!( + env.signature_pqc.is_some(), + "the ML-DSA-65 half must be present — an envelope without it is \ + refused at the far end, which is what made every conformance \ + probe through this helper undrivable" + ); + } + + #[tokio::test] + async fn without_a_pqc_seed_the_helper_still_refuses_loudly() { + // The `None` arm is kept REACHABLE on purpose. It must fail at + // `sign_envelope` naming the missing half — never quietly emit a + // classical-only envelope that dies at someone else's verifier. + let signer = super::software_hybrid_signer("node-573-classical", &[7u8; 32], None) + .expect("a classical-only signer still CONSTRUCTS"); + let mut env = crate::identity::build_envelope( + crate::messages::MessageType::OpaqueEvent, + "node-573-classical", + "node-dest", + &serde_json::json!({ "kind": 7, "payload": "eA==" }), + None, + ) + .expect("build_envelope"); + let err = crate::identity::sign_envelope(&signer, &mut env) + .await + .expect_err("classical-only must refuse"); + let msg = err.to_string(); + assert!( + msg.contains("has no ML-DSA-65 (PQC) half"), + "the refusal must name the missing half — CIRISConformance keys an \ + imperative xfail on this exact token: {msg}" + ); + } + + #[test] + fn the_derived_pubkey_is_the_one_the_signer_signs_with() { + // Closes the register/sign loop: a harness registers this pubkey in + // `federation_keys` and signs with the same seed. If these two ever + // disagree the only symptom is a verify refusal at the far end with + // nothing in it pointing here. + use base64::Engine as _; + let pqc_seed = [9u8; 32]; + let derived = + super::PyEdge::derive_ml_dsa_65_pubkey_base64(&pqc_seed).expect("derive the pubkey"); + let signer = + super::software_hybrid_signer("node-573", &[7u8; 32], Some(pqc_seed)).expect("signer"); + let actual = base64::engine::general_purpose::STANDARD.encode( + futures::executor::block_on(signer.pqc.as_ref().expect("pqc half").public_key()) + .expect("public_key"), + ); + assert_eq!( + derived, actual, + "derive_ml_dsa_65_pubkey_base64 must yield the key the signer signs \ + with, or the registered row cannot verify the envelope" + ); + } + /// Stub transport — exercises the [`EdgeBuilder`] build path /// without binding a real socket. Used by /// [`edge_assembles_from_persist_2x_handles`] below. diff --git a/src/transport/reticulum.rs b/src/transport/reticulum.rs index 5cdb9b0..32a0c89 100644 --- a/src/transport/reticulum.rs +++ b/src/transport/reticulum.rs @@ -292,6 +292,49 @@ fn effective_control_channel_capacity( /// complete after the link is up. const RESOURCE_TRANSFER_TIMEOUT: Duration = Duration::from_secs(120); +/// leviculum#52 — the dwell an operator should leave between +/// `ifac_activate_next` and `ifac_seal_rotation` when they have no better +/// number. **Anchored to [`RESOURCE_TRANSFER_TIMEOUT`], not chosen.** +/// +/// Sealing retires the old IFAC key for inbound, so it rejects any packet still +/// masked with it that arrives from a peer. Old-masked bytes stop being +/// GENERATED at each peer's `activate` — every packet dispatched after it is +/// new-masked, and a receiver accepts those because it installed the new key +/// in phase 1. So the drain window is not the duration of any transfer; a +/// resource straddling `activate` is simply part old-masked and part new. (An +/// earlier draft of this comment claimed leviculum#62 made a whole transfer +/// old-masked at once. It does not — #62 is receiver-side reassembly and says +/// nothing about when the sender masks.) +/// +/// What the window IS bounded by is retry-queue residency, and that has **no +/// time bound at all**. The queue is a bare `VecDeque>` with no age, +/// TTL, or expiry on any entry; `drain_retry_queues` is head-of-line and +/// rate-gated by the interface's `next_slot_ms` (airtime pacing on LoRa). A +/// packet leaves only by successful send, cap-overflow eviction of the OLDEST +/// (cap 1024), interface disconnect, or interface removal — nothing times it +/// out. So the cap bounds DEPTH, not time: a packet stuck at the front of a +/// quiet link leaves only once 1024 more arrive behind it, which is +/// arbitrarily long. Residency is bounded by neither a clock nor any +/// deterministic quantity, so there is no principled number to derive. +/// +/// This is therefore a conservative choice and not a derivation. It reuses +/// the resource transfer timeout because that is the longest window this +/// file already treats as survivable in-flight — not because the two +/// mechanisms are the same. 10 s was proposed; 10 s is shorter than every +/// in-flight window this file already asserts (5 s / 30 s / 30 s / 120 s). +/// Not to be confused with [`crate::scope_lifecycle::DEFAULT_CONVERGENCE_WINDOW`] +/// (300 s), the install→activate DISTRIBUTION clock; see `ifac_seal_rotation`. +/// +/// This is the SAFE default, not the right number for every rotation. An +/// operator ejecting a compromised member should override it downward on +/// purpose, accepting the drops — that is a security decision that depends on +/// why the rotation is happening, which is exactly why it is a constant to +/// deviate from and not a helper that decides for them. leviculum's own 500 ms +/// is a loopback floor upstream explicitly disclaims as a deployment value; +/// nobody in the ecosystem ships a number, because upstream Reticulum's IFAC +/// is static config with no rotation ceremony at all. +pub const DEFAULT_IFAC_ROTATION_DWELL: Duration = RESOURCE_TRANSFER_TIMEOUT; + /// CIRISEdge#353 — the classified outcome of shipping a resource on a link. /// `Busy` is the retryable one-transfer-per-link collision /// (`ResourceError::TransferInProgress`); `Other` is any other send failure. @@ -2397,6 +2440,7 @@ impl Default for ReticulumAuth { /// CIRISEdge#436 — this node's own validated build-attestation bundle, pinned /// at construction: the pre-encoded `CBND` frame served on link-up plus the /// manifest commitment every announce carries. +#[derive(Clone)] struct OwnBuildBundle { /// The bundle bytes pre-wrapped as a `CBND` v1 frame (encode once). frame: Vec, @@ -2405,6 +2449,51 @@ struct OwnBuildBundle { } impl ReticulumTransport { + /// CIRISEdge#568 — leviculum#63's retry-queue gauges: `(queued, cap, + /// dropped_total)`. + /// + /// The queue climbs before it discards, and until leviculum v0.25.0 the + /// only signal was the first drop — so a node shedding traffic read as a + /// quiet node with slow rounds. That is indistinguishable from the + /// cold-dial stall this release also fixes, which is exactly why both + /// numbers need to be on the artifact: a round that took 150 s with a flat + /// retry queue and one that took 150 s while `dropped_total` climbed are + /// different bugs with the same stopwatch reading. + /// + /// **Direction, for rotation callers:** this is THIS node's OUTBOUND + /// backlog. Zero here says this node has nothing further old-masked to + /// send — which is what its PEERS need to know before they seal, not what + /// this node needs to know before it seals. See [`Self::ifac_seal_rotation`]. + #[must_use] + pub fn retry_queue_gauges(&self) -> (usize, usize, u64) { + let s = self.node.plane_stats(); + (s.retry_queued, s.retry_queue_cap, s.retry_dropped_total) + } + + /// CIRISEdge#568 — a cloneable handle to the state a cold dial needs, so + /// the dial can be spawned and survive its round being abandoned. + fn dial_ctx(&self) -> DialCtx { + DialCtx { + node: Arc::clone(&self.node), + local_identity: self.local_identity.clone(), + own_bundle: self.own_bundle.clone(), + dialed_link_dest: Arc::clone(&self.dialed_link_dest), + reusable_dialed_link: Arc::clone(&self.reusable_dialed_link), + link_in_flight: Arc::clone(&self.link_in_flight), + } + } + + /// Delegates to [`DialCtx::reusable_link_to`] — the pool lives on the ctx + /// now, but every existing caller keeps its shape. + async fn reusable_link_to(&self, dest: &DestinationHash) -> Option { + self.dial_ctx().reusable_link_to(dest).await + } + + /// Delegates to [`DialCtx::path_table_snapshot`]. + fn path_table_snapshot(&self) -> String { + self.dial_ctx().path_table_snapshot() + } + /// Construct + start the transport: load-or-generate the /// transport identity, build the Leviculum node with the /// configured TCP interfaces, register edge's own federation @@ -3104,8 +3193,75 @@ impl ReticulumTransport { /// CIRISEdge#492 — phase 3: SEAL the rotation — drop the old IFAC code. Any /// member that never re-keyed is now excluded (readmission requires a fresh - /// grant + re-key). Call after the convergence window. Returns the number of - /// interfaces affected. + /// grant + re-key). Returns the number of interfaces affected. + /// + /// # Leave a dwell after `ifac_activate_next` (leviculum#52) + /// + /// This used to say "call after the convergence window", which named the + /// wrong hazard. That window is about MEMBERSHIP — every member holding the + /// new key. Sealing also retires the old key for INBOUND, so it rejects + /// packets already on the wire under the old mask, and those were sent by + /// the members that DID re-key. An operator who waited for membership + /// convergence and sealed still strands traffic; upstream measured a + /// zero-dwell rotation losing its packet in ~50% of runs, with the peer's + /// `drops_ifac` incrementing exactly once each time. + /// + /// `install` and `activate` are make-before-break and cannot lose a packet. + /// **Only sealing breaks**, and nothing in leviculum imposes the dwell — + /// each phase is an explicit call so the cutover moment stays the + /// operator's. + /// + /// **Which direction seal breaks — read this before trusting any gauge.** + /// `seal` drops the alternate key, and the alternate is consulted in + /// exactly one place in leviculum: `verify_ifac`, the INBOUND path. + /// `apply_ifac` (outbound) never reads it. So sealing changes only what + /// THIS node accepts; what it rejects is old-masked traffic **arriving + /// from peers**. Upstream's failing run showed `drops_ifac` at the + /// *relay* — the relay's seal rejecting a member's in-flight outbound. + /// + /// **What must drain.** Four things carry the retired mask: each peer's + /// retry queue, its socket buffer, bytes in flight, and this node's own + /// receive buffer. The retry queue is the long pole: leviculum masks + /// BEFORE enqueueing and re-sends the queued bytes verbatim, so a packet + /// enqueued before a peer's `activate` stays old-masked for its whole + /// residency — and residency has no time bound, only a count cap (1024). + /// It drains as fast as the interface unblocks and no faster. + /// + /// **What edge can see, and what it is a proxy for.** + /// [`Self::retry_queue_gauges`]'s `retry_queued` counts THIS node's + /// OUTBOUND backlog. Reaching zero therefore licenses this node's PEERS + /// to seal — not this node. The true precondition for this node sealing + /// is "every peer's `retry_queued` is zero", which edge cannot observe. + /// Under operator-driven lockstep rotation — the only mode edge supports, + /// since nothing here sequences the phases — every node polling its own + /// gauge to zero approximates that fleet-wide condition, and the local + /// reading is a reasonable proxy. It is NOT a measurement of what this + /// node's seal will reject: a local zero while one congested peer still + /// holds 800 old-masked packets is a green light that means nothing. + /// + /// **Two clocks, do not harmonize them.** A rotation has two dwells that + /// measure different things, and this repo carries a constant for each: + /// + /// - `install → activate` is bounded by key DISTRIBUTION — the membership + /// clock, [`crate::scope_lifecycle::DEFAULT_CONVERGENCE_WINDOW`] (300 s). + /// Activating early is not a safety failure: upstream keeps the old key + /// accept-only, so a straggler's outbound still lands while its inbound + /// degrades until it upgrades. + /// - `activate → seal` is bounded by packet DRAIN — this clock, + /// [`DEFAULT_IFAC_ROTATION_DWELL`] (120 s), and the only one whose + /// failure is silent. + /// + /// The pre-fix prose collapsed both into "the convergence window", which + /// is how the hazard was mis-framed. A later cleanup that notices two + /// dwell constants and makes them equal would reintroduce it. + /// + /// **The dwell is per-node**, measured from THAT node's own `activate`. A + /// node's seal affects only its own inbound, so a fleet need not seal in + /// lockstep — but no node may seal early relative to its own activate. + /// + /// With no better number, wait [`DEFAULT_IFAC_ROTATION_DWELL`] (120 s) + /// after every node's `retry_queued` has reached zero. leviculum's own + /// harness uses 500 ms and explicitly disclaims it as a loopback floor. #[must_use] pub fn ifac_seal_rotation(&self) -> usize { self.node.ifac_seal_rotation() @@ -4167,58 +4323,6 @@ impl ReticulumTransport { None } - /// Snapshot every known path-table entry. v1.1.0 (CIRISEdge#44) — - /// backed by leviculum's now-public - /// `ReticulumNode::path_table_entries` (each row is a deep - /// `PathTableExport` clone; no mutex-borrowed references escape). - /// - /// `max_hops` filters the result to entries whose `hops <= max_hops` - /// when supplied. `None` returns the full table. The `peer_key_id` - /// field is filled when the destination matches a currently-rooted - /// peer (CIRISEdge#15 cold-start authenticated path); unknown / - /// relay destinations get `None`. - /// - /// Timestamps are wall-clock projections of leviculum's monotonic - /// `expires_ms` — see [`Self::project_monotonic_ms`] for the - /// precision contract. `last_seen_at` is the call-time wall - /// clock (path entries don't carry an insertion timestamp in - /// leviculum's storage shape). - /// CIRISEdge#336 — a compact, single-line snapshot of the node's path - /// table for the [`TransportError::NoRouteToPeer`] diagnostic. Each entry - /// renders as `dest via next_hop hops=N` (or `dest direct hops=N` for a - /// directly-attached neighbor). Synchronous and lock-free — - /// `path_table_entries()` clones leviculum's rows — so it is safe to call - /// on the send failure path. Bounded to a handful of rows so an enormous - /// fabric can't turn one failure into a megabyte log line. - fn path_table_snapshot(&self) -> String { - use std::fmt::Write as _; - const MAX_ROWS: usize = 16; - let rows = self.node.path_table_entries(); - let total = rows.len(); - let mut out = String::new(); - for (i, entry) in rows.iter().take(MAX_ROWS).enumerate() { - if i > 0 { - out.push_str(", "); - } - let dest = hex::encode(entry.hash); - match entry.next_hop { - Some(nh) if entry.hops > 1 => { - let _ = write!(out, "{dest} via {} hops={}", hex::encode(nh), entry.hops); - } - _ => { - let _ = write!(out, "{dest} direct hops={}", entry.hops); - } - } - } - if total > MAX_ROWS { - let _ = write!(out, ", …(+{} more)", total - MAX_ROWS); - } - if total == 0 { - out.push_str(""); - } - out - } - #[cfg(feature = "ffi-uniffi")] pub async fn routing_path_table( &self, @@ -4848,151 +4952,6 @@ impl ReticulumTransport { None } - /// CIRISEdge#353 — the newest LIVE link already attributed to this peer - /// (the reverse path). Scans `link_to_peer_key_id` — populated by - /// `LinkIdentified` (the peer dialed + identified to us) — and keeps only - /// links leviculum still holds `Active` (`link_is_established` resolves - /// the #66 re-key alias, so a re-keyed inbound link still matches). - /// Newest-established wins when a peer holds several. - /// CIRISEdge#532 — dial a peer and bring the link all the way to USABLE: - /// established, identified, and bundle-served. Extracted from `send` so the - /// reuse and single-flight decisions above read as one choice instead of - /// being threaded through a linear sequence. - /// - /// On success the link is published to `reusable_dialed_link`, which is what - /// lets the next send — and every other coordinator's send to this peer — - /// skip all of this. - async fn dial_and_identify( - &self, - destination_key_id: &str, - peer: &ResolvedPeer, - has_path: bool, - establish_timeout: Duration, - ) -> Result { - // CIRISEdge#484 — leviculum v0.16 `connect_awaited` returns the handle - // immediately AND a completion future for `LinkEstablished`, registered - // BEFORE dispatch (edge no longer needs to observe the event loop `listen` - // owns). The future keys on the ORIGINAL dial id — the #342/#66 alias the old - // `link_is_established` poll resolved — takes NO node lock, and resolves - // `Err(LinkClosed)` on link death. Caller owns the wall-clock bound. - let (link, established) = self - .node - .connect_awaited(&peer.dest_hash, &peer.signing_key) - .await - .map_err(|e| TransportError::Io(format!("reticulum connect: {e}")))?; - let link_id = *link.link_id(); - // CIRISEdge#424 — record the dest we dialed for THIS link so an inbound - // reply arriving over it (the initiator-side reverse path a NAT'd peer's - // responder uses) attributes to this peer even though leviculum's - // `link_destination` returns `None` for our own dialed links. - self.dialed_link_dest - .lock() - .await - .insert(link_id, peer.dest_hash); - - // Await `LinkEstablished` on BOTH ends — the peer must have accepted the - // LINK_REQUEST or a resource transfer cannot start. `established` resolves - // `Ok(())` on establishment, `Err(LinkClosed)` if the peer refused / the link - // died first; a timeout means no route or a stalled dial. - let established_ok = matches!( - with_timeout(establish_timeout, established).await, - Some(Ok(())) - ); - if !established_ok { - // CIRISEdge#336 — a no-path target that never established is - // un-routable, not slow: fail fast with the self-diagnosing error - // (naming target dest, key_id, and the paths we DO hold — the - // routable named dest for this peer usually appears there, making - // the explicit-vs-named mismatch obvious). A had-a-path target that - // stalled is a genuine slow/dead link → the opaque timeout stands. - if !has_path { - let target_dest = hex::encode(peer.dest_hash.into_bytes()); - let paths = self.path_table_snapshot(); - tracing::error!( - key_id = %destination_key_id, - target_dest = %target_dest, - has_path, - known_paths = %paths, - "link_request target has no route — un-routable dest (CIRISEdge#336). \ - A no-path dest is broadcast-only and no directly-attached neighbor \ - answered; if the peer is relay-reachable it must be addressed on its \ - announced (named) dest, which appears in known_paths." - ); - return Err(TransportError::NoRouteToPeer { - key_id: destination_key_id.to_string(), - target_dest, - has_path, - paths, - }); - } - log_nat_topology_diagnosis(destination_key_id, establish_timeout); - return Err(TransportError::Timeout(establish_timeout)); - } - - // CIRISEdge#340 — IDENTIFY the link before sending. A Reticulum link is - // anonymous by default; only the initiator may identify it, and the - // responder emits `LinkIdentified` (→ populates its `link_to_peer_key_id` - // via the #314 identity-hash match → attributes our inbound frame) ONLY - // if we do. Without this, every replication frame we send lands on the - // responder as `source_key_id=None` and is dropped `SkippedNoSourceKeyId` - // (#317) — the field-confirmed reason attribution never fired and - // CIRISServer#235 was never verified end-to-end. Ordered before - // `send_resource` on the same link so the LINKIDENTIFY is processed - // first. A failure here means the responder cannot attribute the frame, - // so fail the send (the durable dispatcher retries) rather than ship an - // unattributable resource that will be silently dropped. - self.node - .identify_link(&link_id, &self.local_identity) - .await - .map_err(|e| TransportError::Io(format!("reticulum identify_link: {e}")))?; - - // CIRISEdge#436 — initiator-side bundle serve, ordered AFTER the - // LINKIDENTIFY (so the responder attributes it) and BEFORE the - // resource ship (the fragments ride the link Channel, which never - // contends with the resource lane — leviculum#27). - if let Some(own) = self.own_bundle.as_ref() { - push_own_bundle_frames(&self.node, own, &link_id).await; - } - - // PUBLISH LAST. Only now is the link established AND identified AND - // bundle-served — the three things a reusing sender skips. Publishing - // any earlier would hand another coordinator a link the responder will - // drop frames on (`SkippedNoSourceKeyId`, #317/#340). - self.reusable_dialed_link - .lock() - .await - .entry(peer.dest_hash) - .or_default() - .push(link_id); - - Ok(link_id) - } - - /// CIRISEdge#532 — the live, identified link we already hold to this dest, - /// if any. `None` means a dial is required. - /// - /// Liveness uses the same `link_is_established` gate the reverse-path - /// selector does. A stale entry is EVICTED on the way out rather than left - /// to fail the next send too — the map is a cache over leviculum's link - /// registry, and leviculum is the authority. - async fn reusable_link_to(&self, dest: &DestinationHash) -> Option { - let mut map = self.reusable_dialed_link.lock().await; - let pool = map.get_mut(dest)?; - // Drop links leviculum no longer holds, so a dead entry cannot occupy a - // pool slot forever. The map is a cache over the link registry and the - // registry is the authority. - pool.retain(|id| self.node.link_is_established(id)); - if pool.is_empty() { - map.remove(dest); - return None; - } - let in_flight = self.link_in_flight.lock().await; - // IDLE only. A link mid-transfer is not available: Reticulum runs one - // resource per link, so handing it out serialises the caller behind the - // transfer already on it — which is the regression the M=4 sweep caught. - pool.iter().find(|id| !in_flight.contains(*id)).copied() - } - /// CIRISEdge#532 — claim a link for a transfer, or report it already busy. /// /// Returned as a bool the caller must honour rather than a guard type @@ -5604,26 +5563,53 @@ impl Transport for ReticulumTransport { // peer all observe "no link" and all dial — the establish:identify // gap in #532, where the cost of a link is paid and then discarded. // The gate is per-destination so other peers still dial in parallel. + // CIRISEdge#568 — the dial runs in a SPAWNED task, not inline. + // + // `round_timeout` is 10 s; a pathed peer's `establish_timeout` is + // `LINK_ESTABLISH_TIMEOUT` (30 s). Inline, a cold dial could never + // finish inside the round that started it, and the scheduler + // abandoning that round dropped the half-built link — so every + // retry began cold and the peer converged only when IT dialed US. + // Measured on the two-node ladder: the first Attestation round to a + // fresh peer timed out at +10 s and the next completed round landed + // ~150 s (five cadence ticks) later. + // + // Detached, an abandoned round abandons the WAIT and not the LINK: + // the task runs to completion and publishes into + // `reusable_dialed_link` (the "PUBLISH LAST" step in + // `dial_and_identify`), so the next round takes the fast path above + // instead of dialling cold again. + // + // The gate permit is acquired INSIDE the task on purpose. Held by + // the caller, dropping the caller would release it while the dial + // it is meant to gate is still running — the single-flight property + // #532 installed would hold only for callers that survive. let gate = self.dial_gate_for(peer.dest_hash).await; - let _dial_permit = gate - .acquire() - .await - .map_err(|e| TransportError::Io(format!("dial gate closed: {e}")))?; - // Double-check: a concurrent dial we queued behind may have just - // published a reusable link. Checking only before the gate would - // make the gate a queue of redundant dials rather than a filter. - if let Some(existing) = self.reusable_link_to(&peer.dest_hash).await { - tracing::trace!( - key_id = %destination_key_id, - link = ?existing, - "another send established this peer's link while we waited on \ - the dial gate — reusing it (CIRISEdge#532)" - ); - existing - } else { - self.dial_and_identify(destination_key_id, &peer, has_path, establish_timeout) - .await? - } + let ctx = self.dial_ctx(); + let peer_owned = peer; + let dkid = destination_key_id.to_owned(); + let dial = tokio::spawn(async move { + let _dial_permit = gate + .acquire() + .await + .map_err(|e| TransportError::Io(format!("dial gate closed: {e}")))?; + // Double-check: a concurrent dial we queued behind may have just + // published a reusable link. Checking only before the gate would + // make the gate a queue of redundant dials rather than a filter. + if let Some(existing) = ctx.reusable_link_to(&peer_owned.dest_hash).await { + tracing::trace!( + key_id = %dkid, + link = ?existing, + "another send established this peer's link while we waited on \ + the dial gate — reusing it (CIRISEdge#532)" + ); + return Ok(existing); + } + ctx.dial_and_identify(&dkid, &peer_owned, has_path, establish_timeout) + .await + }); + dial.await + .map_err(|e| TransportError::Io(format!("dial task failed: {e}")))?? }; // CIRISEdge#532 — hold the link for the duration of the transfer, so a @@ -5855,6 +5841,233 @@ impl Transport for ReticulumTransport { } } +/// CIRISEdge#568 — the state a cold dial needs, cloneable so the dial can +/// OUTLIVE the round that asked for it. +/// +/// The defect this exists to fix: `dial_and_identify` ran inline inside the +/// caller's future, and the scheduler abandons a round at `round_timeout` +/// (10 s) while a pathed peer's `LINK_ESTABLISH_TIMEOUT` is 30 s. So a cold +/// dial could never finish inside the round that started it, and abandoning +/// the round DROPPED the half-established link — the round paid a link's cost +/// and discarded it. That is the #532 establish:identify gap one layer up: +/// #532 stopped N coordinators discarding each other's links; nothing stopped +/// a round discarding its own. +/// +/// Every field is an `Arc` or a cheap clone, so handing a copy to a spawned +/// task costs a few refcount bumps. Deliberately a context struct rather than +/// a `Weak` on the transport: `ReticulumTransport::new` returns `Self`, +/// so a self-handle would need installing at every construction site, and a +/// site that forgot would silently lose the detach — a flag that lets two +/// equal values diverge. +#[derive(Clone)] +struct DialCtx { + node: Arc, + local_identity: Identity, + own_bundle: Option, + dialed_link_dest: Arc>>, + reusable_dialed_link: Arc>>>, + link_in_flight: Arc>>, +} + +impl DialCtx { + /// Snapshot every known path-table entry. v1.1.0 (CIRISEdge#44) — + /// backed by leviculum's now-public + /// `ReticulumNode::path_table_entries` (each row is a deep + /// `PathTableExport` clone; no mutex-borrowed references escape). + /// + /// `max_hops` filters the result to entries whose `hops <= max_hops` + /// when supplied. `None` returns the full table. The `peer_key_id` + /// field is filled when the destination matches a currently-rooted + /// peer (CIRISEdge#15 cold-start authenticated path); unknown / + /// relay destinations get `None`. + /// + /// Timestamps are wall-clock projections of leviculum's monotonic + /// `expires_ms` — see [`Self::project_monotonic_ms`] for the + /// precision contract. `last_seen_at` is the call-time wall + /// clock (path entries don't carry an insertion timestamp in + /// leviculum's storage shape). + /// CIRISEdge#336 — a compact, single-line snapshot of the node's path + /// table for the [`TransportError::NoRouteToPeer`] diagnostic. Each entry + /// renders as `dest via next_hop hops=N` (or `dest direct hops=N` for a + /// directly-attached neighbor). Synchronous and lock-free — + /// `path_table_entries()` clones leviculum's rows — so it is safe to call + /// on the send failure path. Bounded to a handful of rows so an enormous + /// fabric can't turn one failure into a megabyte log line. + fn path_table_snapshot(&self) -> String { + use std::fmt::Write as _; + const MAX_ROWS: usize = 16; + let rows = self.node.path_table_entries(); + let total = rows.len(); + let mut out = String::new(); + for (i, entry) in rows.iter().take(MAX_ROWS).enumerate() { + if i > 0 { + out.push_str(", "); + } + let dest = hex::encode(entry.hash); + match entry.next_hop { + Some(nh) if entry.hops > 1 => { + let _ = write!(out, "{dest} via {} hops={}", hex::encode(nh), entry.hops); + } + _ => { + let _ = write!(out, "{dest} direct hops={}", entry.hops); + } + } + } + if total > MAX_ROWS { + let _ = write!(out, ", …(+{} more)", total - MAX_ROWS); + } + if total == 0 { + out.push_str(""); + } + out + } + + /// CIRISEdge#353 — the newest LIVE link already attributed to this peer + /// (the reverse path). Scans `link_to_peer_key_id` — populated by + /// `LinkIdentified` (the peer dialed + identified to us) — and keeps only + /// links leviculum still holds `Active` (`link_is_established` resolves + /// the #66 re-key alias, so a re-keyed inbound link still matches). + /// Newest-established wins when a peer holds several. + /// CIRISEdge#532 — dial a peer and bring the link all the way to USABLE: + /// established, identified, and bundle-served. Extracted from `send` so the + /// reuse and single-flight decisions above read as one choice instead of + /// being threaded through a linear sequence. + /// + /// On success the link is published to `reusable_dialed_link`, which is what + /// lets the next send — and every other coordinator's send to this peer — + /// skip all of this. + async fn dial_and_identify( + &self, + destination_key_id: &str, + peer: &ResolvedPeer, + has_path: bool, + establish_timeout: Duration, + ) -> Result { + // CIRISEdge#484 — leviculum v0.16 `connect_awaited` returns the handle + // immediately AND a completion future for `LinkEstablished`, registered + // BEFORE dispatch (edge no longer needs to observe the event loop `listen` + // owns). The future keys on the ORIGINAL dial id — the #342/#66 alias the old + // `link_is_established` poll resolved — takes NO node lock, and resolves + // `Err(LinkClosed)` on link death. Caller owns the wall-clock bound. + let (link, established) = self + .node + .connect_awaited(&peer.dest_hash, &peer.signing_key) + .await + .map_err(|e| TransportError::Io(format!("reticulum connect: {e}")))?; + let link_id = *link.link_id(); + // CIRISEdge#424 — record the dest we dialed for THIS link so an inbound + // reply arriving over it (the initiator-side reverse path a NAT'd peer's + // responder uses) attributes to this peer even though leviculum's + // `link_destination` returns `None` for our own dialed links. + self.dialed_link_dest + .lock() + .await + .insert(link_id, peer.dest_hash); + + // Await `LinkEstablished` on BOTH ends — the peer must have accepted the + // LINK_REQUEST or a resource transfer cannot start. `established` resolves + // `Ok(())` on establishment, `Err(LinkClosed)` if the peer refused / the link + // died first; a timeout means no route or a stalled dial. + let established_ok = matches!( + with_timeout(establish_timeout, established).await, + Some(Ok(())) + ); + if !established_ok { + // CIRISEdge#336 — a no-path target that never established is + // un-routable, not slow: fail fast with the self-diagnosing error + // (naming target dest, key_id, and the paths we DO hold — the + // routable named dest for this peer usually appears there, making + // the explicit-vs-named mismatch obvious). A had-a-path target that + // stalled is a genuine slow/dead link → the opaque timeout stands. + if !has_path { + let target_dest = hex::encode(peer.dest_hash.into_bytes()); + let paths = self.path_table_snapshot(); + tracing::error!( + key_id = %destination_key_id, + target_dest = %target_dest, + has_path, + known_paths = %paths, + "link_request target has no route — un-routable dest (CIRISEdge#336). \ + A no-path dest is broadcast-only and no directly-attached neighbor \ + answered; if the peer is relay-reachable it must be addressed on its \ + announced (named) dest, which appears in known_paths." + ); + return Err(TransportError::NoRouteToPeer { + key_id: destination_key_id.to_string(), + target_dest, + has_path, + paths, + }); + } + log_nat_topology_diagnosis(destination_key_id, establish_timeout); + return Err(TransportError::Timeout(establish_timeout)); + } + + // CIRISEdge#340 — IDENTIFY the link before sending. A Reticulum link is + // anonymous by default; only the initiator may identify it, and the + // responder emits `LinkIdentified` (→ populates its `link_to_peer_key_id` + // via the #314 identity-hash match → attributes our inbound frame) ONLY + // if we do. Without this, every replication frame we send lands on the + // responder as `source_key_id=None` and is dropped `SkippedNoSourceKeyId` + // (#317) — the field-confirmed reason attribution never fired and + // CIRISServer#235 was never verified end-to-end. Ordered before + // `send_resource` on the same link so the LINKIDENTIFY is processed + // first. A failure here means the responder cannot attribute the frame, + // so fail the send (the durable dispatcher retries) rather than ship an + // unattributable resource that will be silently dropped. + self.node + .identify_link(&link_id, &self.local_identity) + .await + .map_err(|e| TransportError::Io(format!("reticulum identify_link: {e}")))?; + + // CIRISEdge#436 — initiator-side bundle serve, ordered AFTER the + // LINKIDENTIFY (so the responder attributes it) and BEFORE the + // resource ship (the fragments ride the link Channel, which never + // contends with the resource lane — leviculum#27). + if let Some(own) = self.own_bundle.as_ref() { + push_own_bundle_frames(&self.node, own, &link_id).await; + } + + // PUBLISH LAST. Only now is the link established AND identified AND + // bundle-served — the three things a reusing sender skips. Publishing + // any earlier would hand another coordinator a link the responder will + // drop frames on (`SkippedNoSourceKeyId`, #317/#340). + self.reusable_dialed_link + .lock() + .await + .entry(peer.dest_hash) + .or_default() + .push(link_id); + + Ok(link_id) + } + + /// CIRISEdge#532 — the live, identified link we already hold to this dest, + /// if any. `None` means a dial is required. + /// + /// Liveness uses the same `link_is_established` gate the reverse-path + /// selector does. A stale entry is EVICTED on the way out rather than left + /// to fail the next send too — the map is a cache over leviculum's link + /// registry, and leviculum is the authority. + async fn reusable_link_to(&self, dest: &DestinationHash) -> Option { + let mut map = self.reusable_dialed_link.lock().await; + let pool = map.get_mut(dest)?; + // Drop links leviculum no longer holds, so a dead entry cannot occupy a + // pool slot forever. The map is a cache over the link registry and the + // registry is the authority. + pool.retain(|id| self.node.link_is_established(id)); + if pool.is_empty() { + map.remove(dest); + return None; + } + let in_flight = self.link_in_flight.lock().await; + // IDLE only. A link mid-transfer is not available: Reticulum runs one + // resource per link, so handing it out serialises the caller behind the + // transfer already on it — which is the regression the M=4 sweep caught. + pool.iter().find(|id| !in_flight.contains(*id)).copied() + } +} + /// CIRISEdge#482 item 3 — the OWNED subset of transport state that /// [`resolve_announce_cold_start`] needs, so it can run on a dedicated worker /// task (fed from [`EventCtx::announce_tx`]) instead of inline on the single @@ -11394,6 +11607,72 @@ mod scope_native_addressing_tests { .expect("transport") } + // ── CIRISEdge#568 — the dial outlives the round ───────────────── + + /// **The load-bearing property of the detach.** A spawned dial publishes + /// its finished link into `reusable_dialed_link`; the next round reads that + /// same map through the transport. If `dial_ctx()` ever handed out COPIES + /// instead of shared handles, the dial would run to completion, publish + /// into a map nobody reads, and every round would still dial cold — the + /// fix would look applied and change nothing, with no error anywhere. + /// + /// Asserted by pointer identity rather than by round-tripping a synthetic + /// link, because `reusable_link_to` also asks the node whether the link is + /// alive; a fabricated `LinkId` is not, so a value-level round trip would + /// pass for the wrong reason. + #[tokio::test] + async fn the_dial_ctx_shares_the_transports_link_pool() { + let dir = tempfile::tempdir().expect("tempdir"); + let t = bare_transport(dir.path(), "node-568-ctx").await; + let ctx = t.dial_ctx(); + assert!( + Arc::ptr_eq(&ctx.reusable_dialed_link, &t.reusable_dialed_link), + "the pool a detached dial PUBLISHES into must be the pool the next \ + round READS — a copy here makes the whole detach a no-op" + ); + assert!( + Arc::ptr_eq(&ctx.dialed_link_dest, &t.dialed_link_dest), + "link→destination attribution must be shared, or a detached dial's \ + link arrives unattributable (#424)" + ); + assert!( + Arc::ptr_eq(&ctx.link_in_flight, &t.link_in_flight), + "in-flight tracking must be shared, or the liveness check cannot \ + see a detached dial's link" + ); + assert!( + Arc::ptr_eq(&ctx.node, &t.node), + "one node, or the dial establishes on a different stack entirely" + ); + } + + /// **Why the fix is a detach and not a bigger timeout.** + /// + /// The scheduler abandons a round at `DEFAULT_ROUND_TIMEOUT`; a peer with a + /// known path is allowed `LINK_ESTABLISH_TIMEOUT` to establish. The second + /// is larger, so a cold dial can NEVER complete inside the round that + /// started it — that is arithmetic, not load. + /// + /// This pins the relationship rather than the numbers. If someone later + /// "fixes" #568 by raising the round timeout past the establish budget and + /// removes the detach, this test goes green while the underlying waste + /// returns: a round that is abandoned for ANY other reason still discards + /// the link it paid for. The detach is what makes the cost non-discardable; + /// the inequality is only what made it visible. + #[test] + fn a_cold_dial_cannot_fit_inside_a_round() { + use crate::replication::scheduler::SchedulerConfig; + assert!( + SchedulerConfig::DEFAULT_ROUND_TIMEOUT < LINK_ESTABLISH_TIMEOUT, + "round_timeout {:?} vs establish {:?} — if this ever inverts, \ + re-read CIRISEdge#568 before deleting the detach: the detach is \ + not a workaround for the inequality, it is what stops an \ + abandoned round destroying a half-built link", + SchedulerConfig::DEFAULT_ROUND_TIMEOUT, + LINK_ESTABLISH_TIMEOUT, + ); + } + async fn bare_transport(dir: &std::path::Path, key_id: &str) -> ReticulumTransport { let signer = test_signer(key_id);