diff --git a/Cargo.lock b/Cargo.lock index 1b01947..2b5c648 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2350,7 +2350,7 @@ dependencies = [ [[package]] name = "digstore-chain" -version = "0.19.0" +version = "0.19.1" dependencies = [ "aes-gcm", "anyhow", @@ -2388,7 +2388,7 @@ dependencies = [ [[package]] name = "digstore-chunker" -version = "0.19.0" +version = "0.19.1" dependencies = [ "digstore-core", "hex", @@ -2398,7 +2398,7 @@ dependencies = [ [[package]] name = "digstore-cli" -version = "0.19.0" +version = "0.19.1" dependencies = [ "anstream 0.6.21", "anstyle", @@ -2466,7 +2466,7 @@ dependencies = [ [[package]] name = "digstore-core" -version = "0.19.0" +version = "0.19.1" dependencies = [ "aes-gcm-siv", "hex", @@ -2563,7 +2563,7 @@ dependencies = [ [[package]] name = "digstore-remote" -version = "0.19.0" +version = "0.19.1" dependencies = [ "async-trait", "axum", @@ -2621,7 +2621,7 @@ dependencies = [ [[package]] name = "digstore-subscription" -version = "0.19.0" +version = "0.19.1" dependencies = [ "async-trait", "digstore-core", diff --git a/Cargo.toml b/Cargo.toml index 5c91b2e..6fe9fc6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ exclude = ["crates/digstore-prover/guest", "crates/dig-client-wasm"] [workspace.package] edition = "2021" -version = "0.19.0" +version = "0.19.1" license = "GPL-2.0-only" [workspace.dependencies] diff --git a/crates/digstore-remote/tests/fixtures/golden/.gitattributes b/crates/digstore-remote/tests/fixtures/golden/.gitattributes new file mode 100644 index 0000000..9a93a1a --- /dev/null +++ b/crates/digstore-remote/tests/fixtures/golden/.gitattributes @@ -0,0 +1,4 @@ +# Golden read fixtures are byte-exact §5.1 anchors — git must NEVER touch their +# bytes (no CRLF/LF normalization on checkout, or the byte-stability test fails +# on Windows). Treat every file here as binary. +* -text diff --git a/crates/digstore-remote/tests/fixtures/golden/ciphertext.bin b/crates/digstore-remote/tests/fixtures/golden/ciphertext.bin new file mode 100644 index 0000000..42fe392 Binary files /dev/null and b/crates/digstore-remote/tests/fixtures/golden/ciphertext.bin differ diff --git a/crates/digstore-remote/tests/fixtures/golden/inclusion_proof.b64 b/crates/digstore-remote/tests/fixtures/golden/inclusion_proof.b64 new file mode 100644 index 0000000..91c5d17 --- /dev/null +++ b/crates/digstore-remote/tests/fixtures/golden/inclusion_proof.b64 @@ -0,0 +1 @@ +c/rwPPZhjGnQ4ldkhnwl7TCrdqA83BcstwLxhg2fdboAAAAAc/rwPPZhjGnQ4ldkhnwl7TCrdqA83BcstwLxhg2fdbo= \ No newline at end of file diff --git a/crates/digstore-remote/tests/fixtures/golden/manifest.json b/crates/digstore-remote/tests/fixtures/golden/manifest.json new file mode 100644 index 0000000..823bfe3 --- /dev/null +++ b/crates/digstore-remote/tests/fixtures/golden/manifest.json @@ -0,0 +1,13 @@ +{ + "canonical_urn_pinned": "urn:dig:chia:d9c8ae4b6006b5d2d82ecf53f84d74e4bfe7ec4e9fde2a0d71f058b14216ff9a:73faf03cf6618c69d0e25764867c25ed30ab76a03cdc172cb702f1860d9f75ba/index.html", + "canonical_urn_rootless": "urn:dig:chia:d9c8ae4b6006b5d2d82ecf53f84d74e4bfe7ec4e9fde2a0d71f058b14216ff9a/index.html", + "ciphertext_len": 270, + "description": "DIG Network #843 golden read fixture — one index.html capsule, derived deterministically from digstore_core. Shared by #843 (rpc read proof) and #1062 (P2P e2e). Do not fork a second copy.", + "live_at_rpc_dig_net": false, + "plaintext_len": 254, + "publish_followup": "Publish a real store under a captured launcher id, re-point GOLDEN_STORE_ID_HEX, then un-gate rpc_tier_read_proof (set DIG_RPC_LIVE).", + "resource_key": "index.html", + "retrieval_key": "49bfbe2f53ebdfab55d03062bef63f0f004ee83a342cb721f42df6c20883840e", + "root": "73faf03cf6618c69d0e25764867c25ed30ab76a03cdc172cb702f1860d9f75ba", + "store_id": "d9c8ae4b6006b5d2d82ecf53f84d74e4bfe7ec4e9fde2a0d71f058b14216ff9a" +} diff --git a/crates/digstore-remote/tests/fixtures/golden/plaintext.bin b/crates/digstore-remote/tests/fixtures/golden/plaintext.bin new file mode 100644 index 0000000..50e6961 --- /dev/null +++ b/crates/digstore-remote/tests/fixtures/golden/plaintext.bin @@ -0,0 +1,4 @@ + +DIG golden read fixture +

hello from the DIG Network

+

This is the #843 golden capsule: a single index.html resource whose ciphertext, inclusion proof, and root are derived deterministically from digstore_core.

diff --git a/crates/digstore-remote/tests/golden_read_proof.rs b/crates/digstore-remote/tests/golden_read_proof.rs new file mode 100644 index 0000000..471e621 --- /dev/null +++ b/crates/digstore-remote/tests/golden_read_proof.rs @@ -0,0 +1,348 @@ +//! READ-leg golden fixture + rpc.dig.net read proof (super-repo #843). +//! +//! This is the SHARED acceptance input for the single-node "established" read +//! bar: #843 (the rpc-tier read proof) AND #1062 (the multi-node P2P e2e). Both +//! reference ONE committed golden — never a forked second copy. +//! +//! ## What the golden is +//! +//! A single-resource capsule described entirely by three inputs — a fixed +//! `store_id`, a `resource_key`, and a plaintext body — from which EVERY other +//! value is DERIVED deterministically through the one-and-only read-crypto in +//! [`digstore_core`] (the same code the producer, the host, and the browser +//! verifier all run): +//! +//! ```text +//! rootless_urn = urn:dig:chia:/ +//! decryption_key = HKDF(rootless_urn) (digstore_core) +//! ciphertext = AES-256-GCM-SIV(decryption_key, plaintext) (deterministic) +//! leaf = resource_leaf(ciphertext) = SHA-256(ciphertext) +//! root = MerkleTree::from_leaves([leaf]).root (single leaf) +//! retrieval_key = SHA-256(rootless_urn) +//! ``` +//! +//! Because the KDF is a pure function and AES-256-GCM-SIV is deterministic, the +//! ciphertext / proof / root are byte-stable forever. The committed fixture +//! files under `tests/fixtures/golden/` are the §5.1 anti-regression ANCHOR: if +//! any layer of the read crypto ever drifts, [`golden_bytes_are_byte_stable`] +//! fails, and a published `.dig` under this identity would become unreadable. +//! +//! ## Regenerating (only ever to ADD a field, never to change crypto) +//! +//! `cargo test -p digstore-remote --test golden_read_proof regen -- --ignored` +//! rewrites the committed files from the inputs. Regeneration must NEVER change +//! an existing byte — that would be a §5.1 format break. + +use std::path::{Path, PathBuf}; + +use base64::Engine; +use digstore_core::{ + decrypt_chunk, derive_decryption_key, encrypt_chunk, resource_leaf, Bytes32, Decode, Decoder, + Encode, MerkleProof, MerkleTree, Urn, +}; + +// --------------------------------------------------------------------------- +// The golden identity (documented constants — the canonical acceptance input) +// --------------------------------------------------------------------------- + +/// The golden `store_id`. A fixed, documented 32-byte constant standing in for a +/// CHIP-0035 DataStore launcher id. It is NOT a live mainnet store yet (see the +/// rpc-tier test's `DIG_RPC_LIVE` gate); publishing a real store under a captured +/// launcher id and re-pointing this constant is the follow-up that flips the +/// rpc-tier read proof from gated to live. Value: `SHA-256("dig-golden-read-proof-843-store")` +/// — a documented, reproducible 32-byte constant (not a hash-of-key; a launcher-id stand-in). +const GOLDEN_STORE_ID_HEX: &str = + "d9c8ae4b6006b5d2d82ecf53f84d74e4bfe7ec4e9fde2a0d71f058b14216ff9a"; + +/// The golden resource key (verbatim, not lowercased — §5.1 URN rule). +const GOLDEN_RESOURCE_KEY: &str = "index.html"; + +/// The golden plaintext body. Small (< 1 KiB), fixed, human-legible so a +/// byte-comparison failure is obvious. +const GOLDEN_PLAINTEXT: &[u8] = + b"\nDIG golden read fixture\n

hello from the DIG Network

\n\ +

This is the #843 golden capsule: a single index.html resource whose ciphertext, \ + inclusion proof, and root are derived deterministically from digstore_core.

\n"; + +fn fixtures_dir() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests") + .join("fixtures") + .join("golden") +} + +// --------------------------------------------------------------------------- +// Deterministic derivation (the single source of truth for every golden value) +// --------------------------------------------------------------------------- + +/// The rootless canonical URN — the one the retrieval/decryption keys derive from +/// (the root is dropped so the keys survive generations, per the format skill). +fn rootless_urn() -> Urn { + Urn { + chain: "chia".to_string(), + store_id: Bytes32::from_hex(GOLDEN_STORE_ID_HEX).expect("valid store_id hex"), + root_hash: None, + resource_key: Some(GOLDEN_RESOURCE_KEY.to_string()), + } +} + +/// Everything a reader needs to verify + decrypt the golden, derived purely from +/// the documented inputs. This is the exact wire shape `dig.getContent` returns +/// (`ciphertext` + `inclusion_proof` + `root`) plus the `retrieval_key` the +/// caller looks the resource up by. +struct Golden { + store_id: Bytes32, + retrieval_key: Bytes32, + ciphertext: Vec, + proof: MerkleProof, + root: Bytes32, +} + +fn derive_golden() -> Golden { + let urn = rootless_urn(); + let canonical = urn.canonical(); + + let key = derive_decryption_key(&canonical, None); + let ciphertext = encrypt_chunk(&key, GOLDEN_PLAINTEXT); + + // Single-resource capsule: the merkle leaf layer is one D5 resource leaf. + let leaf = resource_leaf(&ciphertext); + let tree = MerkleTree::from_leaves(vec![leaf]); + let proof = tree.prove(0).expect("single-leaf proof"); + let root = proof.root; + + Golden { + store_id: urn.store_id, + retrieval_key: urn.retrieval_key(), + ciphertext, + proof, + root, + } +} + +fn proof_to_b64(proof: &MerkleProof) -> String { + base64::engine::general_purpose::STANDARD.encode(proof.to_bytes()) +} + +fn proof_from_b64(b64: &str) -> MerkleProof { + let bytes = base64::engine::general_purpose::STANDARD + .decode(b64.trim()) + .expect("valid base64 proof"); + let mut dec = Decoder::new(&bytes); + MerkleProof::decode(&mut dec).expect("decodable proof") +} + +// --------------------------------------------------------------------------- +// Regeneration (ignored; run explicitly to ADD a field — never to change crypto) +// --------------------------------------------------------------------------- + +#[test] +#[ignore = "regeneration helper — run with --ignored to (re)write the committed golden files"] +fn regen_golden_fixtures() { + let g = derive_golden(); + let dir = fixtures_dir(); + std::fs::create_dir_all(&dir).unwrap(); + + std::fs::write(dir.join("plaintext.bin"), GOLDEN_PLAINTEXT).unwrap(); + std::fs::write(dir.join("ciphertext.bin"), &g.ciphertext).unwrap(); + std::fs::write(dir.join("inclusion_proof.b64"), proof_to_b64(&g.proof)).unwrap(); + + let pinned_urn = { + let mut u = rootless_urn(); + u.root_hash = Some(g.root); + u.canonical() + }; + let manifest = serde_json::json!({ + "description": "DIG Network #843 golden read fixture — one index.html capsule, \ + derived deterministically from digstore_core. Shared by #843 (rpc read \ + proof) and #1062 (P2P e2e). Do not fork a second copy.", + "store_id": g.store_id.to_hex(), + "resource_key": GOLDEN_RESOURCE_KEY, + "root": g.root.to_hex(), + "retrieval_key": g.retrieval_key.to_hex(), + "canonical_urn_rootless": rootless_urn().canonical(), + "canonical_urn_pinned": pinned_urn, + "plaintext_len": GOLDEN_PLAINTEXT.len(), + "ciphertext_len": g.ciphertext.len(), + "live_at_rpc_dig_net": false, + "publish_followup": "Publish a real store under a captured launcher id, re-point \ + GOLDEN_STORE_ID_HEX, then un-gate rpc_tier_read_proof (set DIG_RPC_LIVE).", + }); + std::fs::write( + dir.join("manifest.json"), + serde_json::to_string_pretty(&manifest).unwrap() + "\n", + ) + .unwrap(); +} + +// --------------------------------------------------------------------------- +// §5.1 anchor: the committed golden decodes byte-identically, forever +// --------------------------------------------------------------------------- + +/// The committed ciphertext + proof MUST equal what today's crypto derives from +/// the documented inputs. A drift in the KDF, the AEAD, the leaf binding, or the +/// merkle fold breaks this — which is exactly the §5.1 regression we must catch, +/// because a published `.dig` under this identity would stop decoding. +#[test] +fn golden_bytes_are_byte_stable() { + let g = derive_golden(); + let dir = fixtures_dir(); + + let committed_ct = std::fs::read(dir.join("ciphertext.bin")).expect("committed ciphertext"); + assert_eq!( + committed_ct, g.ciphertext, + "golden ciphertext drifted — a read-crypto change would brick published .dig content (§5.1)" + ); + + let committed_proof = + std::fs::read_to_string(dir.join("inclusion_proof.b64")).expect("committed proof"); + assert_eq!( + proof_from_b64(&committed_proof), + g.proof, + "golden inclusion proof drifted (§5.1)" + ); + + let committed_pt = std::fs::read(dir.join("plaintext.bin")).expect("committed plaintext"); + assert_eq!(committed_pt, GOLDEN_PLAINTEXT, "golden plaintext drifted"); +} + +/// The manifest's documented `store_id` + `root` MUST match the derivation, so +/// #1431 `verify_pinned_root` and any per-range merkle path can assert against +/// the same identity the manifest advertises to #1062. +#[test] +fn golden_manifest_identity_matches_derivation() { + let g = derive_golden(); + let manifest: serde_json::Value = + serde_json::from_slice(&std::fs::read(fixtures_dir().join("manifest.json")).unwrap()) + .unwrap(); + assert_eq!(manifest["store_id"], g.store_id.to_hex()); + assert_eq!(manifest["root"], g.root.to_hex()); + assert_eq!(manifest["retrieval_key"], g.retrieval_key.to_hex()); +} + +// --------------------------------------------------------------------------- +// The verify-then-decrypt contract every read path (rpc / local / P2P) applies +// --------------------------------------------------------------------------- + +/// The full read gate over the COMMITTED golden bytes, exactly as a client does +/// after fetching from any tier: (1) the served ciphertext's resource leaf equals +/// the proof leaf, (2) the proof recomputes the trusted root, (3) decryption +/// yields the golden plaintext byte-for-byte. Fail-closed at every step. +#[test] +fn committed_golden_verifies_then_decrypts() { + let dir = fixtures_dir(); + let ciphertext = std::fs::read(dir.join("ciphertext.bin")).unwrap(); + let proof = proof_from_b64(&std::fs::read_to_string(dir.join("inclusion_proof.b64")).unwrap()); + let plaintext = std::fs::read(dir.join("plaintext.bin")).unwrap(); + + // (1) content binds to the leaf the proof commits to. + assert_eq!( + resource_leaf(&ciphertext), + proof.leaf, + "served ciphertext must hash to the proof leaf" + ); + // (2) the proof recomputes the (trusted) root. + assert!(proof.verify(), "inclusion proof must verify to its root"); + + // (3) decrypt with the URN-derived key → the golden plaintext. + let key = derive_decryption_key(&rootless_urn().canonical(), None); + let opened = decrypt_chunk(&key, &ciphertext).expect("GCM-SIV tag must verify"); + assert_eq!( + opened, plaintext, + "decrypted golden must equal the plaintext" + ); +} + +/// A tampered ciphertext MUST fail the read gate — proving the golden's proof is +/// a real integrity check, not a rubber stamp. +#[test] +fn tampered_golden_fails_the_gate() { + let mut ciphertext = std::fs::read(fixtures_dir().join("ciphertext.bin")).unwrap(); + ciphertext[0] ^= 0xFF; + let proof = proof_from_b64( + &std::fs::read_to_string(fixtures_dir().join("inclusion_proof.b64")).unwrap(), + ); + assert_ne!( + resource_leaf(&ciphertext), + proof.leaf, + "tampered ciphertext must not match the committed proof leaf" + ); +} + +// --------------------------------------------------------------------------- +// The rpc-tier read proof (#843) — live against rpc.dig.net when a store exists +// --------------------------------------------------------------------------- + +/// Read the golden URN from `rpc.dig.net` via `dig.getContent`, verify the +/// returned inclusion proof against the golden root, decrypt, and byte-compare to +/// the golden plaintext — the definition of "a dig-node can read content" over +/// the public gateway (the §5.3 final fallback tier). +/// +/// GATED behind `DIG_RPC_LIVE` because no store is published under +/// [`GOLDEN_STORE_ID_HEX`] yet. RESUME CONDITION (documented on #843): publish a +/// real store under a captured launcher id, re-point `GOLDEN_STORE_ID_HEX`, then +/// run `DIG_RPC_LIVE=1 cargo test -p digstore-remote --test golden_read_proof +/// rpc_tier_read_proof`. Kept present + compiling so the proof is ready to flip +/// live the moment the store exists — never a red/absent test. +#[tokio::test] +async fn rpc_tier_read_proof() { + if std::env::var("DIG_RPC_LIVE").is_err() { + eprintln!( + "skipping rpc_tier_read_proof: set DIG_RPC_LIVE=1 once the golden store is published \ + to rpc.dig.net (see manifest.json publish_followup / #843)" + ); + return; + } + + let base = std::env::var("DIG_RPC_URL").unwrap_or_else(|_| "https://rpc.dig.net".to_string()); + let client = digstore_remote::DigClient::new(base); + + let g = derive_golden(); + let resp = client + .get_content(&g.store_id, &g.retrieval_key, Some(&g.root)) + .await + .expect("dig.getContent must succeed for the published golden store"); + + // Verify-then-decrypt against the CHAIN-anchored golden root (not the served + // roothash) — the read must fail closed if the gateway served a wrong tree. + assert_eq!( + resp.roothash, g.root, + "served root must equal the golden root" + ); + let mut proof = resp.merkle_proof; + assert_eq!( + resource_leaf(&resp.ciphertext), + proof.leaf, + "served ciphertext must hash to the proof leaf" + ); + proof.root = g.root; // pin verification to the trusted root + assert!( + proof.verify(), + "served proof must verify to the golden root" + ); + + let key = derive_decryption_key(&rootless_urn().canonical(), None); + let opened = decrypt_chunk(&key, &resp.ciphertext).expect("GCM-SIV tag must verify"); + assert_eq!( + opened, GOLDEN_PLAINTEXT, + "rpc.dig.net read must byte-match the golden plaintext" + ); +} + +// --------------------------------------------------------------------------- +// STUB — local-tier / rootless-resolve read leg (connect-lane) +// --------------------------------------------------------------------------- + +/// PENDING SEAM (do not delete): the local-tier read proof — `dign open` against +/// the local node's `/s/:/` GET plus the rootless `X-Dig-Root` +/// resolve path — belongs here alongside the rpc-tier proof, but it depends on +/// the connect-lane dig-node work consuming #1431 `verify_pinned_root` (dig-node +/// #1439 / #747 / #852). It is intentionally left as a documented stub so the +/// seam is visible and #1062's e2e harness knows where the local-tier assertion +/// lands. Track: super-repo #843 (local-tier leg) + #1062. +#[test] +#[ignore = "pending connect-lane dig-node #1439/#747/#852 (verify_pinned_root consumption)"] +fn local_tier_read_proof_stub() { + // Intentionally empty: the local /s GET + rootless X-Dig-Root resolve proof + // is implemented once the local node exposes the pinned-root read path. +}