From 1a1803559baf03e6f0a6dcbed4c9fb20ad2efbc0 Mon Sep 17 00:00:00 2001 From: Markik <54276851+mark-ik@users.noreply.github.com> Date: Tue, 18 Aug 2026 01:52:00 -0400 Subject: [PATCH 1/5] Carry interactive Gemini fetch state --- crates/system/fetch/Cargo.toml | 1 + crates/system/fetch/src/lib.rs | 246 +++++++++++++++++++++++++++---- crates/system/fetch/src/tests.rs | 55 +++++++ 3 files changed, 275 insertions(+), 27 deletions(-) diff --git a/crates/system/fetch/Cargo.toml b/crates/system/fetch/Cargo.toml index b516a8f72..f6af7b958 100644 --- a/crates/system/fetch/Cargo.toml +++ b/crates/system/fetch/Cargo.toml @@ -25,6 +25,7 @@ tokio = { version = "1", features = ["rt-multi-thread", "time", "io-util", "sync tracing.workspace = true url = "2" verso-tile = { git = "https://github.com/merely-made/genet.git", branch = "main" } +zeroize = "1" [lints] workspace = true diff --git a/crates/system/fetch/src/lib.rs b/crates/system/fetch/src/lib.rs index 0af9e74e1..65d0410ff 100644 --- a/crates/system/fetch/src/lib.rs +++ b/crates/system/fetch/src/lib.rs @@ -31,9 +31,10 @@ use std::time::Duration; use armillary::{ActorHandle, Emitter, Wake, spawn}; use eidetic::Store; use netfetcher::{CookieRecord, CookieStore, InMemoryCookieJar, SameSite, SameSiteContext}; -use serde::{Deserialize, Serialize}; use pandect::PersonaId; +use serde::{Deserialize, Serialize}; use tokio::runtime::Builder; +use zeroize::Zeroizing; /// The most redirects a smolweb fetch will follow before giving up. const MAX_REDIRECTS: usize = 5; @@ -53,17 +54,51 @@ const SUBRESOURCE_BODY_CAP: usize = 32 * 1024 * 1024; /// Successfully fetched content: the response content-type (if any) and the /// decoded body as text. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct Fetched { pub content_type: Option, pub body: String, } +/// A page request that needs host participation rather than being reducible +/// to a terminal error string. The fetch actor preserves these arms so a UI +/// host can continue the protocol conversation without parsing prose. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum FetchFailure { + /// A Gemini-style input response. `url` is the final request address after + /// redirects and is therefore the address the submitted query belongs to. + InputRequired { + url: String, + prompt: String, + sensitive: bool, + }, + /// The server requires a client certificate. Identity selection remains + /// a host decision; carrying the target keeps that later conversation + /// typed instead of collapsing it into an ordinary transport failure. + ClientCertificateRequired { + url: String, + prompt: String, + code: Option, + }, + /// A terminal transport, protocol, HTTP, or size-limit failure. + Failed(String), +} + +impl std::fmt::Display for FetchFailure { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::InputRequired { prompt, .. } => write!(f, "input required: {prompt}"), + Self::ClientCertificateRequired { .. } => f.write_str("client certificate required"), + Self::Failed(error) => f.write_str(error), + } + } +} + /// The result of one fetch, tagged with the requested URL so the host routes it /// back to the right node's content slot. pub struct FetchOutcome { pub url: String, - pub result: Result, + pub result: Result, } /// A fetched subresource: raw bytes for an absolute URL (page CSS via @@ -75,6 +110,101 @@ pub struct SubresourceOutcome { pub bytes: Vec, } +/// One Gemini client certificate, assigned to exactly one capsule origin. +/// +/// The host mints the material from its identity layer. The actor enforces the +/// host+effective-port scope on every redirect, so a certificate selected for +/// one capsule is never presented to another. Private bytes are shared rather +/// than copied between effects and commands, and zeroized when the last owner +/// drops. +#[derive(Clone)] +pub struct GeminiClientIdentity { + host: String, + port: u16, + certificate_der: Arc<[u8]>, + private_key_pkcs8_der: Arc>>, +} + +impl std::fmt::Debug for GeminiClientIdentity { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("GeminiClientIdentity") + .field("origin", &self.origin()) + .field("certificate", &"[redacted]") + .field("private_key", &"[redacted]") + .finish() + } +} + +impl PartialEq for GeminiClientIdentity { + fn eq(&self, other: &Self) -> bool { + self.host == other.host + && self.port == other.port + && self.certificate_der.as_ref() == other.certificate_der.as_ref() + && self.private_key_pkcs8_der.as_slice() == other.private_key_pkcs8_der.as_slice() + } +} + +impl Eq for GeminiClientIdentity {} + +impl GeminiClientIdentity { + pub fn new( + capsule_url: &str, + certificate_der: Vec, + private_key_pkcs8_der: Vec, + ) -> Result { + let url = url::Url::parse(capsule_url).map_err(|error| error.to_string())?; + if url.scheme() != "gemini" { + return Err("Gemini client identity requires a gemini:// capsule".to_string()); + } + let host = url + .host_str() + .ok_or_else(|| "Gemini capsule has no host".to_string())? + .to_ascii_lowercase(); + if certificate_der.is_empty() || private_key_pkcs8_der.is_empty() { + return Err("Gemini client identity material is empty".to_string()); + } + Ok(Self { + host, + port: url.port().unwrap_or(1965), + certificate_der: Arc::from(certificate_der), + private_key_pkcs8_der: Arc::new(Zeroizing::new(private_key_pkcs8_der)), + }) + } + + pub fn origin(&self) -> String { + let host = if self.host.contains(':') && !self.host.starts_with('[') { + format!("[{}]", self.host) + } else { + self.host.clone() + }; + if self.port == 1965 { + format!("gemini://{host}") + } else { + format!("gemini://{host}:{}", self.port) + } + } + + pub fn certificate_der(&self) -> &[u8] { + self.certificate_der.as_ref() + } + + fn applies_to(&self, url: &url::Url) -> bool { + url.scheme() == "gemini" + && url + .host_str() + .is_some_and(|host| host.eq_ignore_ascii_case(&self.host)) + && url.port().unwrap_or(1965) == self.port + } + + fn errand_view(&self) -> errand::GeminiClientIdentity<'_> { + errand::GeminiClientIdentity { + certificate_der: self.certificate_der.as_ref(), + private_key_pkcs8_der: self.private_key_pkcs8_der.as_slice(), + } + } +} + /// Per-URL content state behind the focused-node card. #[derive(Clone, Debug)] pub enum ContentState { @@ -121,7 +251,10 @@ fn scheme_of(url: &str) -> Option<&str> { /// A command to the fetch actor. pub enum FetchCommand { /// Fetch `url` as a page document (decoded body as text). - Page(String), + Page { + url: String, + identity: Option, + }, /// Fetch the subresource at the (already absolute) `url` as raw bytes. Subresource(String), /// Fetch the favicon at `url` (already absolute) as raw bytes, remembering it @@ -160,10 +293,12 @@ pub fn spawn_fetcher(wake: Wake) -> (ActorHandle, Receiver { + FetchCommand::Page { url, identity } => { let out = out.clone(); runtime.spawn(async move { - let result = fetch_page(&url).await; + let result = + fetch_page_interactive_capped(&url, PAGE_BODY_CAP, identity.as_ref()) + .await; out.emit(FetchUpdate::Page(FetchOutcome { url, result })); }); } @@ -205,31 +340,57 @@ pub async fn fetch_page(url: &str) -> Result { /// body (§A5): the http path enforces it *while streaming* (no OOM); smolweb is /// already buffered by errand, so it is checked post-hoc (errand bounds its own read). pub async fn fetch_page_capped(url: &str, max_bytes: usize) -> Result { + fetch_page_interactive_capped(url, max_bytes, None) + .await + .map_err(|error| error.to_string()) +} + +/// Fetch one page while preserving protocol responses that require host +/// participation. This is actor-facing: ordinary utility callers retain the +/// terminal `Result` contract above. +async fn fetch_page_interactive_capped( + url: &str, + max_bytes: usize, + identity: Option<&GeminiClientIdentity>, +) -> Result { match scheme_of(url).and_then(errand::Scheme::parse) { Some(scheme) => { - tracing::info!(%url, ?scheme, "smolweb fetch"); - let result = smolweb_fetch(url).await.and_then(|fetched| { + let log_url = url_without_query(url); + tracing::info!(url = %log_url, ?scheme, "smolweb fetch"); + let result = smolweb_fetch(url, identity).await.and_then(|fetched| { if fetched.body.len() > max_bytes { - Err(format!("response exceeds the {max_bytes}-byte cap")) + Err(FetchFailure::Failed(format!( + "response exceeds the {max_bytes}-byte cap" + ))) } else { Ok(fetched) } }); match &result { Ok(fetched) => tracing::info!( - %url, + url = %log_url, content_type = ?fetched.content_type, bytes = fetched.body.len(), "smolweb ok", ), - Err(error) => tracing::warn!(%url, %error, "smolweb failed"), + Err(error) => tracing::warn!(url = %log_url, %error, "smolweb failed"), } result } - None => do_fetch(url, max_bytes).await, + None => do_fetch(url, max_bytes).await.map_err(FetchFailure::Failed), } } +fn url_without_query(raw: &str) -> String { + url::Url::parse(raw) + .map(|mut parsed| { + parsed.set_query(None); + parsed.set_fragment(None); + parsed.to_string() + }) + .unwrap_or_else(|_| "".to_string()) +} + /// Fetch a page without the browser session's cookie jar or other installed /// authenticated HTTP state. Effect providers use this path so resolving a /// transclusion cannot silently borrow the user's browsing authority. @@ -268,14 +429,27 @@ pub async fn fetch_page_crawler(url: &str) -> Result { /// Fetch a smolweb URL through [`errand`], following redirects up to /// [`MAX_REDIRECTS`], and fold the response into a [`Fetched`] the nematic engines -/// render. Non-success statuses (input wanted, cert required, failure) surface as -/// an error string the card shows. -async fn smolweb_fetch(url: &str) -> Result { - let mut current = url::Url::parse(url).map_err(|e| format!("bad URL: {e}"))?; +/// render. Input and certificate statuses stay typed so the host can continue +/// the protocol conversation; terminal failures remain displayable prose. +async fn smolweb_fetch( + url: &str, + identity: Option<&GeminiClientIdentity>, +) -> Result { + let mut current = + url::Url::parse(url).map_err(|error| FetchFailure::Failed(format!("bad URL: {error}")))?; for _ in 0..MAX_REDIRECTS { - let response = errand::fetch_url_timeout(¤t, SMOLWEB_TIMEOUT) - .await - .map_err(|e| e.to_string())?; + let response = match identity.filter(|identity| identity.applies_to(¤t)) { + Some(identity) => { + errand::fetch_url_timeout_with_identity( + ¤t, + identity.errand_view(), + SMOLWEB_TIMEOUT, + ) + .await + } + None => errand::fetch_url_timeout(¤t, SMOLWEB_TIMEOUT).await, + } + .map_err(|error| FetchFailure::Failed(error.to_string()))?; match response.status { errand::Status::Success => { let content_type = smolweb_content_type(¤t, &response); @@ -286,22 +460,40 @@ async fn smolweb_fetch(url: &str) -> Result { }); } errand::Status::Redirect => { - current = current - .join(&response.meta) - .map_err(|e| format!("bad redirect target: {e}"))?; + current = current.join(&response.meta).map_err(|error| { + FetchFailure::Failed(format!("bad redirect target: {error}")) + })?; + } + errand::Status::Input => { + return Err(smolweb_input_failure(¤t, &response)); + } + errand::Status::CertRequired => { + return Err(FetchFailure::ClientCertificateRequired { + url: current.to_string(), + prompt: response.meta, + code: response.raw_status, + }); } - errand::Status::Input => return Err(format!("input required: {}", response.meta)), - errand::Status::CertRequired => return Err("client certificate required".to_string()), errand::Status::Failure => { - return Err(if response.meta.is_empty() { + return Err(FetchFailure::Failed(if response.meta.is_empty() { "request failed".to_string() } else { response.meta - }); + })); } } } - Err("too many redirects".to_string()) + Err(FetchFailure::Failed("too many redirects".to_string())) +} + +fn smolweb_input_failure(current: &url::Url, response: &errand::Response) -> FetchFailure { + FetchFailure::InputRequired { + url: current.to_string(), + prompt: response.meta.clone(), + // Gemini status 11 is the sensitive-input form. Other protocols + // currently expose only an ordinary input code. + sensitive: response.raw_status == Some(11), + } } /// The content-type to render a smolweb response under, in nematic's vocabulary. diff --git a/crates/system/fetch/src/tests.rs b/crates/system/fetch/src/tests.rs index 9cc32117e..4016f962e 100644 --- a/crates/system/fetch/src/tests.rs +++ b/crates/system/fetch/src/tests.rs @@ -67,6 +67,61 @@ fn smolweb_content_type_tags_fixed_schemes_and_passes_others_through() { assert_eq!(smolweb_content_type(&gem, &gem_resp), "text/gemini"); } +#[test] +fn smolweb_input_preserves_prompt_target_and_sensitivity() { + let target = url::Url::parse("gemini://capsule.example/search").unwrap(); + let response = |code| errand::Response { + url: target.clone(), + status: errand::Status::Input, + raw_status: Some(code), + meta: "Search the capsule".into(), + body: Vec::new(), + }; + + assert_eq!( + smolweb_input_failure(&target, &response(10)), + FetchFailure::InputRequired { + url: target.to_string(), + prompt: "Search the capsule".into(), + sensitive: false, + } + ); + assert_eq!( + smolweb_input_failure(&target, &response(11)), + FetchFailure::InputRequired { + url: target.to_string(), + prompt: "Search the capsule".into(), + sensitive: true, + } + ); +} + +#[test] +fn smolweb_trace_address_omits_query_and_fragment() { + assert_eq!( + url_without_query("gemini://capsule.example/search?secret%20answer#part"), + "gemini://capsule.example/search" + ); +} + +#[test] +fn gemini_identity_is_scoped_to_one_capsule_origin() { + let identity = GeminiClientIdentity::new( + "gemini://Capsule.Example/account", + vec![1, 2, 3], + vec![4, 5, 6], + ) + .unwrap(); + assert!(identity.applies_to(&url::Url::parse("gemini://capsule.example/private").unwrap())); + assert!(identity.applies_to(&url::Url::parse("gemini://capsule.example:1965/other").unwrap())); + assert!(!identity.applies_to(&url::Url::parse("gemini://other.example/private").unwrap())); + assert!( + !identity.applies_to(&url::Url::parse("gemini://capsule.example:1966/private").unwrap()) + ); + assert!(!identity.applies_to(&url::Url::parse("https://capsule.example/private").unwrap())); + assert_eq!(identity.origin(), "gemini://capsule.example"); +} + #[test] fn state_tag_distinguishes_transitions() { let ready = ContentState::Ready(Fetched { From bb8ff5f0dbba47800df488f293d2d38c6d83362c Mon Sep 17 00:00:00 2001 From: Markik <54276851+mark-ik@users.noreply.github.com> Date: Tue, 18 Aug 2026 02:01:06 -0400 Subject: [PATCH 2/5] C1 blocked: host chrome cannot see a hold, for the same reason B1 could not A1 landed, so there is finally something real to render: unmet_holds crosses the wire, honored_holds names the satisfied pins, and the frozen realization already lists violations in its tabular alternate and announces them in its AccessKit tree. Cambium cannot render any of it. It has no sceno dependency, so it cannot see a Scene, and nothing in mere consumes cambium either. Both directions checked against the tree rather than assumed. This is Track C arriving at the correction Track B already took. A host renders into cambium's surfaces; the code that knows what a hold is lives mere-side. So C1 needs a mere-side host built on cambium, and none exists, which is the same absence that blocks a driven probe scenario. mer3ly could take it now without a new host, since it already reports a honored count on its export control and the snapshot now carries the violations alongside. That is C1's substance in a web page rather than native chrome, and it costs a rev bump, a wasm rebuild, and a public deploy, so it is a decision rather than a quiet extension of this target. What is deliberately not built: a cambium surface with no consumer. That is the work five gated targets are being held back from, and adjacency to something that just landed is not an exemption. --- ...-08-15_projection_grammar_adoption_plan.md | 23 +- ...-02-22_aspirational_protocols_and_tools.md | 291 ++++++ .../2026-02-23_modern_yacy_gap_analysis.md | 122 +++ .../2026-02-23_storage_economy_and_indices.md | 0 ...7_verse_distributed_index_protocol_v0_1.md | 562 +++++++++++ ..._verse_graph_contribution_protocol_v0_1.md | 885 ++++++++++++++++++ 6 files changed, 1882 insertions(+), 1 deletion(-) create mode 100644 design_docs/mere_docs/research/2026-02-22_aspirational_protocols_and_tools.md create mode 100644 design_docs/mere_docs/research/2026-02-23_modern_yacy_gap_analysis.md create mode 100644 design_docs/mere_docs/research/2026-02-23_storage_economy_and_indices.md create mode 100644 design_docs/mere_docs/research/2026-04-17_verse_distributed_index_protocol_v0_1.md create mode 100644 design_docs/mere_docs/research/2026-04-17_verse_graph_contribution_protocol_v0_1.md diff --git a/design_docs/mere_docs/implementation_strategy/2026-08-15_projection_grammar_adoption_plan.md b/design_docs/mere_docs/implementation_strategy/2026-08-15_projection_grammar_adoption_plan.md index d2966f1da..76b148602 100644 --- a/design_docs/mere_docs/implementation_strategy/2026-08-15_projection_grammar_adoption_plan.md +++ b/design_docs/mere_docs/implementation_strategy/2026-08-15_projection_grammar_adoption_plan.md @@ -341,7 +341,7 @@ single-root host, woodshed then signalman its consumers, and swatches are the agreed cross-product graph-view contract. Cambium doc updates land in `genet/components/cambium/docs/` when a slice opens. -**C1. Satisfaction state in host chrome (consumer half of A1).** +**C1. Satisfaction state in host chrome - BLOCKED 2026-08-16, not by effort.** Context: A1's scene-side record is only honest if a user can see it. Tasks: with A1's consumer, surface pin state in the host's widget chrome (a pinned badge; an unmet-pin state visibly distinct); keep the vocabulary @@ -667,3 +667,24 @@ not settled without it. view's own", which is unanswerable without it, and a field added later would be missing from every link already in circulation. The shelfmark note's reserved `selection` section is now defined against that record. +- 2026-08-16: **C1 investigated and blocked, on the same wall B1 hit.** A1 has + landed, so there is finally something real to render: `Scene.unmet_holds` + crosses the wire, `Scene.honored_holds` names the satisfied pins, and the + frozen realization already lists violations in its tabular alternate and + announces them in its AccessKit tree. The target says surface the same + distinction in cambium host chrome. Cambium cannot: it has no `sceno` + dependency, so it cannot see a `Scene` at all, and nothing in mere consumes + cambium either. Verified from the tree in both directions, not assumed. + This is Track C meeting the correction Track B already took. A host renders + *into* cambium's surfaces; the code that knows what a hold is lives mere-side. + So C1's home is a mere-side host built on cambium, and no such host exists + today, which is the same missing piece that blocks the driven probe scenario. + One consumer could take it now without a new host: mer3ly already reports + "1 pin honored" on its export control, and could report the unmet count + beside it rather than only failing the whole artifact, since the snapshot now + carries the violations. That is C1's substance in a web page rather than + native chrome, and it costs another rev bump, wasm rebuild, and public deploy, + so it is Mark's call rather than a quiet extension of this target. + Not built: a speculative cambium surface with no consumer. That is exactly + the work the five gated targets are being held back from, and C1 does not get + an exemption for being adjacent to something that just landed. diff --git a/design_docs/mere_docs/research/2026-02-22_aspirational_protocols_and_tools.md b/design_docs/mere_docs/research/2026-02-22_aspirational_protocols_and_tools.md new file mode 100644 index 000000000..714db5ebe --- /dev/null +++ b/design_docs/mere_docs/research/2026-02-22_aspirational_protocols_and_tools.md @@ -0,0 +1,291 @@ +> **Recovered research. Never implemented. Not current direction.** +> +> Recovered 2026-08-16 from the git history of the archived `graphshell` +> repository (`Code/archive/graphshell`), whose entire `design_docs/` tree is +> deleted at HEAD, so this text survives only in history. +> +> - Original path: `design_docs/verse_docs/research/2026-02-22_aspirational_protocols_and_tools.md` +> - Source commit: `9a6526a7` +> +> None of it was ever built. It is filed here as research, not as a plan and +> not as a statement of where Mere is going. It speaks the retired Verse +> vocabulary (per `TERMINOLOGY.md`, *Verse* is a retired term and the +> network scope is Mere at network scope), and its relative links point at +> donor docs that no longer exist locally. Read the text below for the ideas, +> not the direction. +> +> The donor file begins at its own section 2, with no title and no section 1; +> it was committed that way and no earlier version exists. Recovered as found. + +--- + + +--- + +## 2. Vision: The "Verse" & User Agency + +The "Verse" is the collaborative, networked dimension of Graphshell. It transforms the browser from a *consumer* of the web to a *participant* in a new web. + +### 2.1 Philosophy & Ramifications +1. **Inversion of Control**: Currently, search engines decide what you see. In the Verse, *you* curate your index, and your *trust network* curates your discovery feed. +2. **The "Tokenized Report"**: Knowledge isn't just a link. It's a subgraph: a collection of nodes, annotations, and edges. The Verse allows packaging this understanding into a portable asset (a file, a hash) that can be shared, signed, and verified. +3. **Resilience**: If a website goes down, the Verse (via IPFS/P2P storage) remembers. Graphshell becomes a distributed archive. +4. **Identity as Key**: Using cryptographic keys (Nostr/DID) for identity means you own your social graph. If a platform bans you, your graph (and your connections) moves with you. + +### 2.2 The "Web of Trust" vs "Web of Algorithms" +The current web optimizes for engagement (ads). The Verse optimizes for **relevance and trust**. +* **Scenario**: You search for "best hiking boots." +* **Google**: Shows SEO-spam and ads. +* **Verse**: Shows nodes your hiking club pinned, reviews from people you follow, and cached pages from your own history. + +### 2.3 The Storage Economy +The real limitation of a decentralized web is storage. +* **Problem**: Who stores the content when the original host goes offline? +* **Solution**: A storage economy. Users earn "credits" or reputation by pinning content (IPFS) for their Verse. +* **Opportunity**: People are rewarded for solving the preservation problem. + +--- + +## 3. Protocols & Transport Ecosystem + +Integrating these protocols moves Graphshell towards the "Verse" vision of a resilient, peer-to-peer knowledge network. + +### 3.1 Decentralized Storage & Sync +* **IPFS (InterPlanetary File System)**: Content-addressing. You ask for content by *hash* (CID), not by location (IP address). + * *Integration*: Node Identity (CID), Storage (Universal Node Content snapshots), Sharing (publishing workspaces). +* **Gun (GunDB)**: A decentralized, offline-first, graph database protocol. + * *Integration*: Meta-alignment for syncing graph data between peers without a server. +* **Protocol Coexistence (libp2p vs iroh)**: + * *libp2p*: Modular, standard for IPFS/Ethereum. Good for raw flexibility. + * *iroh*: Newer, QUIC-native, focused on "syncing bytes". Simpler API surface for Graphshell's specific use case. + +### 3.2 Privacy & Anonymity +* **Tor & I2P**: Onion/Garlic routing to obscure user identity. + * *Integration*: **Resolver Layer**. Graphshell can enforce strict isolation per-node. A `.onion` node uses a dedicated circuit; a clearnet node uses standard HTTPS. +* **Encrypted DNS (DoH)**: DNS over HTTPS prevents ISP spying and censorship. + * *Integration*: Configure Servo's network layer to use a DoH resolver (like 1.1.1.1) by default. + +### 3.3 Social, Identity, & Federation +* **ActivityPub (The "Social Graph")**: The standard for decentralized social networking (Mastodon). + * *Integration*: Treat a Workspace as an Actor. Adding a node emits a `Create` activity. Pair with IPFS to "repost" timestamped snapshots. +* **Matrix (The "Collaboration Layer")**: Decentralized real-time communication. + * *Integration*: Embed chat rooms inside shared workspace nodes; use as a signaling layer for P2P connections. +* **Nostr (Identity & Publishing)**: Cryptographic keys and relays. + * *Integration*: **Portable Identity**. Use Nostr keys (`npub`/`nsec`) as the user's identity for P2P sync. Publish graph snapshots as events. +* **AT Protocol (Bluesky)**: Authenticated Transfer. + * *Integration*: **Algorithmic Choice**. Users can choose "feed generators" to sort their graph or discovery feed. + +### 3.4 Alternative Webs +* **Gemini & Gopher**: Lightweight protocols focusing on text and structure. + * *Integration*: **Native Rendering**. Graphshell can render Gemini content natively (bypassing complex HTML layout) for a distraction-free reading mode. + +--- + +## 4. Semantic Engine & Tooling + +Moving from "Scraping" (extracting raw text) to "Parsing" (extracting meaning). + +### 4.1 Extraction & Parsing +* **Schema.org & JSON-LD**: Standardized vocabulary for structured data. + * *Integration*: The **"Smart Clipper"** parses JSON-LD to populate node metadata (Ingredients, License, Author) automatically. +* **Readability**: Strips clutter (ads, nav) to extract core text. + * *Integration*: Essential for indexing and "Reader Mode" views of graph nodes. +* **PDF (lopdf)**: Parsing PDF documents. + * *Integration*: Treat PDFs as first-class graph nodes with indexable content. +* **DataFrames (Polars)**: Fast data manipulation. + * *Integration*: If a user crawls a dataset (e.g., a wiki table), Polars allows querying and visualizing that data natively. + +### 4.2 Classification +* **Universal Decimal Classification (UDC)**: A faceted library classification system. + * *Integration*: **Ontology Support**. Move beyond flat tags. Use UDC codes to drive graph clustering (physics attraction based on semantic distance) and auto-group tabs. +* **Web Annotation (W3C)**: Standard data model for annotations. + * *Integration*: Store user highlights/notes in a portable format, allowing export to other tools without lock-in. + +### 4.3 Crawling & Search +* **The "Personal Crawler"**: + * *Workflow*: User selects a documentation root -> Command "Crawl links to depth 2" -> Graphshell builds a local, offline-searchable map of that domain. + * *Tech*: `reqwest-middleware` (retries/caching) + `scraper` (HTML parsing). +* **Decentralized Search (YaCy Analysis)**: + * *Critique*: YaCy is Java-based and resource-heavy. + * *Graphshell's Take*: Be the modern, Rust-native evolution. **Local First**: Index what *you* browse (via `tantivy`). **Trusted Federation**: Search only your "Verse" (friends/groups), avoiding global noise. + +### 4.4 The "Asset" Pipeline +* **Philosophy**: "Turn understanding into an asset." +* **Workflow**: Ingest (Browse/Crawl) -> Enrich (Schema/Readability) -> Curate (UDC/Tags) -> Export (Tokenized Report). + +--- + +## 5. Diagnostics & Observability (The "Inspector") + +* **Tokio Tracing**: Instrumenting Rust programs. +* **Graphshell Integration**: + * **Visualizing Servo**: Hook into `tracing` spans to visualize the engine *as a graph*. + * **The "Engine Node"**: A special node showing live topology of Servo's threads (Script, Layout, Webrender) and message channels. + * **Performance**: Visualize backpressure as edge thickness. + +--- + +## 6. Architecture: Protocol Registry & Modularity + +To support this diverse ecosystem without bloating the core, Graphshell uses a modular **Protocol Registry**. + +### 6.1 The Protocol Handler Trait +Instead of hardcoding `http`/`https` logic, we define a trait: + +```rust +pub trait ProtocolHandler { + /// The URL scheme this handler supports (e.g., "ipfs", "gemini"). + fn scheme(&self) -> &str; + + /// Resolve the content (fetch, stream, or proxy). + fn resolve(&self, uri: &str) -> ProtocolResult; + + /// Return capabilities (e.g., supports_search, supports_caching). + fn capabilities(&self) -> ProtocolCapabilities; +} +``` + +### 6.2 The Registry & Opt-In Model +* **Default Set**: `http`, `https`, `file`, `about`. +* **Opt-In Set**: `ipfs`, `gemini`, `gopher`, `magnet`. +* **Discovery**: When a user encounters a new scheme, prompt to enable the handler. +* **Bridge Layer**: Custom protocols can be bridged via internal loopback (e.g., `http://localhost/bridge/gemini/...`) for Servo rendering, or rendered natively. + +--- + +## 7. Extensions & Future Capabilities + +### 7.1 RSS/Atom (Syndication) +* **Concept**: The original decentralized subscription protocol. +* **Graphshell Integration**: + * **Feed Nodes**: A node representing an RSS feed. Edges connect to article nodes. + * **Auto-Update**: The "Personal Crawler" can poll feeds and spawn new nodes for new entries. +* **Crate**: `feed-rs`. + +### 7.2 WebAssembly (Wasm) for Applets & Mods +* **Concept**: Safe, portable binary format for executing code. +* **Graphshell Integration**: + * **Applet Nodes**: Nodes that run a small Wasm binary (calculator, visualizer, game) instead of a web page. + * **Mods**: User-defined physics forces or renderers compiled to Wasm. +* **Crate**: `wasmer` or `wasmtime`. + +### 7.3 Vector Search (Semantic Embeddings) +* **Concept**: Searching by *meaning* rather than keyword matching. +* **Graphshell Integration**: + * **Semantic Association**: "Find nodes related to 'climate change'" (even if they don't use that phrase). + * **Auto-Linking**: Suggest edges between nodes with high semantic similarity. +* **Crate**: `lance` (vector DB) or `candle` (local inference). + +### 7.4 Local LLM Inference (The "Synthesizer") +* **Concept**: Running efficient language models locally to summarize and extract insights without cloud dependencies. +* **Graphshell Integration**: + * **Summarization**: Auto-generate summaries for "Active" nodes to populate tooltips. + * **Extraction**: Turn unstructured page text into structured node metadata (e.g., extracting event dates). + * **Chat with Graph**: RAG (Retrieval-Augmented Generation) combining Vector Search with an LLM to answer questions based on the user's browsing history. +* **Crate**: `candle` (Hugging Face's Rust ML framework) or `burn`. + +### 7.5 CRDTs (Real-Time Collaboration) +* **Concept**: Conflict-free Replicated Data Types allow concurrent edits from multiple peers to merge automatically. +* **Graphshell Integration**: + * **Shared Notes**: Enabling real-time co-editing of text notes attached to nodes. + * **Live Lists**: Managing shared "To Read" queues in a P2P workspace where order matters. +* **Crate**: `automerge` or `yrs` (Rust port of Yjs). + +### 7.6 Web Archiving (WARC) +* **Concept**: The ISO standard file format for web archives, preserving headers and content fidelity. +* **Graphshell Integration**: + * **Forensic Clipping**: Saving the exact network response (headers + body) for a node, not just the DOM. + * **Portability**: Exporting clips that can be viewed in standard tools like ReplayWeb.page or uploaded to the Internet Archive. +* **Crate**: `warc`. + +--- + +## 8. Master Crate Index + +Summary of external crates mapped to Graphshell capabilities. + +| Domain | Crate | Purpose | +| :--- | :--- | :--- | +| **Search** | `tantivy` | Local, high-performance full-text indexing of graph content. | +| **P2P/Sync** | `iroh` | Efficient syncing of graph state and blobs; IPFS alternative. | +| **Networking** | `libp2p` | Modular network stack (DHT, GossipSub) if raw IPFS compat is needed. | +| **Tor** | `arti-client` | Embedding Tor connectivity directly into the resolver layer. | +| **DNS** | `hickory-dns` | DoH support for privacy and censorship resistance. | +| **Parsing** | `scraper` | Lightweight HTML parsing (CSS selectors) for the crawler. | +| **Content** | `readability` | Extracting main article content (stripping ads/nav). | +| **Semantic** | `json-ld` | Extracting structured data from web pages. | +| **Data** | `polars` | Querying and visualizing tabular data found during browsing. | +| **Diagnostics**| `tracing` | Instrumenting the engine to visualize internal topology. | +| **Social** | `activitystreams` | ActivityPub federation for graph sharing. | +| **Chat** | `matrix-sdk` | Real-time collaboration and signaling. | +| **Identity** | `nostr-sdk` | Portable identity and censorship-resistant publishing. | +| **Protocol** | `gemini` | Native rendering of lightweight Gemini content. | +| **Federation** | `atrium-api` | AT Protocol for portable identity and algorithmic choice. | +| **HTTP** | `reqwest-middleware`| Robust HTTP client with retries/caching for the crawler. | +| **Documents** | `lopdf` | Parsing and indexing PDF content. | +| **Syndication**| `feed-rs` | Parsing RSS/Atom feeds for subscription nodes. | +| **Runtime** | `wasmer` | Running Wasm applets and mods safely. | +| **AI/Vector** | `lance` | Vector database for semantic search and auto-linking. | +| **AI/ML** | `candle` | Local LLM inference for summarization and RAG. | +| **Collab** | `automerge` | CRDTs for conflict-free real-time data merging. | +| **Archiving** | `warc` | Standardized web archive format for high-fidelity clips. | + +--- + +## 9. The Registry Ecosystem + +To manage the complexity of a "Knowledge User Agent," Graphshell employs a system of modular registries. These allow features to be composed, swapped, and extended by users or mods. + +### 9.1 Core Registries + +1. **Protocol Registry** (Transport): + * **Role**: Maps URL schemes (`ipfs://`, `gemini://`) to handlers. + * **Pattern**: Opt-in. Users enable protocols as needed. + +2. **Viewer Registry** (Rendering): + * **Role**: Maps MIME types or file extensions to renderers (PDF, Markdown, CSV, 3D Models). + * **Concept**: Decouples content from the browser engine. Not everything needs a webview. + +3. **Command Registry** (Action & Automation): + * **Scope**: Unifies user actions, keybindings, and autonomous agents. + * **Categories**: + * *User Commands*: Manual actions (palette, context menu). + * *Keybinds*: Input mapping. + * *Autonomous Agents*: Background scripts (crawlers, auto-taggers) that emit intents. + * **Pattern**: Piecewise combination. Users can compose commands from defaults or import mod dependencies. + +4. **Lens Registry** (Presentation): + * **Role**: Composable view configurations. + * **Definition**: A "Lens" is a composition of **Theme** + **Layout** + **Physics**. + * **Usage**: Users can layer lenses (e.g., "Dark Mode" + "Tree Layout" + "Low Gravity") or switch contexts entirely. + +5. **Identity Registry** (Auth & Persona): + * **Role**: Manages cryptographic keys and profiles (Work, Personal, Public/Nostr). + * **Function**: Signs reports and sync payloads without leaking keys to every module. + +6. **Ontology Registry** (Meaning): + * **Role**: Manages structured data definitions (Schema.org, UDC). + * **Function**: Provides UI for editing metadata and defines how nodes relate semantically. + +7. **Index Registry** (Recall & Federation): + * **Role**: Manages search backends (Local Tantivy, Peer Indexes). + * **Verse Integration**: The primary mechanism for sharing knowledge. You publish your index; you subscribe to others'. + +### 9.2 Universal Registry Patterns +* **Modularity**: All registries support "Mods" — external definitions that can be loaded/unloaded. +* **Composition**: Defaults can be mixed with user overrides and mod extensions. + + diff --git a/design_docs/mere_docs/research/2026-02-23_modern_yacy_gap_analysis.md b/design_docs/mere_docs/research/2026-02-23_modern_yacy_gap_analysis.md new file mode 100644 index 000000000..203f2b581 --- /dev/null +++ b/design_docs/mere_docs/research/2026-02-23_modern_yacy_gap_analysis.md @@ -0,0 +1,122 @@ +> **Recovered research. Never implemented. Not current direction.** +> +> Recovered 2026-08-16 from the git history of the archived `graphshell` +> repository (`Code/archive/graphshell`), whose entire `design_docs/` tree is +> deleted at HEAD, so this text survives only in history. +> +> - Original path: `design_docs/verse_docs/research/2026-02-23_modern_yacy_gap_analysis.md` +> - Source commit: `1208e352` +> +> None of it was ever built. It is filed here as research, not as a plan and +> not as a statement of where Mere is going. It speaks the retired Verse +> vocabulary (per `TERMINOLOGY.md`, *Verse* is a retired term and the +> network scope is Mere at network scope), and its relative links point at +> donor docs that no longer exist locally. Read the text below for the ideas, +> not the direction. + +--- + +# The Modern YaCy: Gap Analysis & Search Strategy + +**Date**: 2026-02-23 +**Status**: Research / Strategy +**Context**: Analysis of how to evolve the Verse storage economy into a functional decentralized search engine ("The Modern YaCy"). + +--- + +## 1. The Core Problem: Storage vs. Search + +The current Verse architecture (`verse_implementation_strategy.md`) solves **Storage** (hosting encrypted blobs) and **Retrieval** (getting blobs by hash). It does not solve **Discovery** (finding which blob contains the text "Rust Async Tutorial"). + +**YaCy's Approach**: A global Distributed Hash Table (DHT) where every word is a key, and the value is a list of URLs. +* *Pros*: Fully decentralized. +* *Cons*: Extremely chatty, high latency, massive index bloat, poor relevance ranking (spam). + +**Verse's Proposed Approach**: **Federated Index Exchange**. +Instead of scattering words across a DHT, peers build and share **Index Artifacts** (complete, searchable indices for specific domains/topics). + +--- + +## 2. Gap 1: The Index Artifact + +We need a standard format for a portable search index. + +* **Requirement**: A file format that is: + 1. **Compact**: Compressed inverted index. + 2. **Mergeable**: Can be combined with other indices. + 3. **Queryable**: Can be searched efficiently (ideally mapped into memory). +* **Solution**: **Tantivy Segments**. + * Graphshell already plans to use `tantivy` for local search. + * A "Published Index" is just a serialized Tantivy segment containing the indexed content of a Graph or Workspace. + * This artifact is stored as a **Verse Blob** (immutable, content-addressed). + +## 3. Gap 2: The Query Protocol (Remote vs. Local) + +How does a user search? + +### Scenario A: Local Search (The "Download" Model) +* **User Action**: Subscribes to "Rust Community Index". +* **Mechanism**: Graphshell downloads the Index Blob (Tier 2 transaction). +* **Execution**: The index is mounted locally. Queries run at native speed. +* **Pros**: Privacy (queries never leave device), speed. +* **Cons**: Storage/Bandwidth heavy. Good for curated, high-value indices. + +### Scenario B: Remote Search (The "Service" Model) +* **User Action**: Queries "latest crypto news" (too big to download). +* **Mechanism**: User sends query to a **Search Provider** node. +* **Execution**: Provider runs query against their massive hosted index and returns results. +* **Economics**: User pays micro-transaction (Verse Token) per query. +* **Pros**: Access to massive datasets (Petabytes). +* **Cons**: Privacy leakage (provider sees query). + +**Conclusion**: Verse must support **both**. The protocol needs a `QueryRequest` message type. + +--- + +## 4. Gap 3: The Crawler Economy + +Where does the index data come from? + +* **Passive**: Users publish their own browsing history (anonymized reports). +* **Active**: **Bounty-Based Crawling**. + 1. **Bounty**: A Curator creates a Verse for "Scientific Papers". They post a bounty: "100 Tokens for indexing arxiv.org". + 2. **Work**: Peers (Crawlers) scrape the target, extract text/metadata, and build an Index Artifact. + 3. **Proof**: Crawlers submit the Index Artifact. + 4. **Validation**: Validators spot-check the index (does it actually contain arxiv content?). + 5. **Reward**: Tokens released to Crawler. + +This turns "crawling" into a gig-economy job, decoupling it from the "search engine" company. + +--- + +## 5. Implementation Roadmap: Search Layer + +### Phase 1: Local Indexing (Graphshell Core) +* Integrate `tantivy`. +* Index local nodes (title, URL, tags, cached content). +* Enable `Ctrl+F` full-text search over local graph. + +### Phase 2: Index Export (Publishing) +* Command: "Publish Workspace Index". +* Action: Serialize local Tantivy segment for that workspace. +* Result: A `.index` blob stored in Verse. + +### Phase 3: Federated Search (Consumption) +* UI: "Add Search Source". Input: Verse/Peer ID. +* Mechanism: Download remote `.index` blob, mount as `MultiSearcher` in Tantivy. +* Result: Local queries hit both local and remote indices transparently. + +### Phase 4: Remote Query Protocol +* Define `Query` and `ResultSet` structs. +* Implement RPC over Iroh QUIC streams. +* Add "Search Provider" role to Peer capabilities. +``` + +c:\Users\mark_\OneDrive\code\rust\graphshell\design_docs\DOC_README.md +```diff +- verse_docs/technical_architecture/GRAPHSHELL_P2P_COLLABORATION.md - P2P collaboration architecture and integration model. +- verse_docs/research/2026-02-23_storage_economy_and_indices.md - Speculative research on storage economy (Proof of Access) and composable indices. +- verse_docs/implementation_strategy/verse_implementation_strategy.md - Hybrid economic model (Direct vs. Brokered) and technical implementation strategy. +- verse_docs/research/2026-02-23_modern_yacy_gap_analysis.md - Gap analysis and strategy for decentralized search (Index Artifacts, Remote Query). + +## Archive Checkpoints diff --git a/design_docs/mere_docs/research/2026-02-23_storage_economy_and_indices.md b/design_docs/mere_docs/research/2026-02-23_storage_economy_and_indices.md new file mode 100644 index 000000000..e69de29bb diff --git a/design_docs/mere_docs/research/2026-04-17_verse_distributed_index_protocol_v0_1.md b/design_docs/mere_docs/research/2026-04-17_verse_distributed_index_protocol_v0_1.md new file mode 100644 index 000000000..a5367e47c --- /dev/null +++ b/design_docs/mere_docs/research/2026-04-17_verse_distributed_index_protocol_v0_1.md @@ -0,0 +1,562 @@ +> **Recovered research. Never implemented. Not current direction.** +> +> Recovered 2026-08-16 from the git history of the archived `graphshell` +> repository (`Code/archive/graphshell`), whose entire `design_docs/` tree is +> deleted at HEAD, so this text survives only in history. +> +> - Original path: `design_docs/archive_docs/checkpoint_2026-04-17/verse_docs/technical_architecture/2026-04-17_verse_distributed_index_protocol_v0.1.md` +> - Source commit: `6ab4c22f` +> +> None of it was ever built. It is filed here as research, not as a plan and +> not as a statement of where Mere is going. It speaks the retired Verse +> vocabulary (per `TERMINOLOGY.md`, *Verse* is a retired term and the +> network scope is Mere at network scope), and its relative links point at +> donor docs that no longer exist locally. Read the text below for the ideas, +> not the direction. +> +> The filename was normalised from `..._v0.1.md` to `..._v0_1.md` to match its +> successor. It was already archived in the donor: VGCP replaced it as protocol +> authority the same day, and VGCP is recovered beside it as +> `2026-04-17_verse_graph_contribution_protocol_v0_1.md`. + +--- + +# Verse Distributed Index Protocol (VDIP) v0.1 + +> Archived on 2026-04-17. Superseded by [2026-04-17_verse_graph_contribution_protocol_v0_1.md](../../../../verse_docs/technical_architecture/2026-04-17_verse_graph_contribution_protocol_v0_1.md). This file was moved from active Verse technical architecture docs into the archive checkpoint after VGCP replaced it as the active protocol authority. + +**Status:** Draft v0.1 +**Document type:** Core protocol specification with reference profiles and Graphshell mapping +**Audience:** Protocol design, Graphshell implementation planning, future interop work + +This document promotes the April 16 draft set into a single canonical spec for the Verse distributed index protocol. It defines the protocol core, keeps transport and ecosystem choices out of the normative layer where possible, and records the current Graphshell implementation boundary explicitly. + +## 1. Scope + +VDIP defines how peers exchange signed, immutable, content-addressed graphlets: shareable Entry, Visit, and NavigationEdge artifacts plus optional acceleration packages. + +The protocol core covers: + +- graph artifact objects and signing rules +- content canonicalization and compatibility profiles +- community admission and revocation semantics +- search and ranking semantics over accepted local artifacts and communal graph structure + +The protocol core does not require: + +- public DHT-scale discovery +- raw HTML or WARC redistribution by default +- global cross-community federation +- on-chain economics, FLora, or Nostr/DVM integration + +Graphshell remains a host and renderer. Verso remains the bilateral peer layer. VDIP belongs to the community-scale Verse layer. + +## 2. Design Principles + +- Local-first: capture, canonicalization, indexing, and trust evaluation happen locally. +- Immutable artifacts: shared artifacts are append-only and content-addressed. +- Derived-data sharing by default: raw captured content remains local unless explicitly exported. +- Separation of durable truth and acceleration: durable graph artifacts stay valid even if the local search engine changes. +- Transport independence: the protocol core defines bytes and semantics, not one mandatory transport stack. +- Graph-native collaboration: communal structure matters, so navigation edges are first-class protocol objects rather than a deferred afterthought. + +## 3. Normative Language + +The key words `MUST`, `SHOULD`, and `MAY` are normative. + +## 4. Terminology + +| Term | Definition | +|------|------------| +| `Entry` | Stable local content/resource identity in Graphshell's local Entry/Visit/Owner substrate. | +| `Visit` | One situated local occurrence or arrival at an Entry. | +| `Owner` | Local actor or cursor context, such as a pane, tab, or device-local navigation owner. Owner context is never shared directly by VDIP v0.1. | +| `EntryCard` | Shareable projection of a local Entry. Carries canonicalized content identity and search-facing metadata. | +| `VisitCard` | Shareable projection of a local Visit. Carries situated occurrence, arrival semantics, and reference to an EntryCard. | +| `ObservationCard` | Historical draft term superseded here by `VisitCard`. | +| `NavigationEdgeCard` | Shareable projection of a transition between two VisitCards. | +| `SplitPackage` | Optional acceleration artifact containing a prebuilt search index bundle and manifest. | +| `CommunityManifest` | Community-scoped state defining governance, profile compatibility, and the active commit head. | +| `IndexCommit` | Signed append-only commit that adds artifacts and revocation references to a community history. | +| `RevocationRecord` | Immutable tombstone-like record removing an artifact from the active accepted set without rewriting history. | +| `ValidatorReceipt` | Optional signed validation result about a candidate artifact. | +| `UDC` | Universal Decimal Classification-derived semantic tagging namespace used for topical scoping, faceting, and ranking. | +| `Graphlet` | Any shareable subset of `{EntryCard, VisitCard, NavigationEdgeCard}` contributed together or separately. | +| `index_profile_hash` | BLAKE3 hash of the canonical index compatibility profile. | + +### 4.1 UDC Tags and Scope + +In this document, `UDC` refers to Universal Decimal Classification-derived semantic tags used as a shared topic namespace. + +- `udc_tags` are the tags attached to one observation. +- `udc_scope` is the topical boundary declared by a community. +- `udc_summary` is an aggregate summary over a package. +- `udc_match` is the ranking component that scores topical fit between query, observation, and community scope. + +Communities `SHOULD` document the exact encoding and granularity they accept. A typical tag may look like `004.738.5`, but v0.1 leaves representation as a profile-level choice so long as the community treats it consistently. + +## 5. Identity, Serialization, and Addressing + +- Identities `MUST` be Ed25519 keypairs. +- Signed objects `MUST` be serialized as canonical CBOR. +- Signatures `MUST` cover canonical manifest bytes, never opaque compressed payload bytes. +- Content identity `MUST` use raw BLAKE3-256. +- Implementations `MAY` expose those digests through a CIDv1 wrapper, but raw BLAKE3 is normative. +- Archive packaging `MUST` be deterministic before hashing. + +Rationale: raw BLAKE3 stays close to the underlying blob-transfer model while preserving a clean path to CIDv1 interop later. + +## 6. Compatibility Profiles + +VDIP defines three compatibility layers. Implementations `MUST` treat these profiles as part of interoperability, not as purely local implementation detail. + +### 6.1 CanonicalizationProfile + +Defines: + +- extraction method +- normalization rules +- volatile-field stripping +- whitespace and token normalization rules + +### 6.2 FingerprintProfile + +Defines: + +- exact-hash algorithm +- near-duplicate algorithm +- shingling rules +- parameterization and seeds + +For v0.1, implementations `MUST` support MinHash as the near-duplicate baseline. Parameterization remains profile-defined. + +### 6.3 IndexProfile + +Defines: + +- index schema +- tokenizer and analyzer chain +- canonicalization profile version +- fingerprint profile version +- search engine compatibility markers + +The compatibility fingerprint is: + +```text +index_profile_hash = BLAKE3( + canonical_cbor({ + canonicalization_profile, + fingerprint_profile, + index_schema, + tokenizer_config, + analyzer_config, + engine_compatibility, + }) +) +``` + +`SplitPackage`s are binary-compatible only when their `index_profile_hash` matches an accepted profile for the target community. + +## 7. Content Canonicalization Pipeline + +Before an EntryCard or VisitCard is recorded, content `MUST` be canonicalized: + +1. Extraction: readability-style main-content extraction or equivalent profile-defined algorithm. +2. Normalization: remove volatile tokens, session identifiers, ad markup, and profile-declared noise. +3. Exact fingerprinting: compute BLAKE3 over canonical extracted text or fields. +4. Near-duplicate fingerprinting: compute MinHash over profile-defined shingles. +5. UDC tagging: derive semantic tags from explicit user annotation, classifiers, or both. + +Raw HTML, WARC payloads, screenshots, and similar captures `SHOULD` remain local by default. + +### 7.1 Local Substrate Projection Boundary + +Graphshell's local model may distinguish `Entry`, `Visit`, and `Owner` as separate concepts. VDIP v0.1 preserves that distinction on the wire for shareable graph structure while keeping Owner private. + +- `EntryCard` is the shareable projection of a local Entry. +- `VisitCard` is the shareable projection of a local Visit and references one EntryCard. +- `NavigationEdgeCard` is the shareable projection of a transition between two VisitCards. +- `Owner` identity, owner-specific branch state, local cursor position, and other private coordination context never leave the device. +- Communities may exchange partial graphlets, but a valid graphlet preserves the dependency order `EntryCard -> VisitCard -> NavigationEdgeCard`. +- This is a conscious privacy boundary: shareable communal graph structure is preserved, private ownership structure is not. + +## 8. Core Protocol Objects + +### 8.1 TransitionKind + +```rust +enum TransitionKind { + LinkClick, + TypedUrl, + Redirect, + Back, + Forward, + Reload, + Restore, + Imported, + Unknown, +} +``` + +### 8.2 EntryCard + +```rust +struct EntryCard { + entry_id: [u8; 32], + canonical_url: String, + title: Option, + snippet: Option, + content_hash: [u8; 32], + minhash: Vec, + udc_tags: Vec, + profile_hash: [u8; 32], +} + +struct SignedEntryCard { + card: EntryCard, + signer: [u8; 32], + signature: [u8; 64], +} +``` + +`entry_id` is the stable content/resource identity carried across visits and packages. + +### 8.3 VisitCard + +```rust +struct VisitCard { + visit_id: [u8; 32], + entry_id: [u8; 32], + observed_at: u64, + arrival_kind: Option, + profile_hash: [u8; 32], +} + +struct SignedVisitCard { + card: VisitCard, + signer: [u8; 32], + signature: [u8; 64], +} +``` + +`VisitCard` is the graph-native successor to the earlier `ObservationCard` term. + +`arrival_kind` captures Visit-level transition semantics without publishing full owner-specific tree state. + +### 8.4 NavigationEdgeCard + +```rust +struct NavigationEdgeCard { + edge_id: [u8; 32], + from_visit: [u8; 32], + to_visit: [u8; 32], + transition_kind: TransitionKind, + observed_at: u64, + profile_hash: [u8; 32], +} + +struct SignedNavigationEdgeCard { + card: NavigationEdgeCard, + signer: [u8; 32], + signature: [u8; 64], +} +``` + +Edges are first-class because communal web-mapping depends on structure, not just isolated visits. + +### 8.5 SplitPackageManifest + +```rust +struct SplitPackageManifest { + package_id: [u8; 32], + payload_size: u64, + profile_hash: [u8; 32], + min_observed_at: u64, + max_observed_at: u64, + entry_count: u64, + visit_count: u64, + edge_count: u64, + entry_refs: Vec<[u8; 32]>, + visit_refs: Vec<[u8; 32]>, + edge_refs: Vec<[u8; 32]>, + udc_summary: Vec<(String, u32)>, + contributor: [u8; 32], + supersedes: Vec<[u8; 32]>, + created_at: u64, +} + +struct SignedSplitPackage { + manifest: SplitPackageManifest, + signature: [u8; 64], +} +``` + +The payload is an implementation-defined, deterministic archive of search-engine artifacts and graph-side acceleration data. The manifest is the signed and gossiped unit. + +### 8.6 CommunityManifest + +```rust +struct CommunityManifest { + community_id: [u8; 32], + name: String, + description: String, + udc_scope: Vec, + primary_profile: [u8; 32], + accepted_profiles: Vec<[u8; 32]>, + admin_keys: Vec<[u8; 32]>, + moderator_keys: Vec<[u8; 32]>, + ranking_weights: RankingWeights, + invite_policy: InvitePolicy, + preferred_head: Option<[u8; 32]>, + head_epoch: u64, +} + +struct RankingWeights { + bm25: f32, + udc_match: f32, + trust: f32, + freshness: f32, + novelty: f32, + graph_structure: f32, +} +``` + +`accepted_profiles` allows explicit dual-profile migration periods. `preferred_head` resolves the active commit-set question for v0.1. + +### 8.7 IndexCommit + +```rust +struct IndexCommit { + commit_id: [u8; 32], + parents: Vec<[u8; 32]>, + community_id: [u8; 32], + added_entries: Vec<[u8; 32]>, + added_visits: Vec<[u8; 32]>, + added_edges: Vec<[u8; 32]>, + added_packages: Vec<[u8; 32]>, + added_receipts: Vec<[u8; 32]>, + revocations: Vec<[u8; 32]>, + timestamp: u64, + author: [u8; 32], +} + +struct SignedCommit { + commit: IndexCommit, + signature: [u8; 64], +} +``` + +The commit graph is a DAG. Consumers `MUST` treat the active accepted set as the closure of commits reachable from the community's `preferred_head`. + +### 8.8 RevocationRecord + +```rust +enum ArtifactKind { + Entry, + Visit, + NavigationEdge, + SplitPackage, + ValidatorReceipt, +} + +struct RevocationRecord { + revocation_id: [u8; 32], + community_id: [u8; 32], + target_kind: ArtifactKind, + target_id: [u8; 32], + reason_code: String, + note: Option, + revoked_at: u64, + revoked_by: [u8; 32], +} + +struct SignedRevocation { + record: RevocationRecord, + signature: [u8; 64], +} +``` + +Revocation removes an artifact from the active set but does not rewrite historical records. + +### 8.9 ValidatorReceipt + +```rust +enum ValidationDecision { + Accept, + Reject, + Warn, +} + +struct ValidatorReceipt { + receipt_id: [u8; 32], + community_id: [u8; 32], + subject_kind: ArtifactKind, + subject_id: [u8; 32], + validator: [u8; 32], + decision: ValidationDecision, + reason_code: Option, + observed_at: u64, + expires_at: Option, +} + +struct SignedValidatorReceipt { + receipt: ValidatorReceipt, + signature: [u8; 64], +} +``` + +Validator receipts are optional in v0.1. Communities `MAY` require them in local admission policy, and they also provide the accountability path for curator-attested imports of donated unsigned artifacts. + +## 9. Community State and Head Selection + +VDIP resolves commit-head selection minimally in v0.1: + +- The authoritative active head is `CommunityManifest.preferred_head`. +- Only admin keys `MUST` be allowed to advance `preferred_head`. +- Consumers `MUST` apply only commits reachable from `preferred_head` when computing the active accepted set. +- Consumers `MAY` maintain local provisional heads for staging or review, but those heads `MUST NOT` be treated as canonical community state unless published through an admin-authorized manifest update. +- `head_epoch` `MUST` increase monotonically when `preferred_head` changes. + +This keeps DAG history available without leaving active-state selection undefined. + +This is a pragmatic v0.1 simplification, not a claim that admin-published heads are the intended long-term decentralization model. + +## 10. Admission, Revocation, and Profile Migration + +- Communities `MUST` define an invite or access policy. +- Communities `SHOULD` require individually signed `EntryCard`, `VisitCard`, and `NavigationEdgeCard` artifacts for ordinary participation. +- Communities `MAY` apply additional admission checks using validator receipts, trust scores, or local moderation rules. +- Communities `MAY` admit donated unsigned graph artifacts only through signed attestation by a validator, curator, or import gateway. +- When unsigned donated artifacts are admitted, accountability attaches to the attesting signer, not to an anonymous donor. +- An accepted `VisitCard` `MUST` reference an accepted `EntryCard` present in the same or an earlier reachable commit. +- An accepted `NavigationEdgeCard` `MUST` reference accepted `VisitCard`s present in the same or an earlier reachable commit. +- Revoked artifacts `MUST NOT` remain in the active accepted set after the relevant `RevocationRecord` becomes reachable from the preferred head. +- Communities `SHOULD` use `accepted_profiles` to support controlled profile migration. +- During migration, communities `MAY` accept graph artifacts from more than one profile, but `SplitPackage` mounting still requires exact binary compatibility with one accepted profile. + +## 11. Trust and Ranking + +Trust is community-local and implementation-defined, but the protocol assumes these inputs are available to local policy: + +- invite lineage +- endorsements and flags +- identity age +- utilization in local search +- novelty relative to existing community content +- graph structure quality, edge frequency, and redirect anomaly signals + +Implementations `MAY` compute trust using EigenTrust-like or other graph-based algorithms. + +Recommended enforcement rule for v0.1: + +- trust gates `SplitPackage` mounting and optional artifact admission +- query-time ranking combines textual score with UDC relevance, trust, freshness, novelty, and graph-structure signals + +## 12. Search Semantics + +Search execution `MUST` be local over the consumer's accepted artifacts. + +1. Scope the query to one or more communities. +2. Gather locally accepted entries, visits, edges, and mounted split packages reachable from each community's preferred head. +3. Query across the local search engine's readers for entries and visits. +4. Deduplicate exact matches by `entry_id` and `content_hash`. +5. Cluster near-duplicates using MinHash similarity. +6. Apply visit-level freshness and arrival semantics. +7. Apply edge-derived graph priors, redirect-chain penalties, or provenance/path boosts where a community chooses to use them. +8. Rank with community-defined weights. +9. Return results with provenance. + +In v0.1, exact identity is centered on `EntryCard`, situated relevance is centered on `VisitCard`, and communal structure is centered on `NavigationEdgeCard`. + +The composite score is community-defined but typically has the form: + +```text +score = w_bm25 * bm25 + + w_udc * udc_match + + w_trust * contributor_trust + + w_fresh * freshness_decay(observed_at) + + w_novelty * novelty_vs_community + + w_graph * graph_structure_signal +``` + +Global mandatory merge is out of scope for v0.1. Fan-out across accepted local artifacts is sufficient. + +## 13. Privacy Baseline + +- VDIP v0.1 addresses artifact-sharing privacy more directly than query privacy. +- Query privacy is typically more sensitive than artifact privacy because queries reveal current intent, not only past action. +- Collection and sharing of Entry, Visit, and NavigationEdge data are opt-in and community-scoped. +- Raw capture data `SHOULD` remain local by default. +- Shared artifacts `SHOULD` prefer derived search data over raw page archives. +- Query privacy is separate from artifact privacy and is deferred beyond v0.1. +- Owner context, owner-specific branch state, and private cursor structure `MUST NOT` be shared as protocol artifacts. +- Implementations `SHOULD` support local-only search over already-fetched artifacts so remote query disclosure is not mandatory. + +## 14. Reference Profiles + +These profiles are reference implementation guidance, not mandatory protocol rules. + +### 14.1 Profile A: Iroh-First Trusted Exchange + +- blob transfer: `iroh-blobs` +- community announcements: `iroh-gossip` +- replicated community state: `iroh-docs` +- connectivity and relays: `iroh` + +This is the most direct fit for early Graphshell experimentation. + +### 14.2 Profile B: Broader Discovery Overlay + +Public discovery overlays such as libp2p-based routing or other announcement fabrics may be added later. They are not required for v0.1 conformance. + +## 15. Graphshell Implementation Mapping + +The protocol is ahead of the current Graphshell implementation. These reuse points and gaps are the current boundary: + +- [../../../crates/graph-memory/src/lib.rs](../../../crates/graph-memory/src/lib.rs) and [../../../crates/graphshell-core/src/graph/mod.rs](../../../crates/graphshell-core/src/graph/mod.rs) express the local Entry/Visit/Owner-style navigation substrate. That substrate is not incidental to Verse; it is the source model for shareable communal graphlets. +- In that framing, `EntryCard` is the shareable projection of Entry identity, `VisitCard` is the shareable projection of situated occurrence, and `NavigationEdgeCard` is the shareable projection of communal traversal structure. +- Graph-memory matters because it lets Verse preserve the web's collaborative shape without exporting private owner context. The protocol keeps communal graph structure while dropping owner-specific cursors and branches. + +- [../../../mods/native/verse/mod.rs](../../../mods/native/verse/mod.rs) already provides Ed25519-backed iroh identity, trusted-peer storage, and workspace-grant concepts that can seed community identity and admission work. +- [../../../model/archive.rs](../../../model/archive.rs) already contains signed portable archive objects and privacy classes; that is a useful precedent for signed artifact envelopes, but it is not yet VDIP's object model. +- [../../../app/clip_capture.rs](../../../app/clip_capture.rs) already exposes structured capture data from web content; this is a precursor to canonicalization, not the canonicalization pipeline itself. +- [../../../services/query/mod.rs](../../../services/query/mod.rs) and [../../../services/facts/mod.rs](../../../services/facts/mod.rs) provide local structured querying over projected history facts, but they do not yet implement distributed graphlet exchange, split-package search, or community ranking. + +Missing pieces before Graphshell can claim a VDIP implementation: + +- canonicalization profile machinery +- BLAKE3 and MinHash artifact fingerprinting for entries and visits +- first-class emission of shareable visit and navigation-edge graphlets from graph-memory +- Tantivy or equivalent split-package production and mounting +- community manifest replication beyond bilateral sync +- commit and revocation application logic +- trust-graph computation for admission or mount-time gating + +## 16. Rollout Order + +1. EntryCard, VisitCard, and NavigationEdgeCard serialization with canonical CBOR and signatures. +2. Canonicalization profiles plus BLAKE3 and MinHash generation for graph artifacts. +3. Local-only community manifest and preferred-head handling. +4. Two-peer graphlet exchange using a reference transport. +5. Local search across accepted entries and visits, with optional edge-informed ranking and provenance. +6. SplitPackage production and mounting over accepted graphlets. +7. Community trust and optional validator receipts. +8. Broader discovery overlays and advanced privacy features. + +## 17. Deferred Work + +- batch-signing extensions +- dense graphlet summarization and higher-order communal path models +- head selection without admin-authoritative `preferred_head` +- private community discovery with existence-hiding properties +- advanced query privacy +- mandatory public discovery overlays +- cross-community search federation +- raw snapshot redistribution by default +- economics, FLora, and Nostr/DVM integration + +## 18. References + +- [2026-04-16_verse_index_protocol_drafts.md](../../../../verse_docs/technical_architecture/2026-04-16_verse_index_protocol_drafts.md) +- [VERSE_AS_NETWORK.md](../../../../verse_docs/technical_architecture/VERSE_AS_NETWORK.md) +- [2026-02-23_verse_tier2_architecture.md](../../../../verse_docs/technical_architecture/2026-02-23_verse_tier2_architecture.md) diff --git a/design_docs/mere_docs/research/2026-04-17_verse_graph_contribution_protocol_v0_1.md b/design_docs/mere_docs/research/2026-04-17_verse_graph_contribution_protocol_v0_1.md new file mode 100644 index 000000000..d59f9c935 --- /dev/null +++ b/design_docs/mere_docs/research/2026-04-17_verse_graph_contribution_protocol_v0_1.md @@ -0,0 +1,885 @@ +> **Recovered research. Never implemented. Not current direction.** +> +> Recovered 2026-08-16 from the git history of the archived `graphshell` +> repository (`Code/archive/graphshell`), whose entire `design_docs/` tree is +> deleted at HEAD, so this text survives only in history. +> +> - Original path: `design_docs/verse_docs/technical_architecture/2026-04-17_verse_graph_contribution_protocol_v0_1.md` +> - Source commit: `6ab4c22f` +> +> None of it was ever built. It is filed here as research, not as a plan and +> not as a statement of where Mere is going. It speaks the retired Verse +> vocabulary (per `TERMINOLOGY.md`, *Verse* is a retired term and the +> network scope is Mere at network scope), and its relative links point at +> donor docs that no longer exist locally. Read the text below for the ideas, +> not the direction. +> +> This is the later half of a protocol pair: VGCP superseded VDIP on the same +> day. Its predecessor is recovered beside it as +> `2026-04-17_verse_distributed_index_protocol_v0_1.md`. + +--- + +# Verse Graph Contribution Protocol (VGCP) v0.1 + +**Status:** Draft v0.1 +**Document type:** Core protocol specification with reference profiles and Graphshell mapping +**Audience:** Protocol design, Graphshell implementation planning, future interop work +**Supersedes:** 2026-04-17_verse_distributed_index_protocol_v0.1.md (VDIP v0.1) + +This document replaces VDIP v0.1. The change is substantive, not cosmetic. VDIP treated search observations as the unit of contribution and edges as an afterthought. VGCP treats graph contributions as the unit, with isolated observations as the degenerate (zero-edge) case. The protocol is graph-native; Verses are communities that map the web, and the map is the shareable artifact. + +## 1. Scope + +VGCP defines how peers exchange graph-shaped knowledge about the web — across HTTP, smolweb protocols (Gemini, Gopher, Scroll, Spartan, and others), and in principle any protocol whose resources admit canonicalization — as signed, immutable, content-addressed artifacts. + +The protocol core covers: + +- the Entry/Visit/Owner substrate and its projection rule +- graph contribution objects and signing rules +- protocol-neutral content canonicalization and compatibility profiles +- structural edge semantics grounded in protocol-defined relationships +- attestation and aggregation semantics across contributions +- community admission, revocation (whole and fragmentary), profile migration, and declarative filtering +- search and ranking semantics over accepted local artifacts + +The protocol core does not require: + +- public DHT-scale discovery +- raw payload redistribution by default +- global cross-community federation +- on-chain economics, storage-time-bank tokenization, or governance staking +- Nostr/DVM, Matrix, or other social host integrations + +Graphshell remains a host and renderer. Verso remains the bilateral peer layer. VGCP belongs to the community-scale Verse layer. + +## 2. Design Principles + +- Graph-native: contributions are subgraphs. Single orphans are the zero-edge case, not a separate artifact type. +- Structure over behavior: shared edges describe protocol-defined relationships between resources, not local navigation events. How a contributor traversed the graph is optional aggregate metadata; the graph itself is structural. +- Protocol-neutral at the core: the object model accommodates any protocol whose resources have stable canonicalization and whose references can be projected into edges. +- Local-first: capture, canonicalization, indexing, and trust evaluation happen locally. +- Immutable artifacts, mutable projection: shared artifacts are append-only and content-addressed. The community's *active accepted set* is a read-time projection that honors revocations and filters. +- Projection-not-mirror: shared artifacts are projections of local state that strip Visit and Owner context by construction. +- Derived-data sharing by default: raw captured content remains local unless explicitly exported. +- Transport independence: the protocol core defines bytes and semantics, not one mandatory transport stack. +- Communal authority: manifest-level authority is attested by the process the community chose, not by a designated individual. + +## 3. Normative Language + +The key words `MUST`, `SHOULD`, and `MAY` are normative. + +## 4. Terminology + +| Term | Definition | +|------|------------| +| `Entry` | Deduplicated resource identity (the node). A URL plus canonicalized content, under a community's canonicalization profile. Shareable. | +| `Visit` | A situated local occurrence of an Entry: when the contributor was there, in what context, via what transition. Local-only; never shared. | +| `Owner` | A local cursor-bearing actor (tab, pane, session, graph view). Local-only; never shared. | +| `EntryRecord` | The shareable projection of an Entry: identity hash, content-equivalence hash, metadata, and community-scoped fingerprints. | +| `EdgeRecord` | The shareable projection of a structural relationship between two Entries. | +| `GraphContribution` | A signed, canonicalized bundle of EntryRecords and EdgeRecords from one contributor. The unit of contribution. | +| `SplitPackage` | Optional acceleration artifact: a prebuilt search-index bundle over accepted contributions. | +| `CommunityManifest` | Community-scoped state defining governance, profile compatibility, filter policy, and the active commit head. | +| `SignedCommunityManifest` | Canonical manifest bytes plus the set of governance attestations that authorize this manifest version. | +| `ManifestGovernance` | Rule that determines which attestations authorize a manifest update. | +| `ManifestAttestation` | One attester's signature over canonical `CommunityManifest` bytes. | +| `IndexCommit` | Signed append-only commit that adds contributions and revocations to a community history. | +| `RevocationRecord` | Immutable tombstone-like record removing a whole contribution, or specific Entries or Edges within it, from the active accepted set. | +| `FilterPolicy` | Declarative community-level rules applied at aggregation time to include or exclude contribution elements. | +| `ValidatorReceipt` | Optional signed validation result about a candidate artifact. | +| `KeyRotation` | Cross-signed declaration that one identity key is succeeded by another for contributor-equivalence purposes. | +| `Attestation` | The fact of an EntryRecord or EdgeRecord appearing in a contribution; weighting and aggregation are attestation-derived. | +| `profile_hash` | Wire-field name for the BLAKE3 hash of the canonical index compatibility profile; prose may call this the community's `index_profile_hash`. | + +## 5. The Entry/Visit/Owner Substrate + +VGCP presumes a local data model compatible with the Entry/Visit/Owner substrate. + +- **Entry**: deduplicated identity of a web resource. One Entry per canonicalized (URL, content) tuple under a profile. +- **Visit**: a concrete persisted occurrence. Always distinct; `VisitId`s are never reused. +- **Owner**: a local cursor-bearing actor carrying Visit parentage and per-Owner forward-choice state. + +### 5.1 The Projection Rule + +A GraphContribution is the Entry-level projection of a subgraph of the contributor's local Entry graph — the graph induced by structural relationships between Entries the contributor has captured. The projection: + +- `MUST` preserve: Entry identity and content-equivalence hashes, community-relevant Entry metadata, Edge endpoints, edge kinds and their protocol-defined metadata. +- `MUST` strip: VisitIds, Visit timestamps at navigation granularity, Owner identity, Owner forward-choice state, local parent pointers between Visits. +- `SHOULD` coarsen: temporal information. Timestamps `SHOULD` be expressed in buckets (e.g., day-granularity). +- `MAY` include: optional aggregate navigation metadata on Edges (traversal counts, observation windows) as a separate concern from the Edge's structural existence. + +This projection rule is the privacy boundary between Graphshell (local) and Verse (shared). It is structural, not conventional. + +### 5.2 Structural vs. Behavioral Edges + +A central design commitment: Edges represent *structural* relationships that any contributor fetching the source resource would derive identically — links defined in a page's markup, transclusions declared in its syntax, redirects declared by the server. Edges are not records of user behavior. + +Behavioral navigation (what was clicked, what was typed, what was back-buttoned) lives on local Visits and does not cross the projection boundary as structural signal. It `MAY` cross as optional aggregate metadata — "this `Link` was traversed N times in this window by this contributor" — but the existence of the Edge is structural, not behavioral. + +Rationale: structural edges are verifiable by any contributor re-canonicalizing the source Entry, which collapses most forgery attack surface. Behavioral edges are attestation-only and provide no basis for cross-contributor corroboration. + +## 6. Identity, Serialization, and Addressing + +- Identities `MUST` be Ed25519 keypairs. +- Signed objects `MUST` be serialized as canonical CBOR. +- Signatures `MUST` cover canonical manifest bytes, never opaque compressed payload bytes. +- Content identity `MUST` use raw BLAKE3-256. +- Implementations `MAY` expose those digests through a CIDv1 wrapper, but raw BLAKE3 is normative. +- Archive packaging `MUST` be deterministic before hashing. + +### 6.1 Key Rotation + +Identity keys are long-lived but not assumed permanent. + +```rust +struct KeyRotation { + rotation_id: [u8; 32], + old_key: [u8; 32], + new_key: [u8; 32], + rotated_at: u64, +} + +struct SignedKeyRotation { + rotation: KeyRotation, + old_signature: [u8; 64], + new_signature: [u8; 64], +} +``` + +Rules for v0.1: + +- a rotation is valid only if both the old key and new key sign the same + canonical `KeyRotation` bytes, +- rotations are not active until a community explicitly accepts them through an + `IndexCommit`, +- accepted rotations form a linear chain in v0.1; forked rotation graphs are + deferred, +- contributions signed by a retired key remain valid historical facts, +- lost predecessor keys cannot rotate; recovery mechanisms are deferred. + +## 7. Compatibility Profiles + +VGCP defines three compatibility layers. + +### 7.1 CanonicalizationProfile + +Protocol-scoped. A profile may declare canonicalization rules for one or more protocols. Mixed-protocol communities declare rules for each protocol they accept. + +Per protocol, a profile defines: + +- resource fetch and extraction method +- main-content extraction (where applicable) +- normalization rules (volatile-field stripping, session-identifier scrubbing, whitespace and token normalization) +- metadata extraction, including explicit UDC tags where the protocol supports them (Scroll is the notable case) + +### 7.2 FingerprintProfile + +Defines: + +- exact-hash algorithm (BLAKE3-256 in v0.1) +- near-duplicate algorithm (MinHash in v0.1) +- shingling rules, parameterization, and seeds + +### 7.3 IndexProfile + +Defines: + +- entry record schema +- edge record schema +- edge kind enumeration (see Section 9.3) +- protocol-specific profile_extension schemas (see Section 9.3) +- tokenizer and analyzer chain +- canonicalization profile version +- fingerprint profile version +- search engine compatibility markers + +Compatibility fingerprint: + +```text +index_profile_hash = BLAKE3( + canonical_cbor({ + canonicalization_profile, + fingerprint_profile, + entry_schema, + edge_schema, + edge_kind_enum, + protocol_extension_schemas, + tokenizer_config, + analyzer_config, + engine_compatibility, + }) +) +``` + +`SplitPackage`s are binary-compatible only when their `index_profile_hash` matches an accepted profile for the target community. + +On the wire, structures carry this digest in a field named `profile_hash`. +`index_profile_hash` is prose shorthand for the same compatibility fingerprint. + +## 8. Canonicalization Pipeline + +Before a contribution is assembled, local state `MUST` be canonicalized: + +1. **Entry canonicalization**: for each Entry, apply protocol-appropriate extraction and normalization per the target community's CanonicalizationProfile. Compute `content_hash` as the hash of canonical `(URL, content)` bytes under the profile, compute `content_only_hash` as the hash of canonical content bytes under that same profile, and compute `minhash` over the canonicalized text projection used for near-duplicate clustering. Derive or extract UDC tags — from explicit document metadata where the protocol supports it, from classifier output otherwise. +2. **Structural edge extraction**: for each Entry in the contribution, extract protocol-defined structural relationships (links, includes, redirects, references) to other Entries also in the contribution. Edges pointing outside the contribution's Entry set are excluded. +3. **Optional behavioral enrichment**: Edges `MAY` be annotated with aggregate traversal metadata from local Visits. Timestamps are coarsened to buckets. Duplicate (from, to, kind) tuples are merged. +4. **Privacy-class filtering**: contributor-local policy `MUST` exclude any Entry marked private, and `MUST` drop any Edge whose endpoints include an excluded Entry. Filtering happens before canonicalization so excluded material does not influence the signed bytes. +5. **Canonical ordering**: Entries sorted by `content_hash`; Edges sorted by `(from, to, kind, canonical_extension_digest)`. +6. **Signing**: canonical CBOR of the GraphContribution is signed with the contributor's Ed25519 key. + +Raw payload capture data `SHOULD` remain local by default. + +## 9. Core Protocol Objects + +### 9.1 GraphContribution + +The primary artifact. + +```rust +struct GraphContribution { + contribution_id: [u8; 32], // BLAKE3 of canonical manifest minus this field and signature + entries: Vec, // 1..N; orphans are valid contributions + edges: Vec, // 0..M; zero edges is valid + contributor: [u8; 32], + profile_hash: [u8; 32], + created_at: u64, // coarse timestamp (bucket granularity recommended) +} + +struct SignedGraphContribution { + contribution: GraphContribution, + signature: [u8; 64], +} +``` + +**Structural invariants:** + +- Every `EdgeRecord.from` and `EdgeRecord.to` `MUST` equal a `content_hash` of some `EntryRecord` in the same contribution. +- A contribution `MAY` be disconnected. A single contribution may contain multiple connected components and orphan nodes. +- `entries.len() >= 1`. +- `edges` `MAY` be empty. + +### 9.2 EntryRecord + +```rust +struct EntryRecord { + url: String, // includes scheme: "https://", "gemini://", "gopher://", "scroll://", etc. + protocol: String, // canonical lowercase: "https", "gemini", "gopher", "scroll", ... + content_hash: [u8; 32], // BLAKE3 of canonical (URL, content) bytes; Entry identity + content_only_hash: [u8; 32], // BLAKE3 of canonical content bytes; cross-URL exact-content equivalence + minhash: Vec, + udc_tags: Vec, + udc_source: UdcSource, // Explicit, Classifier, Hybrid + title: Option, + snippet: Option, + observed_at_bucket: u64, +} + +enum UdcSource { + Explicit, // UDC declared in document (e.g., Scroll metadata) + Classifier, // UDC inferred by local classifier + Hybrid, // Explicit, supplemented by classifier +} +``` + +Entry identity in v0.1 is canonical `(URL, content)` rather than content-only. +Two mirrors or republications carrying the same body bytes at different URLs are +distinct Entries and therefore have distinct `content_hash` values, because +canonicalization includes canonical URL in the identity hash. + +`content_only_hash` is an additional exact-content equivalence fingerprint, not +an alternate Entry identity. Entries that share a `content_only_hash` remain +separate Entries with separate attestation and ranking histories; the shared +hash exists so implementations can ask read-time questions like "which Entries +carry this same canonical content across different URLs?" without collapsing +those Entries into one node. + +`udc_source` is a signal-quality indicator. Community ranking `MAY` weight `Explicit` contributions more heavily. + +### 9.3 EdgeRecord + +```rust +struct EdgeRecord { + from: [u8; 32], // content_hash of source EntryRecord + to: [u8; 32], // content_hash of destination EntryRecord + kind: EdgeKind, + label: Option, // universal: visible link/menu/reference text where applicable + source_protocol: String, // protocol of the 'from' Entry + profile_extensions: Vec<(String, Vec)>, // profile-defined, protocol-scoped metadata + // Optional aggregate navigation metadata; may be omitted entirely. + traversal_count: Option, + first_observed_bucket: Option, + last_observed_bucket: Option, +} + +enum EdgeKind { + Link, // one resource references another for the reader to follow + Include, // one resource transcludes another (mostly HTTP/HTML: iframe, img, script, style) + Redirect, // one resource replaces another (HTTP 3xx, meta-refresh, protocol-defined redirection) + Reference, // relationship metadata between resources (canonical, alternate, prev/next, hreflang; mostly HTML) +} +``` + +`EdgeKind` is closed and bounded by what's defined in widely-deployed protocol standards rather than by local user behavior. Additions require a profile bump. + +`profile_extensions` carry protocol-specific metadata. The IndexProfile declares the schema per protocol. Examples of expected extension fields: + +- `source_protocol = "https"`; `kind = Link`: `region` (Nav, Main, Aside, Footer, Header, Other), `rel` (nofollow, noopener, sponsored, ...) +- `source_protocol = "gemini"`; `kind = Link`: typically none beyond `label` +- `source_protocol = "gopher"`; `kind = Link`: `item_type` (Gopher item type octet) +- `source_protocol = "scroll"`; `kind = Link`: Scroll-defined `Link` context; possibly UDC-scoped + +Implementations `MUST NOT` populate `profile_extensions` with keys the profile does not declare. Canonicalization sorts extension pairs by key. + +**Open uncertainty (disclaimed as future work):** The granularity of HTML `Link` context — specifically, whether `region` as a coarse enum (Nav, Main, Aside, Footer, Header, Other) is the right resolution, or whether finer DOM-path or CSS-selector context should be preserved — is not settled. Coarse region is likely sufficient for ranking and compact for serialization, but may under-specify contexts communities care about. This is deferred to a later profile revision; the `profile_extensions` structure accommodates refinement without breaking the edge model. + +### 9.4 SplitPackageManifest + +```rust +struct SplitPackageManifest { + package_id: [u8; 32], + payload_size: u64, + profile_hash: [u8; 32], + min_observed_at: u64, + max_observed_at: u64, + contribution_refs: Vec<[u8; 32]>, + entry_count: u64, + edge_count: u64, + udc_summary: Vec<(String, u32)>, + contributor: [u8; 32], + supersedes: Vec<[u8; 32]>, + created_at: u64, +} + +struct SignedSplitPackage { + manifest: SplitPackageManifest, + signature: [u8; 64], +} +``` + +### 9.5 CommunityManifest + +```rust +struct CommunityManifest { + community_id: [u8; 32], + name: String, + description: String, + version: u64, + previous_manifest_hash: Option<[u8; 32]>, + udc_scope: Vec, + supported_protocols: Vec, // "https", "gemini", "scroll", etc. + primary_profile: [u8; 32], + accepted_profiles: Vec<[u8; 32]>, + admin_keys: Vec<[u8; 32]>, + moderator_keys: Vec<[u8; 32]>, + manifest_governance: ManifestGovernance, + ranking_weights: RankingWeights, + filter_policy: FilterPolicy, + invite_policy: InvitePolicy, + preferred_head: Option<[u8; 32]>, + head_epoch: u64, +} + +struct RankingWeights { + bm25: f32, + udc_match: f32, + udc_explicit_bonus: f32, // extra weight for Entries with UdcSource::Explicit + trust: f32, + freshness: f32, + novelty: f32, + edge_support: f32, // weight of edge-attestation evidence + structural_centrality: f32, // weight of graph-centrality signal +} + +struct FilterPolicy { + entry_url_blocklist: Vec, // URL patterns, profile-defined matching + entry_url_allowlist: Option>, // if Some, only matches admitted + entry_content_blocklist: Vec<[u8; 32]>, // specific content_hashes + udc_tag_blocklist: Vec, + udc_tag_allowlist: Option>, + protocol_blocklist: Vec, // e.g., block "finger" + protocol_allowlist: Option>, + edge_kind_blocklist: Vec, // e.g., block Include to limit tracking-pixel surfaces + contributor_blocklist: Vec<[u8; 32]>, + max_entries_per_contribution: Option, + max_edges_per_contribution: Option, +} + +enum ManifestGovernance { + Genesis { + bootstrap_key: [u8; 32], + }, + Threshold { + steward_keys: Vec<[u8; 32]>, + threshold: u32, + }, + Delegated { + required_attesters: Vec<[u8; 32]>, + threshold: u32, + policy_doc: Option, + }, +} + +struct ManifestAttestation { + attester: [u8; 32], + signature: [u8; 64], +} + +struct SignedCommunityManifest { + manifest: CommunityManifest, + attestations: Vec, +} +``` + +Rules for manifest governance in v0.1: + +- `Genesis` is valid only for version `0` manifests and is self-attesting, +- non-genesis manifests `MUST NOT` use `Genesis`, +- `Threshold` uses a fixed steward key set and an `N-of-M` rule, +- `Delegated` is the protocol hook for voting systems, rotating councils, or + automated governance; protocol validation is still only on + `required_attesters` plus `threshold`, while `policy_doc` is advisory for + humans. + +When validating manifest `N+1`, consumers `MUST` validate its attestations +against manifest `N`'s `manifest_governance`, regardless of what governance rule +`N+1` declares for its own successors. Governance evolves, but each transition +is validated by the prior rule. + +Manifest verification procedure: + +1. Canonicalize the candidate `CommunityManifest` bytes using canonical CBOR. +2. Verify each `ManifestAttestation.signature` against those bytes. +3. If `version == 0`, require `previous_manifest_hash == None`, require the + manifest to declare `ManifestGovernance::Genesis`, and require a valid + self-attestation by the declared `bootstrap_key`. +4. If `version > 0`, load the previous accepted manifest, read its + `manifest_governance`, and evaluate the candidate attestations against that + prior rule. +5. Succeed iff the set of distinct valid attesters satisfies the applicable + governance rule. + +Filter policy updates, ranking weight changes, and profile acceptance changes +are manifest updates governed by `manifest_governance`; they are not admin-only +actions. + +### 9.6 IndexCommit + +```rust +struct IndexCommit { + commit_id: [u8; 32], + parents: Vec<[u8; 32]>, + community_id: [u8; 32], + added_contributions: Vec<[u8; 32]>, + added_packages: Vec<[u8; 32]>, + added_receipts: Vec<[u8; 32]>, + added_rotations: Vec<[u8; 32]>, + revocations: Vec<[u8; 32]>, + timestamp: u64, + author: [u8; 32], +} + +struct SignedCommit { + commit: IndexCommit, + signature: [u8; 64], +} +``` + +### 9.7 RevocationRecord + +Revocation supports whole-artifact removal and fragmentary removal of Entries or Edges within a contribution. Signed artifacts are never mutated; fragmentary revocation applies at read-time projection. + +```rust +enum ArtifactKind { + GraphContribution, + SplitPackage, + ValidatorReceipt, + KeyRotation, +} + +enum RevocationTarget { + WholeArtifact { + kind: ArtifactKind, + target_id: [u8; 32], + }, + Entry { + contribution_id: [u8; 32], + content_hash: [u8; 32], + }, + Edge { + contribution_id: [u8; 32], + from: [u8; 32], + to: [u8; 32], + kind: EdgeKind, + }, +} + +struct RevocationRecord { + revocation_id: [u8; 32], + community_id: [u8; 32], + target: RevocationTarget, + reason_code: String, + note: Option, + revoked_at: u64, + revoked_by: [u8; 32], +} + +struct SignedRevocation { + record: RevocationRecord, + signature: [u8; 64], +} +``` + +Revocation of an Entry within a contribution `MUST` also suppress (at projection time) all Edges in that contribution touching that Entry. Consumers compute this closure when assembling the active accepted set. + +At high revocation ratios within a single contribution, communities `MAY` +choose to issue `WholeArtifact` revocation rather than many fragmentary +revocations. This is an operational heuristic; the protocol does not define a +threshold. + +### 9.8 ValidatorReceipt + +```rust +enum ValidationDecision { + Accept, + Reject, + Warn, +} + +struct ValidatorReceipt { + receipt_id: [u8; 32], + community_id: [u8; 32], + subject_kind: ArtifactKind, + subject_id: [u8; 32], + validator: [u8; 32], + decision: ValidationDecision, + reason_code: Option, + observed_at: u64, + expires_at: Option, +} + +struct SignedValidatorReceipt { + receipt: ValidatorReceipt, + signature: [u8; 64], +} +``` + +Validator receipts are optional in v0.1. + +## 10. Attestation and Aggregation + +Community-level state is the aggregation of accepted contributions after filter and revocation are applied. + +### 10.1 Active Accepted Set + +For a given community, the active accepted set `MUST` be computed as: + +1. Start with all contributions referenced by commits reachable from `preferred_head`. +2. Drop contributions whose `WholeArtifact` revocation is reachable from `preferred_head`. +3. For each remaining contribution, apply `FilterPolicy`: + - drop the contribution if contributor is in `contributor_blocklist` or contribution exceeds size limits + - drop Entries that match blocklist rules or fail allowlist; drop all Edges touching those Entries + - drop Edges that match `edge_kind_blocklist` or whose `source_protocol` is blocked +4. Apply fragmentary revocations: drop specifically-revoked Entries (and Edges touching them), drop specifically-revoked Edges. +5. The surviving Entry and Edge records are the active accepted set. + +For contributor-equivalence counting, identities connected by an accepted +linear `KeyRotation` chain are treated as one contributor. Rotated-and-accepted +keys do not produce separate attestation counts. + +### 10.2 Entry Aggregation + +For each unique `content_hash` in the active accepted set: + +- **attestation count**: distinct contributors attesting this Entry +- **trust-weighted attestation**: sum of per-contributor trust over attestations +- **first-seen / last-seen buckets** +- **UDC tag consensus**: weighted union, preferring `UdcSource::Explicit` attestations + +Multiple contributions from the same contributor count as one attestation. +Contributions signed by keys linked through an accepted `KeyRotation` chain are +treated as coming from that same contributor for counting purposes. + +### 10.3 Content Equivalence + +Implementations `MAY` derive a read-time content-equivalence view keyed by +`content_only_hash`. + +For each unique `content_only_hash` in the active accepted set, implementations +`MAY` compute: + +- the set of distinct `content_hash` identities carrying that content, +- the set of distinct URLs carrying that content, +- distinct-contributor attestation counts across all matching Entries, +- trust-weighted corroboration across those Entries, +- first-seen / last-seen buckets for the equivalence class. + +This is an equivalence view, not a merge rule. Content equivalence `MUST NOT` +erase per-Entry identity, provenance, or trust state. Ranking remains per +Entry unless an implementation explicitly introduces content-cluster-aware +presentation at query time. + +### 10.4 Edge Aggregation + +For each unique `(from, to, kind)` tuple (with protocol-specific extensions compared per the profile): + +- **attestation count**: distinct contributors attesting this structural edge +- **trust-weighted attestation** +- **aggregate traversal count**: optional; sum of per-contribution counts, trust-weighted +- **first-seen / last-seen buckets** + +Edge attestation is the primary check on structural manipulation: a forged edge lacks corroboration from other contributors who fetched the source Entry and derived its structural links. + +### 10.5 Aggregation is Read-Time + +Aggregation `MUST NOT` produce a new signed artifact. It is a local computation over the active accepted set, governed by the profile. This preserves the append-only artifact model. + +## 11. Community State, Manifest Governance, and Head Selection + +### 11.1 Manifest Updates vs. Head Advancement + +VGCP separates two kinds of authority: + +- **high-frequency operational authority**: advancing `preferred_head`, bulk + operational moderation, and other day-to-day workflow handled by + `admin_keys` and `moderator_keys`, +- **low-frequency governance authority**: changing the manifest itself, + including `ranking_weights`, `filter_policy`, `accepted_profiles`, and future + successors, handled by attestations satisfying `manifest_governance`. + +Rules: + +- the authoritative active head is `CommunityManifest.preferred_head`, +- only `admin_keys` `MUST` be allowed to advance `preferred_head`, +- `moderator_keys` handle per-artifact operational decisions such as receipts, + revocations, and moderation workflows, +- manifest updates are authorized only by attestations satisfying the previous + manifest's `manifest_governance`, +- `admin_keys` do not bypass manifest governance, +- consumers `MUST` apply only commits reachable from the latest accepted + manifest's `preferred_head` when computing the active accepted set, +- consumers `MAY` maintain local provisional heads for staging; such heads + `MUST NOT` be treated as canonical unless published through accepted manifest + state, +- `head_epoch` `MUST` increase monotonically when `preferred_head` changes. + +This keeps operational head movement lightweight without collapsing governance +authority into admin-only control. + +### 11.2 Manifest Update Side Effects + +Manifest updates are not merely descriptive; some changes have required local +side effects. + +- changing `ranking_weights` invalidates query caches and ranking-derived + materializations, +- changing `filter_policy` requires recomputing the active accepted set, +- changing `accepted_profiles` requires unmounting any `SplitPackage` whose + `profile_hash` no longer matches an accepted profile, +- changing `supported_protocols` may likewise remove previously-mounted data + from the active accepted set. + +Admin-only head advancement does not authorize these semantic changes; they flow +through the manifest update path. + +## 12. Admission, Revocation, Filtering, and Profile Migration + +- Communities `MUST` define an invite or access policy. +- Communities `MAY` apply additional admission checks using validator receipts, trust scores, or local moderation rules. +- Filter policy applies categorically to all contributions at read time; revocation applies to specific targeted artifacts or fragments. +- Revoked artifacts and fragments `MUST NOT` appear in the active accepted set after the relevant `RevocationRecord` becomes reachable from the preferred head. +- Communities `SHOULD` use `accepted_profiles` to support controlled profile migration. +- During migration, communities `MAY` accept contributions from more than one profile, but `SplitPackage` mounting still requires exact binary compatibility with one accepted profile. + +## 13. Trust and Ranking + +Trust is community-local and implementation-defined. VGCP assumes these inputs: + +- invite lineage +- endorsements and flags +- identity age +- attestation density (independent corroboration of the contributor's Entries and Edges) +- utilization in local search +- novelty relative to existing community content + +Implementations `MAY` compute trust using EigenTrust-like or other graph-based algorithms. + +### 13.1 Structural Manipulation + +Because Edges are structural rather than behavioral, most manipulation vectors collapse into verifiability problems: + +- **edge forgery**: a forged `Link` between real Entries A and B is detectable — any contributor fetching and canonicalizing A independently derives its real link set. Forged edges fail to accumulate corroborating attestations. +- **contribution stuffing**: padding contributions with junk Entries to inflate attestation counts is bounded by `max_entries_per_contribution` in `FilterPolicy`. +- **coordinated attestation**: colluding contributors can still mutually corroborate forged edges by all serving falsified content. Mitigation is social (trust graph structure) and reputational (low-trust clusters discount their own corroboration). + +Recommended enforcement: + +- trust gates `SplitPackage` mounting and optional contribution admission +- high `edge_support` in ranking requires attestation from ≥ N distinct trusted contributors +- raw counts are never used; aggregation is trust-weighted throughout + +### 13.2 Query-Time Ranking + +Query-time ranking combines textual score with UDC relevance (bonused when explicit), trust, freshness, novelty, edge-support, and structural centrality. + +## 14. Search Semantics + +Search execution `MUST` be local over the consumer's active accepted set. + +1. Scope the query to one or more communities. +2. Gather contributions in the active accepted set. +3. Query across the local search engine's readers; compute Entry and Edge aggregations. +4. Deduplicate exact Entry-identity matches by `content_hash`. +5. Optionally cluster exact cross-URL matches by `content_only_hash`. +6. Cluster near-duplicates via MinHash. +7. Rank with community-defined weights. +8. Return results with provenance. + +Composite score: + +```text +score = w_bm25 * bm25 + + w_udc * udc_match + + w_udc_explicit_bonus * (udc_source == Explicit ? 1 : 0) + + w_trust * contributor_trust + + w_fresh * freshness_decay(observed_at) + + w_novelty * novelty_vs_community + + w_edge_support * edge_attestation_weight + + w_centrality * graph_centrality +``` + +### 14.1 Graph-Aware Queries + +Implementations `MAY` expose graph-shaped queries beyond keyword search: + +- "Entries reachable from X within N Edges of a given kind" +- "Entries with high betweenness centrality under UDC scope S" +- "Edges with attestation count ≥ K" +- "Entries with identical content across multiple URLs" +- "Cross-protocol references: Entries in one protocol referencing Entries in another" + +Standardization deferred. + +## 15. Privacy Baseline + +- Raw capture data `SHOULD` remain local by default. +- Shared artifacts `SHOULD` prefer derived search and graph data over raw payloads. +- The projection rule (Section 5.1) is the structural privacy boundary. + +### 15.1 Structural Privacy Considerations + +Graph-shaped contributions carry more information than isolated observations: + +- subgraph shape can be fingerprinting; sufficiently unique structure may be attributable even under pseudonymous identities. +- including optional behavioral traversal metadata compounds this. + +Contributors `SHOULD` be able to choose contribution richness: orphans only, structural-only (no traversal metadata), or structural-plus-behavioral. Contributors preferring minimum disclosure can contribute zero-edge contributions and still participate fully. + +Query privacy is deferred. + +## 16. Protocol Support + +VGCP is protocol-neutral at the core. `EntryRecord.protocol` and `EdgeRecord.source_protocol` are canonical lowercase strings corresponding to URI schemes. A community's `CommunityManifest.supported_protocols` declares which it admits; contributions using other protocols are filtered at aggregation. + +### 16.1 Expected Protocol Families + +VGCP anticipates support for, at minimum: + +- **HTTP / HTTPS**: the high-ceremony case. Rich `Link` metadata (region, rel), extensive `Include` semantics, HTTP-status `Redirect`s, HTML-specific `Reference`s (canonical, alternate, prev/next, hreflang). Most complex canonicalization. +- **Gemini**: line-based gemtext. Links are `=> URL [label]` on their own lines. No inline linkage; no transclusion; no `Reference`s in the HTML sense. `Link` edges with a `label` extension are sufficient for most contributions. Simple, clean canonicalization. +- **Gopher / Gopher+**: menu-structured with typed items. Menu entries project to `Link` edges with an `item_type` extension. Gopher+ adds metadata views; canonicalization profile chooses which views to include. +- **Scroll**: notable for native UDC incorporation. Canonicalization `SHOULD` extract UDC tags directly from document metadata, setting `UdcSource::Explicit`. `Link` semantics resemble Gemini's; communities standardizing on Scroll benefit from high-quality classification without classifier inference. +- **Spartan**: closely related to Gemini; similar canonicalization shape. +- **Nex**, **Text**, **SuperText**: minimalist text-first protocols. `Link` extraction depends on protocol specifics; generally a reduced `Link`-only edge model. +- **Mercury**, **Scorpion**, **Guppy**, **Molerat**, **Terse**: smolweb protocols with varying linkage models. Each requires its own canonicalization profile entries; the `EdgeKind` model should cover them without new kinds. + +### 16.2 Protocols with Sparse Graph Structure + +Some protocols fit the resource-reference-graph model, but naturally produce +sparser graphs. That is not a protocol defect; it is often exactly what their +native use case implies. + +- **Finger**: a query-response protocol for user information. Finger URLs identify people, not documents; linkage between Finger resources is not a standard feature. Finger Entries are therefore usually orphans, but that is still useful for people-indexing communities. Sparse graphs are fine when the community goal is directory-style person discovery rather than dense web mapping. +- **FSP**: a file distribution protocol. Files can be Entries, and directory structure yields a small but meaningful set of `Link`-like containment relationships between directories and files. Sparse graph structure is still useful for file-archive communities. + +Communities `MAY` admit these protocols for completeness even when the resulting graph is sparse. + +### 16.3 Cross-Protocol References + +An edge `MAY` cross protocol families — a Scroll page linking to a Gemini capsule, an HTML page linking to a Gopher menu, and so on. The `source_protocol` of the Edge is the protocol of the source Entry; the target's protocol is implicit in the target's URL. Cross-protocol edges are structurally the same as within-protocol edges and participate identically in aggregation and ranking. + +## 17. Reference Profiles + +### 17.1 Profile A: Iroh-First Trusted Exchange + +- blob transfer: `iroh-blobs` +- community announcements: `iroh-gossip` +- replicated community state: `iroh-docs` +- connectivity and relays: `iroh` + +### 17.2 Profile B: Broader Discovery Overlay + +Public discovery overlays such as libp2p-based routing may be added later. Not required for v0.1 conformance. + +## 18. Graphshell Implementation Mapping + +- [../../../mods/native/verse/mod.rs](../../../mods/native/verse/mod.rs): Ed25519-backed iroh identity, trusted-peer storage, workspace-grant concepts usable for community identity and admission. +- [../../../model/archive.rs](../../../model/archive.rs): signed portable archive objects and privacy classes; useful precedent for signed envelopes. +- [../../../app/clip_capture.rs](../../../app/clip_capture.rs): structured capture data; a precursor to canonicalization. +- [../../../services/query/mod.rs](../../../services/query/mod.rs), [../../../services/facts/mod.rs](../../../services/facts/mod.rs): local structured querying over projected history facts; no distributed contribution support yet. + +Gaps before Graphshell can claim a VGCP implementation: + +- Entry/Visit/Owner substrate implementation (history-core port from atlas-engineer/history-tree) +- protocol-aware canonicalization profile machinery, starting with HTTPS and Gemini +- BLAKE3 and MinHash fingerprinting over canonicalized Entries +- structural edge extraction per supported protocol +- GraphContribution assembly with privacy-class filtering +- optional traversal-metadata enrichment from local Visits +- attestation-aware aggregation with FilterPolicy and fragmentary-revocation application +- Tantivy or equivalent split-package production and mounting +- community manifest replication beyond bilateral sync +- commit and revocation application logic +- trust-graph computation for admission and mount-time gating + +## 19. Rollout Order + +1. Entry/Visit/Owner substrate in Graphshell (precondition; tracked separately). +2. GraphContribution serialization, canonical CBOR, per-contribution signing. +3. Canonicalization profiles for HTTPS; BLAKE3 and MinHash generation. +4. Structural edge extraction for HTTPS (Link with region/rel, Include, Redirect, Reference). +5. Two-peer contribution exchange using a reference transport. (Stress-test the wire format early.) +6. Local-only community manifest, preferred-head handling, FilterPolicy application. +7. Local graph-aware search across active accepted set. +8. Fragmentary revocation support. +9. Second protocol: Gemini canonicalization and edge extraction. Validates multi-protocol design. +10. SplitPackage production and mounting. +11. Attestation-aware aggregation and ranking. +12. Community trust and optional validator receipts. +13. Scroll canonicalization with explicit-UDC extraction. +14. Additional smolweb protocols as needed. +15. Broader discovery overlays and advanced privacy features. + +## 20. Deferred Work + +- Decentralized head selection (replacing admin-key-only head advancement as the sole operational mechanism). +- Batch-signing extensions for large contributions. +- Private community discovery with existence-hiding properties. +- Advanced query privacy (oblivious search, client-side filtering). +- Mandatory public discovery overlays. +- Cross-community search federation. +- Raw snapshot redistribution by default. +- Standardized graph-shaped query extension. +- **HTML Link context granularity**: whether coarse `region` is adequate or DOM-path / CSS-selector context should be preserved in `profile_extensions`. Accommodated by the current extension structure; revisit in a future profile revision. +- **Smolweb protocol profiles**: community-maintained canonicalization profiles for Gemini, Gopher, Scroll, Spartan, Nex, Mercury, Scorpion, Text, Guppy, Molerat, Terse, FSP, SuperText, and others. Protocol support is ordered by community demand. +- **Storage economics**: time-bank model, tokenized storage receipts, contextual credit for hosting, threshold-gated issuance tied to round-trip network work. GraphContribution is the intended substrate for staking and governance; economics spec is separate. +- **Governance**: staking-based privilege assignment within Verses, Verse-level budget allocation of staked storage, hosted social primitives (Nostr NIPs, Matrix rooms, etc.) as community-chosen layers atop VGCP. + +## 21. References + +- 2026-04-17_verse_distributed_index_protocol_v0.1.md (superseded) +- 2026-04-16_verse_index_protocol_drafts.md +- VERSE_AS_NETWORK.md +- 2026-02-23_verse_tier2_architecture.md +- atlas-engineer/history-tree (upstream library for Entry/Visit/Owner substrate) +- 2026-04-17_graph_memory_architecture_note.md (Graphshell-specific memory architecture note) From 2af1e0e3af6b1eda4cf2d93af5d596289a188beb Mon Sep 17 00:00:00 2001 From: Markik <54276851+mark-ik@users.noreply.github.com> Date: Tue, 18 Aug 2026 02:04:23 -0400 Subject: [PATCH 3/5] The iMac receipt is done, not blocked The plan recorded the first attempt's apparent event-loop hang as a genet-side defect. It was not one: the run was waiting on a macOS permission prompt on the machine's own screen, invisible to the driver. Accepting it and re-running the identical command passed, with 3 frames, 0 blank, 3 distinct digests. Kept as a lane property rather than an anecdote, since it will recur: a headed run on macOS can block on a prompt the driver cannot see, so a first run there wants someone at the screen and a hang is a prompt until ruled out. --- ...8-10_receipt_artifacts_replication_plan.md | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/design_docs/mere_docs/implementation_strategy/2026-08-10_receipt_artifacts_replication_plan.md b/design_docs/mere_docs/implementation_strategy/2026-08-10_receipt_artifacts_replication_plan.md index 50d6ef6bb..77c0bf6ae 100644 --- a/design_docs/mere_docs/implementation_strategy/2026-08-10_receipt_artifacts_replication_plan.md +++ b/design_docs/mere_docs/implementation_strategy/2026-08-10_receipt_artifacts_replication_plan.md @@ -506,12 +506,21 @@ Wayland screen, its PNG came home, and the local pane now shows was read back out through the first-party door from the replicating store, so it is evidence rather than a claim copied from the manifest. -**Intel iMac — blocked, not on this lane.** Everything up to the app works: -preflight, attach, PATH, and the build all succeed, and the smoke example -runs on the iMac's own screen with Metal drawing. The scenario then never -reaches Done — the process spins in the winit event loop indefinitely. It is -not the paint-count trap; the example already pumps a redraw per step. A -genet-side investigation, tracked there rather than here. +**Intel iMac — done.** `cambium-genet-winit-host`'s `smoke.scn` ran on the +iMac's own Aqua session and its receipt came home and was authored: +`RESULT ok`, `frames: 3 captured, 0 blank, 3 distinct digests, 2 distinct +sizes`. The example's own guard is what makes that worth something — it +refuses to report ok unless a frame had real pixels *and* the frames around +a state change differ. + +The first attempt appeared to hang in the winit event loop and was recorded +here as a genet-side defect. It was not one: the run was waiting on a macOS +permission prompt on the machine's own screen, which nothing on this side +could see. Accepting it and re-running the identical command passed. Worth +keeping as a lane property rather than an anecdote — **a headed run on a +macOS machine can block on a prompt that is invisible to the driver**, so a +first run there wants someone at the screen, and a "hang" is a prompt until +ruled out. **Four defects in `remote-receipt.ps1`, each found by running it rather than reading it** (its first real cross-machine use): From f6fec5966d4e38e0786c5350189d54e86207347f Mon Sep 17 00:00:00 2001 From: Markik <54276851+mark-ik@users.noreply.github.com> Date: Tue, 18 Aug 2026 02:07:18 -0400 Subject: [PATCH 4/5] Finish recovering the orphaned Verse research from the archived graphshell The 2026-07-23 repo consolidation archived graphshell, and that repo's whole design_docs tree is deleted at HEAD, so its 633 donor docs survive only in git history. Eight of them were federation, distributed-index, and storage-economy research that no mere doc carried: the 2026-05-27 full harvest summarizes VGCP in a single table row, and the other seven were never indexed at all. Five landed in the previous commit's sweep. This finishes the set. New here: - 2026-02-27_freenet_takeaways_for_verse (04d68365): split shared-state logic from private-identity logic, keep capability surfaces narrow per provider. - 2026-03-28_libp2p_nostr_synergy_for_verse (f36fc49b): names the control plane / data plane split, where Nostr carries small signed events that reference CIDs and libp2p delivers the bytes behind them. Carries the donor's repo-wide MPL-2.0 notice verbatim, which is worth a look against this repo's MIT/Apache posture. - 2026-02-04_donor_docs_search_findings_summary (04d68365, donor filename SEARCH_FINDINGS_SUMMARY.md): a ten-topic survey of 27 donor docs, kept for its decision record rather than its mostly dead links. Dated from its own date line. Repaired: 2026-02-23_storage_economy_and_indices was committed as an empty file; it now has its content. DOC_README gains a Recovered donor research block covering all eight, and its inheritance section now records that a local archived clone sits at Code/archive/graphshell with the rest still reachable through history, fullest tree at 401e2fcc. Every recovered file carries a provenance header naming its donor path and source commit, and saying plainly that none of it was ever implemented. It is filed as research, not as direction, and it speaks the retired Verse vocabulary. Deliberately left out of this commit: two technical_architecture docs carrying another agent's in-flight subgraph-to-nested-graph terminology edits. --- design_docs/DOC_README.md | 21 + ...2-04_donor_docs_search_findings_summary.md | 640 ++++++++++++++++++ .../2026-02-23_storage_economy_and_indices.md | 110 +++ .../2026-02-27_freenet_takeaways_for_verse.md | 117 ++++ ...26-03-28_libp2p_nostr_synergy_for_verse.md | 286 ++++++++ ...7_verse_distributed_index_protocol_v0_1.md | 2 +- 6 files changed, 1175 insertions(+), 1 deletion(-) create mode 100644 design_docs/mere_docs/research/2026-02-04_donor_docs_search_findings_summary.md create mode 100644 design_docs/mere_docs/research/2026-02-27_freenet_takeaways_for_verse.md create mode 100644 design_docs/mere_docs/research/2026-03-28_libp2p_nostr_synergy_for_verse.md diff --git a/design_docs/DOC_README.md b/design_docs/DOC_README.md index 323001e24..23b3f4066 100644 --- a/design_docs/DOC_README.md +++ b/design_docs/DOC_README.md @@ -289,6 +289,25 @@ names as receipts). - [borrowed_ideas_brief](mere_docs/research/2026-06-25_borrowed_ideas_brief.md) — **research / idea harvest**: ideas worth borrowing from adjacent projects (spatial canvases, local-first p2p, agentic tools), filtered through Mere's spatial+p2p+agentic nature and Mark's curation of a longer brainstorm; the architecture-level companion to the carve grammar harvest in the djot-editor plan. Per axis plus crossings, each with its source and a net-new / already-scoped / dependency status: **plex re-centering** (TheBrain; resolved 2026-06-25 as one escalating "Center": camera-center default, then hold-to-gather for the soft radial relayout, hard relayout deferred; rides a small selectable-collapsing-menu-row upgrade), **Cambria schema lenses** (engram schema-drift over federation; home = alembic plan), **capability-scoped subgraph sharing** (p2panda-dependent; rides the persona / federation plans), **MCP-native graph**, **speculative branches + provenance** (a pull-request-for-your-graph paired with assert-on-every-agent-mutation), **live query regions** (Tinderbox made spatial; drawn connections filtered by the active edge config, ties to graph_signals_layer), **multiplayer presence** (already scoped, see operator_presence_overlay), and **the living document** (live Potluck blocks / Peritext CRDT / Burn-wgpu semantic neighbors). Cheap-first: the `=query` polyglot block, provenance-on-agent-actions, the rung-1 camera center. - [shared_engram_commons_brief](mere_docs/research/2026-07-24_shared_engram_commons_brief.md) — **direction note (2026-07-24; both decisions answered 2026-07-27)**: the communal graph is a profile over the existing substrate, not an engine. Deterministic multi-writer convergence, fold-time authority, group-key rotation, epoch retention, message mutation, facet edits, Knot-owned text merge, encrypted chat, and carrier-byte identity now have executable receipts. Outrider owns the LXMF boundary codec; calls have a separate product plan. Direct-PHY RF passed on real T114 and Heltec V4 hardware. +### Recovered donor research (2026-08-16) + +Eight donor docs recovered from the git history of the archived `graphshell` +repo (`Code/archive/graphshell`, whose `design_docs/` tree is deleted at HEAD), +where the 2026-07-23 repo consolidation had orphaned them. Each carries a +provenance header naming its original donor path and source commit. **None of +this was ever implemented.** It is research, not direction, and it speaks the +retired Verse vocabulary. The companion three smolweb browser docs went to +Turnstone's `design_docs/` instead. + +- [verse_graph_contribution_protocol_v0_1](mere_docs/research/2026-04-17_verse_graph_contribution_protocol_v0_1.md) — **VGCP v0.1** (donor `verse_docs/technical_architecture/`, commit `6ab4c22f`): the protocol authority that replaced VDIP. Entry/Visit/Owner projection boundary; structural verifiable-by-fetch edges rather than behavioral ones; per-protocol canonicalization profiles; BLAKE3 + CIDv1; privacy-filter-before-sign ordering; Ed25519 to did:key identity with Genesis/Threshold/Delegated rule systems and revocation as a read-time projection. This is the full text behind the [full docs harvest](mere_docs/research/2026-05-27_graphshell_docs_full_harvest.md) §7 row that summarized it. +- [verse_distributed_index_protocol_v0_1](mere_docs/research/2026-04-17_verse_distributed_index_protocol_v0_1.md) — **VDIP v0.1** (donor `archive_docs/checkpoint_2026-04-17/`, commit `6ab4c22f`): VGCP's predecessor, archived on the same day it was superseded. Signed immutable content-addressed graphlets, community admission and revocation semantics, search and ranking over accepted artifacts, with transport kept out of the normative layer. Filename normalized from `..._v0.1.md`. +- [modern_yacy_gap_analysis](mere_docs/research/2026-02-23_modern_yacy_gap_analysis.md) — (donor `verse_docs/research/`, commit `1208e352`): why a YaCy-style global word DHT was rejected, and **Federated Index Exchange** proposed instead: portable mergeable Tantivy segments published as content-addressed blobs, plus the query-protocol and ranking gaps that follow from it. +- [libp2p_nostr_synergy_for_verse](mere_docs/research/2026-03-28_libp2p_nostr_synergy_for_verse.md) — (donor `verse_docs/research/`, commit `f36fc49b`): names the **control plane / data plane** split (Nostr carries small signed events that reference CIDs; libp2p delivers the bytes behind those CIDs) and works through its consequences and seams. Carries the donor's repo-wide MPL-2.0 notice verbatim, unreviewed against this repo's MIT/Apache posture. +- [storage_economy_and_indices](mere_docs/research/2026-02-23_storage_economy_and_indices.md) — (donor `verse_docs/research/`, commit `1208e352`): proof-of-access as **active service rather than passive storage**: sharding, signed access receipts, minting, and provenance metadata carried on an otherwise fungible token. Precursor to the donor's `proof_of_access_ledger_spec.md`. +- [freenet_takeaways_for_verse](mere_docs/research/2026-02-27_freenet_takeaways_for_verse.md) — (donor `verse_docs/research/`, commit `04d68365`): a short external pattern review of Freenet. The durable takeaways are splitting shared-state logic from private-identity logic, and keeping capability surfaces narrow per mod or provider. +- [aspirational_protocols_and_tools](mere_docs/research/2026-02-22_aspirational_protocols_and_tools.md) — (donor `verse_docs/research/`, commit `9a6526a7`): the widest early survey. IPFS/GunDB/libp2p/iroh, Tor/I2P/DoH, syndication, Wasm mods, vector search, local inference, CRDTs, WARC, plus a protocol-handler-trait and opt-in registry sketch. Begins at its own section 2; no section 1 was ever committed. +- [donor_docs_search_findings_summary](mere_docs/research/2026-02-04_donor_docs_search_findings_summary.md) — (donor `verse_docs/research/SEARCH_FINDINGS_SUMMARY.md`, commit `04d68365`): a ten-topic survey of 27 donor docs written 2026-02-04, worth keeping for its **decision record**: CRDT/OT not planned for MVP, DOM serialization explicitly avoided, sanitization plus Servo sandboxing for untrusted node data, YaCy-style search deferred to phase 3+, ghost nodes low priority. Its internal links are mostly dead. Dated here from its own date line; the donor filename was undated. + ## mere_docs/design/ - [commons_profile_v1](mere_docs/design/2026-07-27_commons_profile_v1.md) — **executable profile 2026-07-27**: the first communal graph, Knot-document, and encrypted-chat contract. It fixes stable Personae-root authority with Servitor capability checks, pending/effective/revoked projection, explicit p2panda Data Encryption profiles, automated safe epoch retention, immutable message edit/delete facts, facet-grained graph edits, Knot-owned bounded text merge, and partial-history behavior. Outrider keeps LXMF at the boundary; Direct-PHY carrier identity passed on real T114 and Heltec V4 hardware. @@ -385,6 +404,8 @@ Full per-sub-crate snapshot (with load-bearing vs aspirational status) at The donor `graphshell` repo was **GitHub-archived (read-only) and its local clone deleted 2026-05-27**; its 633 design docs were swept into two curated indexes that are the entry points for any remaining pull: the [full docs harvest](mere_docs/research/2026-05-27_graphshell_docs_full_harvest.md) and the [concept brief](mere_docs/research/2026-05-17_graphshell_harvest_brief.md). Fetch detail from the GitHub archive when a slice needs it; the old `../../graphshell/design_docs/` local path no longer resolves. +**Recovery note, 2026-08-16.** A local archived clone of the donor also sits at `Code/archive/graphshell`. Its `design_docs/` tree is deleted at HEAD, so the docs are reachable only through git history (`git show :`); the fullest tree is commit `401e2fcc` (2026-04-29, 612 files). Eight donor docs orphaned by the 2026-07-23 repo consolidation were recovered from there into `mere_docs/research/` (the recovered-donor-research block above); three smolweb browser docs went to Turnstone's `design_docs/`. Everything else remains where the two harvest indexes left it. + Specifically, the following live in the GitHub archive (read-only), surfaced via the harvest indexes above: `TERMINOLOGY.md` (pre-Mere terms), `engram_spec.md` (the 1100+ line engram spec), `VERSO_AS_PEER.md`, `COMMS_AS_APPLETS.md`, `coop_session_spec.md`, and `cable_coop_minichat_spec.md`. ## Status diff --git a/design_docs/mere_docs/research/2026-02-04_donor_docs_search_findings_summary.md b/design_docs/mere_docs/research/2026-02-04_donor_docs_search_findings_summary.md new file mode 100644 index 000000000..6465d0d60 --- /dev/null +++ b/design_docs/mere_docs/research/2026-02-04_donor_docs_search_findings_summary.md @@ -0,0 +1,640 @@ +> **Recovered research. Never implemented. Not current direction.** +> +> Recovered 2026-08-16 from the git history of the archived `graphshell` +> repository (`Code/archive/graphshell`), whose entire `design_docs/` tree is +> deleted at HEAD, so this text survives only in history. +> +> - Original path: `design_docs/verse_docs/research/SEARCH_FINDINGS_SUMMARY.md` +> - Source commit: `04d68365` +> +> None of it was ever built. It is filed here as research, not as a plan and +> not as a statement of where Mere is going. It speaks the retired Verse +> vocabulary (per `TERMINOLOGY.md`, *Verse* is a retired term and the +> network scope is Mere at network scope), and its relative links point at +> donor docs that no longer exist locally. Read the text below for the ideas, +> not the direction. +> +> The donor filename was undated (`SEARCH_FINDINGS_SUMMARY.md`); it is dated +> here from the document's own date line, 2026-02-04. It surveys 27 donor design +> docs, so most of its internal links are dead; the durable content is the +> decision record (what was planned, what was explicitly rejected, and why). + +--- + +# Design Docs Search Findings Summary + +Comprehensive analysis of design documentation across Graphshell project. Searched: `design_docs/` and `design_docs/archive_docs/` folders. + +Note: This file now lives in `verse_docs/`. Some links still assume the `design_docs/` root; add `../` if needed. + +**Date**: February 4, 2026 +**Files Analyzed**: 27 markdown files across main and archive directories + +--- + +## 1. P2P Synchronization, Collaborative Features, Shared Graph Updates + +### Current Plan Status: **Deferred to Phase 3+** + +**Key Files:** +- [ARCHITECTURE_DECISIONS.md](../../archive_docs/checkpoint_2026-02-01/technical_architecture/ARCHITECTURE_DECISIONS.md#L510-L533) (Section 21: Future Architecture) +- [IMPLEMENTATION_ROADMAP.md](../../graphshell_docs/implementation_strategy/IMPLEMENTATION_ROADMAP.md#L518) +- [VERSE.md](VERSE.md) + +**Findings:** + +**Phase 1-2: Local-only (MVP)** +- MVP prioritizes local persistence with incremental saves (every 30 seconds) +- No P2P/sync implementation in Phase 1 +- Architecture explicitly designed for future modularity + +**Phase 3+: Planned Optional Modules** +From [ARCHITECTURE_DECISIONS.md](../../archive_docs/checkpoint_2026-02-01/technical_architecture/ARCHITECTURE_DECISIONS.md#L510): +``` +Phase 2+: Optional modules + - Local sync (file-based) + - P2P sync (YaCy-style, Syncthing-like) + - Distributed storage (IPFS, Arweave) + - Token system (only if needed for incentives) +``` + +**Trait-Based Design for Future Sync:** +```rust +pub trait SyncBackend { + fn push(&self, graph: &Graph) -> Result<()>; + fn pull(&self) -> Result; + fn merge(&self, local: &Graph, remote: &Graph) -> Result; +} + +impl SyncBackend for LocalFilesystem { } +// Later: impl SyncBackend for P2PSync { } +``` + +**Shared Graph Updates Strategy:** +- Graph changes tracked via **dirty tracking** (only changed nodes/edges written) +- Session history: Keep last 10 full snapshots +- On open: Load latest session + replay unsaved deltas +- No CRDT/OT (operational transformation) currently planned for MVP + +**Rationale (from [PROJECT_PHILOSOPHY.md](../../archive_docs/checkpoint_2026-01-29/PROJECT_PHILOSOPHY.md)):** +> "Local-first storage, with optional P2P sync later" +> "Personal/Local-First, Not Collaborative-First" + +- MVP focus: Spatial UX for single user +- Sync needed only if P2P proven useful in Phase 3 +- Personal use case (one user, one machine) is primary in MVP + +--- + +## 2. Firefox Architecture Decisions: Process Isolation, Multiprocess, Content Process Management + +### Heavy Influence on Graphshell Architecture + +**Key Files:** +- [ARCHITECTURE_DECISIONS.md](../../archive_docs/checkpoint_2026-02-01/technical_architecture/ARCHITECTURE_DECISIONS.md#L491-L509) (Section 20: Process Isolation) +- [IMPLEMENTATION_ROADMAP.md](../../graphshell_docs/implementation_strategy/IMPLEMENTATION_ROADMAP.md#L9-L100) +- [DOC_README.md](../../DOC_README.md) + +**Findings:** + +**Process Isolation Pattern (from Firefox):** +From [ARCHITECTURE_DECISIONS.md](../../archive_docs/checkpoint_2026-02-01/technical_architecture/ARCHITECTURE_DECISIONS.md#L491): +``` +Graph UI runs in compositor thread (trusted) +Webviews run in sandboxed Servo processes (untrusted) +``` + +**Launch Commands:** +```bash +cargo run --release -- -M -S +# -M: Multiprocess (Servo spawns content processes) +# -S: Sandbox (gaol, seccomp applied to content) +``` + +**Origin-Based Process Management (Servo's Model):** +From [ARCHITECTURE_DECISIONS.md](../../archive_docs/checkpoint_2026-02-01/technical_architecture/ARCHITECTURE_DECISIONS.md#L54): +> "Firefox's approach (kill unused processes) more efficient than fixed pool + serialization" +> "No serialization latency (create/destroy processes, not serialize DOM)" + +**Process Lifecycle:** +1. User creates node from origin A → Servo spawns process for A +2. User closes all nodes from origin A → Servo kills process for A +3. User creates node from origin A again → New process spawns +4. **No webview reuse, no serialization, no dormant processes** + +**Why This Matters:** +- Origin-grouped nodes map naturally to origin-grouped processes +- Firefox validates this pattern at scale (used in production) +- Servo's `-M` flag already implements this; no custom implementation needed +- Memory: Processes killed when all nodes for that origin close + +**Alternative Rejected:** +- Fixed pool (simpler, but wasteful) +- Webview pooling with serialization (adds latency) + +**Servo's Advantage Over Chrome Approach:** +- Servo's origin grouping is proven (Firefox uses it) +- No explicit DOM serialization needed +- Better payoff despite higher complexity upfront + +--- + +## 3. CRDT (Conflict-free Replicated Data Type) or Operational Transformation Approaches + +### Current Plan: **NOT PLANNED for MVP** + +**Key Files:** +- [ARCHITECTURE_DECISIONS.md](../../archive_docs/checkpoint_2026-02-01/technical_architecture/ARCHITECTURE_DECISIONS.md#L510-L533) (Modularity for future sync) +- [PROJECT_PHILOSOPHY.md](../../archive_docs/checkpoint_2026-01-29/PROJECT_PHILOSOPHY.md#L100-L180) + +**Findings:** + +**Explicit Non-Use in MVP:** +From [PROJECT_PHILOSOPHY.md](../../archive_docs/checkpoint_2026-01-29/PROJECT_PHILOSOPHY.md#L170): +> "No need for crdt/conflict resolution in MVP" +> "Personal use case (one user, one machine) is primary" + +**Why CRDTs Deferred:** +- Designed for single-user, local-first workflow +- P2P sync (Phase 3+) would need conflict resolution +- Dependency injection via `SyncBackend` trait allows CRDT implementation later +- Current focus: Reliable local persistence with version control + +**Conflict Resolution Strategy (for future P2P):** +The architecture stub shows: +```rust +pub trait SyncBackend { + fn merge(&self, local: &Graph, remote: &Graph) -> Result; +} +``` +- Merge function signature prepared but not implemented +- Implementations could use: CRDTs, OT, last-write-wins, or custom resolution +- Decision deferred to Phase 3 when collaboration needs are clear + +**Version Control Instead of CRDT:** +Current approach for MVP: +- Session snapshots: Keep last 10 full versions +- Dirty tracking: Only changed data persisted +- Replay semantics: Unsaved deltas replayed on load +- Simple, local, effective for single user + +--- + +## 4. Anytype, Obsidian, Notion, OneNote, Google Docs, Office Architectures + +### Current Plan: **NO DIRECT ARCHITECTURAL BORROWING, PARTIAL FEATURE INSPIRATION** + +**Key Files:** +- [PROJECT_PHILOSOPHY.md](../../archive_docs/checkpoint_2026-01-29/PROJECT_PHILOSOPHY.md) (Feature set comparisons) +- [COMPREHENSIVE_SYNTHESIS.md](../../archive_docs/checkpoint_2026-01-29/COMPREHENSIVE_SYNTHESIS.md) + +**Findings:** + +**NOT Mentioned in Design Docs:** +No explicit architectural analysis of these products. No CRDT/sync patterns borrowed. + +**Implied Lessons (Inferred from Graphshell Philosophy):** + +**Different Problem Space:** +- **Notion/Obsidian**: Block-based notetaking, collaborative editing +- **Graphshell**: Spatial browser, knowledge graph visualization +- **Google Docs/Office**: Real-time collaborative editing +- **Graphshell MVP**: Single-user, local-first, graph-centric + +**Feature Inspiration (from [PROJECT_PHILOSOPHY.md](../../archive_docs/checkpoint_2026-01-29/PROJECT_PHILOSOPHY.md)):** +- Session management (like browser tabs, but explicit "sessions") +- DOM inspector (similar to Obsidian web clipper) +- Export formats: JSON, PNG, interactive HTML (similar to Obsidian) +- Sidebar option (optional tabs, for users preferring traditional interface) + +**What Graphshell Explicitly Rejects:** +- Real-time collaborative editing (Phase 3+ only) +- Block-based nested structure (graph-first instead) +- Commercial sync infrastructure (P2P if implemented) + +**Recommended Future Analysis (Phase 2-3):** +- Study Obsidian's local-first sync approach +- Review how Notion handles schema migrations +- Consider OneNote's conflict-free merge strategies if P2P implemented + +--- + +## 5. YaCy-Style Decentralized Search + +### Current Plan: **Explicitly Mentioned for Phase 3+** + +**Key Files:** +- [ARCHITECTURE_DECISIONS.md](../../archive_docs/checkpoint_2026-02-01/technical_architecture/ARCHITECTURE_DECISIONS.md#L518) (P2P sync options) +- [verse_docs/VERSE.md](VERSE.md) (Phase 3 tokenization) + +**Findings:** + +**Direct Reference:** +From [ARCHITECTURE_DECISIONS.md](../../archive_docs/checkpoint_2026-02-01/technical_architecture/ARCHITECTURE_DECISIONS.md#L518): +``` +Phase 2+: Optional modules + - P2P sync (YaCy-style, Syncthing-like) +``` + +**YaCy Model Consideration:** +YaCy = peer-to-peer search engine where users share search indices. + +**Graphshell's Potential Application:** +- Phase 3+: Optional P2P search across shared graphs +- Users could seed indexed graph fragments +- Other users discover knowledge through DHT (distributed hash table) +- Decentralized alternative to centralized search + +**Not Detailed Yet:** +- No technical spec for YaCy integration +- Would be part of larger P2P sync infrastructure +- Paired with token incentives (Phase 3 research) + +**Related Concept: Verse Indexers** +From [verse_docs/VERSE.md](VERSE.md): +``` +Peer roles: +- Indexers/deduplicators: dedupe and index reports for efficient queries +``` + +This aligns with YaCy's distributed indexing model. + +--- + +## 6. DOM Serialization/Deserialization Approaches + +### Current Plan: **EXPLICITLY AVOIDED** + +**Key Files:** +- [ARCHITECTURE_DECISIONS.md](../../archive_docs/checkpoint_2026-02-01/technical_architecture/ARCHITECTURE_DECISIONS.md#L55, #L221) +- [IMPLEMENTATION_ROADMAP.md](../../graphshell_docs/implementation_strategy/IMPLEMENTATION_ROADMAP.md#L54-L67) + +**Findings:** + +**Explicit Non-Implementation:** +From [ARCHITECTURE_DECISIONS.md](../../archive_docs/checkpoint_2026-02-01/technical_architecture/ARCHITECTURE_DECISIONS.md#L221): +> "No explicit DOM serialization (too complex, Servo handles it)" + +From [ARCHITECTURE_DECISIONS.md](../../archive_docs/checkpoint_2026-02-01/technical_architecture/ARCHITECTURE_DECISIONS.md#L55): +> "No serialization latency (create/destroy processes, not serialize DOM)" + +**Why Serialization Rejected:** +1. **Complexity**: DOM state is complex to serialize/deserialize reliably +2. **Servo Responsibility**: Servo manages DOM; no need to duplicate +3. **Process Model**: Kill/spawn processes instead of serializing state +4. **Latency**: Serialization causes ~150ms UI lag spikes (unacceptable) + +**What IS Serialized:** +Only graph metadata serialized to JSON: +```json +{ + "nodes": [ + { + "id": "node123", + "url": "https://example.com", + "title": "Page Title", + "favicon": "...", + "tags": ["research", "example"], + "created_at": "2025-02-04T...", + "metadata": { ... } + } + ], + "edges": [...] +} +``` + +**Webview State Handling:** +- Each origin's webview is a separate Servo process +- Process created on demand, destroyed when not needed +- No attempt to preserve/restore DOM state +- User refreshes page if needed (fast, within 1-2 seconds) + +**Alternative Considered and Rejected:** +- Fixed pool with dormant processes (serialization overhead) +- Process pooling with state serialization (latency spike) + +--- + +## 7. Untrusted Data Handling in Graph Nodes + +### Current Plan: **SANITIZATION + SERVO SANDBOXING** + +**Key Files:** +- [ARCHITECTURE_DECISIONS.md](../../archive_docs/checkpoint_2026-02-01/technical_architecture/ARCHITECTURE_DECISIONS.md#L461-L478) (Section 19) +- [IMPLEMENTATION_ROADMAP.md](../../graphshell_docs/implementation_strategy/IMPLEMENTATION_ROADMAP.md#L868) + +**Findings:** + +**Decision from [ARCHITECTURE_DECISIONS.md](../../archive_docs/checkpoint_2026-02-01/technical_architecture/ARCHITECTURE_DECISIONS.md#L461):** +> "Sanitize user-visible data. Validate URLs. Trust Servo for webview sandboxing." + +**Implementation Strategy:** + +**Input Sanitization (Compositor):** +```rust +fn sanitize_label(input: &str) -> String { + input.chars() + .filter(|c| c.is_alphanumeric() || c.is_whitespace() || "-_.".contains(*c)) + .collect() +} + +fn validate_url(url: &str) -> Result { + let parsed = Url::parse(url)?; + match parsed.scheme() { + "http" | "https" | "file" => Ok(parsed), + _ => Err(InvalidScheme), + } +} +``` + +**Threats Addressed:** + +1. **Node Labels from Untrusted Sources:** + - Page title from `` tag: Sanitize before display + - Open Graph metadata: Sanitize before display + - User-entered tags: Validate input + +2. **URLs:** + - Only http, https, file schemes allowed + - Other schemes (javascript:, data:) rejected + - URL parsing via robust library (Url crate) + +3. **Webview Content:** + - Runs in sandboxed Servo process (gaol, seccomp) + - Process isolation prevents escape to compositor + - Can't directly manipulate graph nodes + - Sandboxing handled by Servo, not reimplemented + +**Attack Surface:** +- Compositor displays user-visible data (sanitized) +- Webview processes can't access compositor memory +- Graph data protected by process boundary + +**No Direct DOM Inspection:** +- Don't manually parse/serialize DOM +- Use Servo's safe interfaces only +- Avoid raw HTML parsing + +--- + +## 8. Export/Import Formats and Interoperability + +### Current Plan: **PHASE 2 FEATURE, BASIC SUPPORT** + +**Key Files:** +- [IMPLEMENTATION_ROADMAP.md](../../graphshell_docs/implementation_strategy/IMPLEMENTATION_ROADMAP.md#L348, #L539, #L606) +- [ARCHITECTURE_DECISIONS.md](../../archive_docs/checkpoint_2026-02-01/technical_architecture/ARCHITECTURE_DECISIONS.md) + +**Findings:** + +**Planned Export Formats:** +From [IMPLEMENTATION_ROADMAP.md](../../graphshell_docs/implementation_strategy/IMPLEMENTATION_ROADMAP.md#L606): +``` +- [ ] Export options (PNG, SVG, JSON) +``` + +**Phase 2 Milestones:** + +| Format | Phase | Status | Purpose | +|--------|-------|--------|---------| +| **JSON** | 2 | Planned | Graph persistence, interoperability | +| **PNG** | 2 | Planned | Static visualization, sharing | +| **SVG** | 2 | Planned | Vector export, scalable | +| **Interactive HTML** | 3+ | Research | Standalone graphs with embedded webviews | + +**JSON Format Details:** +From [IMPLEMENTATION_ROADMAP.md](../../graphshell_docs/implementation_strategy/IMPLEMENTATION_ROADMAP.md#L738): +``` +Serialization: serde_json +Standard, readable +``` + +**Save Performance Target:** +From [IMPLEMENTATION_ROADMAP.md](../../graphshell_docs/implementation_strategy/IMPLEMENTATION_ROADMAP.md#L630): +``` +Serialization: 10K graph < 500ms +``` + +**Import Capabilities:** + +From [IMPLEMENTATION_ROADMAP.md](../../graphshell_docs/implementation_strategy/IMPLEMENTATION_ROADMAP.md#L496): +``` +- [ ] Import from Chrome/Firefox bookmarks.html +``` + +**Future Interoperability (Phase 3+):** +- Node-level sharing: `graphshell://node?id=abc123&title=...&url=...&tags=...` +- Standalone JSON cards for individual nodes +- Interactive HTML export (complex; deferred) + +**Alternative Approaches (Deferred):** +- OPML export (for outliner interop) +- RDF/semantic web formats +- Markdown graph notation + +--- + +## 9. Session/Browsing History Storage Concepts + +### Current Plan: **PHASE 2 FEATURE, PARTIALLY SPECIFIED** + +**Key Files:** +- [ARCHITECTURE_DECISIONS.md](../../archive_docs/checkpoint_2026-02-01/technical_architecture/ARCHITECTURE_DECISIONS.md#L158-L180) (Section 6: Persistence) +- [IMPLEMENTATION_ROADMAP.md](../../graphshell_docs/implementation_strategy/IMPLEMENTATION_ROADMAP.md#L291, #L529-L550) +- [PROJECT_PHILOSOPHY.md](../../archive_docs/checkpoint_2026-01-29/PROJECT_PHILOSOPHY.md#L180-L230) + +**Findings:** + +**Session Persistence Strategy:** +From [ARCHITECTURE_DECISIONS.md](../../archive_docs/checkpoint_2026-02-01/technical_architecture/ARCHITECTURE_DECISIONS.md#L160): +> "Incremental saves with version control and session history" + +**Storage Structure:** +``` +~/.config/graphshell-graph/ +├── sessions/ +│ ├── current.json # Latest full session +│ ├── session_1707000000.json +│ ├── session_1707000900.json +│ └── ... (up to 10 versions) +├── preferences.toml +├── keybinds.toml +└── theme.toml +``` + +**Auto-Save Mechanism:** +From [ARCHITECTURE_DECISIONS.md](../../archive_docs/checkpoint_2026-02-01/technical_architecture/ARCHITECTURE_DECISIONS.md#L165): +``` +- Auto-save every 30 seconds (configurable) +- Dirty tracking: Only changed nodes/edges written +- Session history: Keep last 10 full snapshots +- On close: Write complete session +- On open: Load latest session + replay unsaved deltas +``` + +**Browsing History Types (from [IMPLEMENTATION_ROADMAP.md](../../graphshell_docs/implementation_strategy/IMPLEMENTATION_ROADMAP.md#L277)):** +``` +Edge types: Hyperlink (blue), Bookmark (green), History (gray), Manual (red) +``` + +**Phase 2 Session Features:** +From [IMPLEMENTATION_ROADMAP.md](../../graphshell_docs/implementation_strategy/IMPLEMENTATION_ROADMAP.md#L545-L548): +``` +- [ ] Session history: + - Track visited nodes (like browser history) + - Search history + - Clear history +``` + +**Phase 3+ Concepts (from [PROJECT_PHILOSOPHY.md](../../archive_docs/checkpoint_2026-01-29/PROJECT_PHILOSOPHY.md#L220)):** +``` +Option A: Each session is a separate graph file +Option B: One graph with timestamps; can rewind/replay +Option C: Like browser history; can collapse old branches +``` + +**Not Yet Specified:** +- Session naming/tagging +- Cross-session search +- Session comparison (diff two sessions) +- History visualization timeline + +--- + +## 10. Ghost Nodes Concept + +### Current Plan: **PHASE 2+ OPTIONAL FEATURE, LOW PRIORITY** + +**Key Files:** +- [COMPREHENSIVE_SYNTHESIS.md](../../archive_docs/checkpoint_2026-01-29/COMPREHENSIVE_SYNTHESIS.md#L207-L226) +- [PROJECT_PHILOSOPHY.md](../../archive_docs/checkpoint_2026-01-29/PROJECT_PHILOSOPHY.md#L100-L130, #L275-L290) + +**Findings:** + +**Definition:** +From [PROJECT_PHILOSOPHY.md](../../archive_docs/checkpoint_2026-01-29/PROJECT_PHILOSOPHY.md#L100): +> "Use ghost nodes to preserve structure when removing items" + +**Concept Explanation:** +From [COMPREHENSIVE_SYNTHESIS.md](../../archive_docs/checkpoint_2026-01-29/COMPREHENSIVE_SYNTHESIS.md#L210): +> "When you delete a node, keep the edges visible (as 'ghost edges'), but dim/style them differently" + +**Use Case:** +From [PROJECT_PHILOSOPHY.md](../../archive_docs/checkpoint_2026-01-29/PROJECT_PHILOSOPHY.md#L285): +> "Knowledge organization; you remove a page but want to remember it was related to others" + +**Implementation Recommendation (Phase 2):** +From [COMPREHENSIVE_SYNTHESIS.md](../../archive_docs/checkpoint_2026-01-29/COMPREHENSIVE_SYNTHESIS.md#L220): +``` +- Optional feature: "Show ghost connections" toggle +- When node deleted, create "GhostEdge" (visual only, no target) +- Render as dashed/faded line +- Can be turned off in settings +``` + +**Data Structure Addition:** +```rust +pub enum Edge { + Normal { from: NodeKey, to: NodeKey, ty: EdgeType }, + Ghost { from: NodeKey, to: TombstonedNodeId, ty: EdgeType }, +} +``` + +**Visual Representation:** +- Dashed lines (vs solid for normal edges) +- Reduced opacity/gray color +- Optional: Toggle in settings to hide/show + +**Status in MVP:** +- **NOT in Phase 1** (MVP) +- **Proposed for Phase 2** (low priority) +- **Rationale**: Nice-to-have; doesn't block core functionality +- **Complexity**: Adds ~100 lines of code; straightforward + +**Related Features (Deferred):** +- Tombstoning (mark nodes as deleted but keep metadata) +- Undo/redo for deletions +- Ghost node visualization in history timeline + +--- + +## Summary Table: All Topics + +| Topic | Phase | Status | Key File | Quote/Reference | +|-------|-------|--------|----------|-----------------| +| **P2P Sync** | 3+ | Deferred | ARCHITECTURE_DECISIONS.md | "Design for modularity, but don't implement P2P/sync in MVP" | +| **Collaborative Editing** | 3+ | Deferred | PROJECT_PHILOSOPHY.md | "Real-time sync deferred" | +| **CRDT/OT** | 3+ | Not Planned | PROJECT_PHILOSOPHY.md | "No need for crdt/conflict resolution in MVP" | +| **Firefox Patterns** | 1 | Implemented | ARCHITECTURE_DECISIONS.md | "Leverage Servo's origin-grouped multiprocess" | +| **Process Isolation** | 1 | Core Design | ARCHITECTURE_DECISIONS.md | "Compositor separate from content processes" | +| **YaCy Search** | 3+ | Proposed | ARCHITECTURE_DECISIONS.md | "P2P sync (YaCy-style, Syncthing-like)" | +| **DOM Serialization** | N/A | Rejected | ARCHITECTURE_DECISIONS.md | "No explicit DOM serialization (too complex)" | +| **Untrusted Data** | 1 | Core Design | ARCHITECTURE_DECISIONS.md | "Sanitize labels, validate URLs, trust Servo sandboxing" | +| **Export/Import** | 2 | Planned | IMPLEMENTATION_ROADMAP.md | "Export as PNG/JSON; import from browser bookmarks" | +| **Sessions/History** | 1-2 | Partial | ARCHITECTURE_DECISIONS.md | "Keep last 10 full snapshots, incremental saves" | +| **Ghost Nodes** | 2+ | Optional | COMPREHENSIVE_SYNTHESIS.md | "Use ghost edges to preserve structure when deleting" | + +--- + +## Architecture Philosophy Summary + +**Core Principles (from PROJECT_PHILOSOPHY.md):** +1. **Learning-first**: Ship early, iterate based on use +2. **Local-first**: MVP = single-user, local persistence +3. **Sense-making**: Built for research/knowledge organization +4. **Modularity**: Phase 1 sets foundation for P2P/sync later +5. **Optionality**: Multiple view modes, export formats, physics presets + +**What's **NOT** in MVP:** +- Collaborative editing / real-time sync +- CRDTs or conflict resolution +- DOM serialization / process pooling +- YaCy-style distributed search +- 3D graph visualization +- Token/economic system + +**What **IS** in MVP:** +- Force-directed graph visualization +- Origin-grouped multiprocess (Servo's `-M` flag) +- Process isolation (compositor vs content) +- Local persistence with version history (last 10 snapshots) +- Input sanitization + Servo sandboxing +- Basic search (prefix match, upgrade to fuzzy in Week 3) +- Basic export (JSON; PNG/SVG in Phase 2) + +--- + +## Recommendations for Further Research + +1. **CRDT Implementation** (If P2P sync needed in Phase 3): + - Study Yjs, Automerge, or similar Rust implementations + - CRDTs suited for collaborative graph editing + +2. **YaCy/Decentralized Search** (If Phase 3 P2P enabled): + - Review YaCy protocol (DHT-based peer indexing) + - Design schema for graph fragment sharing + +3. **Obsidian Sync Pattern** (For local-first inspiration): + - Obsidian's vault system (local folder + optional sync) + - Useful for understanding feature parity + +4. **Notion/OneNote Architectures** (For feature inspiration): + - Block-based nesting (orthogonal to Graphshell's graph model) + - Conflict resolution strategies for future reference + +5. **Webview Pool Optimization** (Phase 2): + - Monitor if process creation/destruction adds latency + - Consider Verse's IPC-based helper process model if bottleneck found + +6. **Freenet Pattern Review** (External architecture comparison, 2026-02-27): + - Apply shared-state vs identity-secrets authority split. + - Apply capability-first provider/mod contracts. + - Enforce doc-to-test linkage to prevent spec/implementation drift. + - See: [2026-02-27_freenet_takeaways_for_verse.md](2026-02-27_freenet_takeaways_for_verse.md) + +--- + +**Document Generated**: 2025-02-04 +**Total Design Docs Reviewed**: 27 files +**Search Queries**: 10 comprehensive semantic searches + 15 targeted grep searches + diff --git a/design_docs/mere_docs/research/2026-02-23_storage_economy_and_indices.md b/design_docs/mere_docs/research/2026-02-23_storage_economy_and_indices.md index e69de29bb..3bec3faf8 100644 --- a/design_docs/mere_docs/research/2026-02-23_storage_economy_and_indices.md +++ b/design_docs/mere_docs/research/2026-02-23_storage_economy_and_indices.md @@ -0,0 +1,110 @@ +> **Recovered research. Never implemented. Not current direction.** +> +> Recovered 2026-08-16 from the git history of the archived `graphshell` +> repository (`Code/archive/graphshell`), whose entire `design_docs/` tree is +> deleted at HEAD, so this text survives only in history. +> +> - Original path: `design_docs/verse_docs/research/2026-02-23_storage_economy_and_indices.md` +> - Source commit: `1208e352` +> +> None of it was ever built. It is filed here as research, not as a plan and +> not as a statement of where Mere is going. It speaks the retired Verse +> vocabulary (per `TERMINOLOGY.md`, *Verse* is a retired term and the +> network scope is Mere at network scope), and its relative links point at +> donor docs that no longer exist locally. Read the text below for the ideas, +> not the direction. + +--- + +# Verse: Storage Economy & Composable Indices (Speculative) + +**Date**: 2026-02-23 +**Status**: Speculative Research / RFC +**Context**: Refines the economic model from `VERSE.md` based on "Proof of Access" and defines the "Index" data structure. + +--- + +## 1. The Storage Economy: Proof of Access + +The core shift is from **Passive Storage** (getting paid to hold data) to **Active Service** (getting paid to serve data). + +### 1.1 The Mechanism: Sharding & Receipts +1. **Sharding**: Content (Reports, Graphs, Indices) is encrypted and split into fixed-size shards (e.g., 256KB). +2. **Hosting**: A Peer ("The Cache") stores shards. They cannot read the content (encrypted), but they can verify integrity (hashes). +3. **Access**: A User requests a shard. +4. **The Receipt (The "Fractional Coin")**: + * The User receives the shard and verifies the hash. + * The User signs a cryptographic **Receipt**: `Sign(User_ID + Host_ID + Shard_Hash + Timestamp)`. + * This Receipt is sent to the Host. +5. **Minting**: + * The Host collects Receipts. + * Receipts are "cashed in" to the network protocol. + * **Validation**: The network checks the signatures and ensures the User had the "bandwidth credits" to request data. + * **Reward**: The Host receives **Verse Tokens** (fungible). + +### 1.2 Token Metadata & Provenance +While the Verse Token is fungible (1 VT = 1 VT), the minting process preserves **Provenance Metadata** in the ledger history. +* **Serial Number**: We can trace a batch of tokens back to the specific *service event* (serving shards X, Y, Z to users A, B, C). +* **Reputation**: Tokens minted from serving high-demand, rare indices might carry more "Reputation Weight" for governance, even if they spend the same as other tokens. + +### 1.3 The Economic Loop +1. **Earn**: Host storage -> Serve shards -> Collect Receipts -> Mint Tokens. +2. **Spend**: Use Tokens to buy **Access Keys** or **Indices**. +3. **Trade**: Exchange Tokens for **Tokenized Reports** (rare data). + +--- + +## 2. The Index: A Composable Knowledge Graph + +An "Index" in Verse is not a database table. It is a **Graphshell Graph**. + +### 2.1 Structure +An Index is a portable, content-addressed Graphshell Workspace containing: +1. **Nodes**: Content IDs (CIDs) pointing to Reports or other Graphs. +2. **Edges**: Relationships (traversals, citations, "see also"). +3. **Semantics**: UDC tags, user tags, and embeddings. + +### 2.2 Composition (The "Merge") +Because Indices are Graphs, they are **mutually composable**. +* **Scenario**: + * Index A: "Rust Async Ecosystem" (Nodes: Tokio, async-std, blogs). + * Index B: "WebAssembly Tooling" (Nodes: Yew, Leptos, bindgen). +* **Composition**: A user loads both. Graphshell merges them. + * **Result**: A new Graph containing all nodes. + * **Emergent Value**: If both indices reference `wasm-bindgen`, that node becomes a bridge, visually connecting the two clusters. + +### 2.3 Navigability +* **Graphshell**: Browses the Index as a spatial map. You "fly" through the index. +* **Verso**: Renders the content within the Index nodes. +* **Verse**: The network that distributes the shards of the Index. + +--- + +## 3. Tokenized Data Types + +### 3.1 The Report (The Atom) +* **Content**: "User X navigated A -> B at Time T". +* **Value**: Raw behavioral signal. +* **Token**: NFT (Unique observation). + +### 3.2 The Index (The Molecule) +* **Content**: A curated graph of Reports and Metadata. +* **Value**: Curation, organization, semantic tagging. +* **Token**: Access-Gated NFT (The "Book"). + * Creators sell access to their Index. + * Buyers pay in Verse Tokens. + * Hosts earn Verse Tokens for serving the Index shards. + +--- + +## 4. Comparison to Existing Models + +| Concept | Filecoin / IPFS | The Graph (GRT) | Verse (Proposed) | +| :--- | :--- | :--- | :--- | +| **Unit of Work** | Proof of Spacetime (Storing) | Indexing/Querying | **Proof of Access (Serving)** | +| **Data Structure** | Files / Blobs | Subgraphs (API) | **Spatial Graphs (UI/UX)** | +| **Consumption** | Download | API Call | **Navigation / Browsing** | +| **Incentive** | Persistence | Query Speed | **Availability & Curation** | + +## 5. Summary +This model aligns the economic incentive (serving data) with the user need (accessing knowledge). The "Index as Graph" concept ensures that the data structure of the network is native to the Graphshell client, making the "Verse" literally a traversable universe of graphs. diff --git a/design_docs/mere_docs/research/2026-02-27_freenet_takeaways_for_verse.md b/design_docs/mere_docs/research/2026-02-27_freenet_takeaways_for_verse.md new file mode 100644 index 000000000..94ea9127c --- /dev/null +++ b/design_docs/mere_docs/research/2026-02-27_freenet_takeaways_for_verse.md @@ -0,0 +1,117 @@ +> **Recovered research. Never implemented. Not current direction.** +> +> Recovered 2026-08-16 from the git history of the archived `graphshell` +> repository (`Code/archive/graphshell`), whose entire `design_docs/` tree is +> deleted at HEAD, so this text survives only in history. +> +> - Original path: `design_docs/verse_docs/research/2026-02-27_freenet_takeaways_for_verse.md` +> - Source commit: `04d68365` +> +> None of it was ever built. It is filed here as research, not as a plan and +> not as a statement of where Mere is going. It speaks the retired Verse +> vocabulary (per `TERMINOLOGY.md`, *Verse* is a retired term and the +> network scope is Mere at network scope), and its relative links point at +> donor docs that no longer exist locally. Read the text below for the ideas, +> not the direction. + +--- + +# Freenet (freenet.org) Takeaways for Verse (2026-02-27) + +**Status**: Research Notes / External Pattern Review +**Scope**: Identify reusable architecture patterns from Freenet for Verse without inheriting unrelated complexity. + +## Sources Reviewed + +- https://freenet.org/ +- https://freenet.org/quickstart/ +- https://freenet.org/faq/ +- https://freenet.org/resources/manual/components/overview/ +- https://freenet.org/resources/manual/components/contracts/ +- https://freenet.org/resources/manual/components/delegates/ +- https://freenet.org/resources/manual/architecture/p2p-network/ +- https://freenet.org/resources/manual/architecture/irouting/ +- https://freenet.org/resources/manual/architecture/transport/ +- https://freenet.org/ghostkey/ + +## What Is Useful for Verse + +### 1. Split shared-state logic from private-identity logic + +Freenet separates public/shared contract execution from private delegate execution. +Verse should keep this boundary explicit: + +- Shared sync/state lanes: deterministic, replayable, testable. +- Identity/secret lanes: key material, trust, access control, signing. + +Why this matters for Verse: +- Reduces identity seam bleed into compositor/runtime paths. +- Makes access-denied and grant logic easier to audit and test as a separate authority. + +### 2. Capability-first runtime contracts + +Freenet's model is interface-forward (contracts/delegates/UI each have a clear role). +Verse should similarly lock down narrow capability surfaces for mods/providers: + +- Storage capability +- Sync/messaging capability +- Identity/trust capability +- Diagnostics capability + +Why this matters for Verse: +- Prevents implicit cross-module coupling. +- Makes provider swaps and migration slices lower risk. + +### 3. Local-node + browser-UI developer loop + +Freenet's quickstart story keeps local execution first and visible. +Verse should preserve the same operational property: + +- Deterministic local harness scenarios before distributed complexity. +- One canonical end-to-end scenario per major subsystem (sync, access control, diagnostics). + +Why this matters for Verse: +- Keeps Tier 1 quality gates concrete and repeatable. +- Avoids distributed-debug-first development. + +### 4. Protocol docs should map to executable checks + +Freenet manual is useful structurally, but some pages acknowledge implementation/spec drift. +Verse should adopt the good part (clear protocol docs) while adding a strict anti-drift guard: + +- Each protocol claim links to tests/harness receipts. +- Each critical channel family has schema assertions. +- Doc updates are required when contracts change. + +Why this matters for Verse: +- Maintains trust in architecture docs during rapid migration. + +## What to Avoid Copying + +### 1. Spec/implementation drift + +Do not allow architecture docs to become aspirational-only. +For Verse, any transport/sync claim should be tied to an existing test or marked explicitly as proposed. + +### 2. Premature economics coupling + +Freenet includes identity/economic mechanisms (Ghost Key, trust signals). +Verse should avoid coupling core sync correctness to token/economic layers at current maturity. + +### 3. Over-centralized "hint" paths + +A key current Graphshell/Verse risk is keeping singular hint paths central (focus/render routing). +Borrow interface clarity, not centralized orchestration that bypasses adapters. + +## Recommended Verse Follow-Ons + +1. Formalize `shared-state` vs `identity-secrets` authority boundaries in Verse architecture docs. +2. Introduce a capability matrix table for Verse providers/mods (allowed operations by subsystem). +3. Require doc-to-test linkage for every `verse.sync.*` and `verse.identity.*` contract claim. +4. Keep transport and protocol specs explicitly labeled `implemented` vs `proposed`. + +## Fit Assessment + +Adopt Freenet's **separation discipline** and **interface-first framing**. +Do not adopt broader network/economic complexity until Verse Tier 1 and Tier 2 done-gates are stable. + diff --git a/design_docs/mere_docs/research/2026-03-28_libp2p_nostr_synergy_for_verse.md b/design_docs/mere_docs/research/2026-03-28_libp2p_nostr_synergy_for_verse.md new file mode 100644 index 000000000..59a0c55c1 --- /dev/null +++ b/design_docs/mere_docs/research/2026-03-28_libp2p_nostr_synergy_for_verse.md @@ -0,0 +1,286 @@ +> **Recovered research. Never implemented. Not current direction.** +> +> Recovered 2026-08-16 from the git history of the archived `graphshell` +> repository (`Code/archive/graphshell`), whose entire `design_docs/` tree is +> deleted at HEAD, so this text survives only in history. +> +> - Original path: `design_docs/verse_docs/research/2026-03-28_libp2p_nostr_synergy_for_verse.md` +> - Source commit: `f36fc49b` +> +> None of it was ever built. It is filed here as research, not as a plan and +> not as a statement of where Mere is going. It speaks the retired Verse +> vocabulary (per `TERMINOLOGY.md`, *Verse* is a retired term and the +> network scope is Mere at network scope), and its relative links point at +> donor docs that no longer exist locally. Read the text below for the ideas, +> not the direction. +> +> The donor carried a repo-wide MPL-2.0 notice; it is kept verbatim below and +> has not been reviewed against this repository's MIT/Apache posture. + +--- + +<!-- This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at https://mozilla.org/MPL/2.0/. --> + +# libp2p + Nostr Synergy for Verse + +**Date**: 2026-03-28 +**Status**: Research / Design Exploration +**Purpose**: Analyze how libp2p and Nostr compose for Verse community-scale networking. Identify the unifying architectural pattern, enumerate concrete integration points, and surface gaps in the current design documentation. + +**Related**: + +- [`../technical_architecture/VERSE_AS_NETWORK.md`](../technical_architecture/VERSE_AS_NETWORK.md) — Verse network position; bilateral/community boundary +- [`../technical_architecture/2026-02-23_verse_tier2_architecture.md`](../technical_architecture/2026-02-23_verse_tier2_architecture.md) — Tier 2 dual-transport, VerseBlob, community primitives, Nostr signaling (§8) +- [`../technical_architecture/2026-03-05_verse_nostr_dvm_integration.md`](../technical_architecture/2026-03-05_verse_nostr_dvm_integration.md) — NIP-90 DVM compute layer, Verse-specific NIP-72 tags +- [`../../graphshell_docs/implementation_strategy/system/2026-03-05_network_architecture.md`](../../graphshell_docs/implementation_strategy/system/2026-03-05_network_architecture.md) — Three-context + two-fabric model; NIP-72/NIP-29 Verse integration (§4) +- [`../../nostr_docs/technical_architecture/nostr_relay_spec.md`](../../nostr_docs/technical_architecture/nostr_relay_spec.md) — Embedded relay Community mode +- [`../technical_architecture/2026-03-05_verse_economic_model.md`](../technical_architecture/2026-03-05_verse_economic_model.md) — Three-track economic model (sats/FIL/reputation) + +--- + +## 1. The Unifying Pattern: Control Plane / Data Plane + +The existing Verse architecture already separates Nostr and libp2p responsibilities, but the separation hasn't been given a name. Naming it sharpens every downstream decision. + +| Plane | Protocol | What flows | Persistence | Addressing | +|-------|----------|-----------|-------------|------------| +| **Control** | Nostr (WebSocket relays) | Identity, governance, membership, announcements, social signals, discovery metadata, payment receipts | Relay-persisted, globally queryable | `npub` / event ID | +| **Data** | libp2p (QUIC / GossipSub / Bitswap) | VerseBLOBs, index segments, engrams, WARC archives, bulk content | Ephemeral mesh, DHT-addressed | CIDv1 / PeerId | + +The rule: **Nostr events reference CIDs; libp2p delivers the content behind those CIDs.** Neither protocol does the other's job. + +The DVM integration doc states this directly: *"Nostr is never used for bulk data transfer. It is a signalling and discovery bus only."* This research document treats that statement as the architectural invariant and explores its consequences. + +### 1.1 Why This Split Works + +- **Nostr relays are optimized for small signed events** (~64 KB max per NIP-01). They provide global discoverability, relay-backed persistence, and a social graph. They are not built for streaming megabytes. +- **libp2p is optimized for content-addressed bulk transfer.** Kademlia DHT, GossipSub, and Bitswap are designed for exactly this. They have no built-in persistence or social layer. +- **Neither protocol needs to be extended to cover the other's role.** The interface is a CID reference in a Nostr event tag — trivially parseable, universally addressable. + +### 1.2 Interaction Points + +Nostr and libp2p meet at well-defined seams: + +1. **Nostr event references a CID** → libp2p retrieves the content. +2. **Nostr NIP-72 defines a community** → libp2p forms the content swarm. +3. **Nostr NIP-29 enforces membership** → libp2p distributes content to authenticated members. +4. **Nostr kind 30078 carries multiaddrs** → libp2p connects using them. +5. **Nostr NIP-90 dispatches compute jobs** → libp2p delivers input/output content. +6. **Nostr NIP-57 zaps settle payment** → libp2p transfer generates the Proof of Access receipt. + +Each interaction crosses the plane boundary exactly once. No protocol reaches into the other's domain. + +--- + +## 2. Integration Point Analysis + +### 2.1 Community Bootstrap via Nostr + +**Problem**: Tier 2 architecture §12.4 flags community bootstrapping as an open question — how do the first 100 users find each other when there is no DHT yet? + +**Solution**: Nostr is the bootstrap layer. + +The DVM integration doc already specifies Verse-specific tags in the community kind 34550 event: + +```json +{ + "kind": 34550, + "tags": [ + ["d", "<community-id>"], + ["verse_dht_bootstrap", "<libp2p_multiaddr_1>", "<libp2p_multiaddr_2>"], + ["verse_community_id", "<hex-community-id>"], + ["verse_manifest_cid", "<CIDv1 of CommunityManifest blob>"] + ] +} +``` + +This gives the community's initial bootstrap addresses. But bootstrap can go further: + +**Member self-advertisement**: Each community member publishes a replaceable kind 30078 event advertising their own libp2p multiaddr when they come online. Tag structure: + +```json +{ + "kind": 30078, + "tags": [ + ["d", "verse-peer-<community-id>"], + ["verse_community_id", "<hex-community-id>"], + ["libp2p_multiaddr", "<multiaddr>"], + ["online_since", "<unix-timestamp>"] + ] +} +``` + +This creates a **self-healing bootstrap set**: new joiners query Nostr relays for the community's kind 30078 events, extract multiaddrs from recent member announcements, and try them in parallel. No hardcoded rendezvous servers needed. Stale multiaddrs (peer offline) fail quickly and the joiner falls through to the next. + +The embedded relay in Community mode (nostr_relay_spec.md §3.3) doubles as a bootstrap cache: it stores these member-multiaddr events locally, so even when public relays churn, community operators maintain a local discovery index. + +### 2.2 Nostr Social Graph as libp2p Trust Signal + +**Problem**: libp2p's Kademlia DHT is trustless by design — any peer can join. But Verse communities need trust differentiation: the Tier 2 rebroadcast levels (Core → Extended → Public) and the curated governance model require distinguishing trusted peers from anonymous participants. + +**Solution**: Feed Nostr-derived trust into libp2p's peer scoring. + +GossipSub 1.1 has a built-in peer scoring framework. Application-layer scoring callbacks can influence message propagation priority. The Nostr social graph provides the trust signal: + +| Nostr signal | GossipSub scoring effect | +|-------------|------------------------| +| Peer's `npub` is in your NIP-02 follows list | Elevated score — messages propagated preferentially | +| Peer's `npub` is in the community moderator set (kind 34550) | Maximum trust — messages validated and relayed first | +| Peer's `npub` has high reputation in Proof of Access ledger | Positive score bonus proportional to reputation tier | +| Peer's `npub` is in your contacts list as `Blocked` | Score floor — messages deprioritized or dropped | +| Peer's `npub` is unknown (no social graph signal) | Neutral score — standard GossipSub behavior | + +This doesn't require modifying GossipSub — it's application-layer scoring fed into libp2p's existing peer scoring API. The Nostr social graph acts as a pre-existing web of trust that the data plane can leverage without having to build its own. + +### 2.3 NIP-29 Membership as libp2p Swarm Gate + +**Problem**: NIP-29 provides relay-enforced membership for private Verse spaces, but the enforcement only covers Nostr event access on that relay. The libp2p swarm for bulk content distribution has no equivalent membership gate — any peer with the community's GossipSub topic can attempt to join. + +**Solution**: Use the NIP-29 relay as a membership attestation issuer for the libp2p swarm. + +Flow: + +1. Peer authenticates to the NIP-29 relay via NIP-42 AUTH. +2. Relay verifies membership and issues a short-lived **swarm attestation**: a signed Nostr event (custom kind) containing the peer's `npub`, their `PeerId`, and an expiry timestamp. +3. Peer presents this attestation to libp2p peers when connecting to the private community's GossipSub topic. +4. Receiving peers verify the attestation signature against the relay's pubkey (published in the community kind 34550 definition) and check expiry. +5. Peers with a valid attestation are accepted into the swarm. Peers without one are rejected. + +```rust +/// Relay-issued attestation for libp2p swarm admission +struct SwarmAttestation { + /// NIP-29 relay that issued this attestation + relay_pubkey: NostrPubkey, + /// The authenticated member + member_npub: NostrPubkey, + /// The member's libp2p PeerId (derived from same Ed25519 root) + member_peer_id: PeerId, + /// Community this attestation is valid for + community_id: CommunityId, + /// Wall-clock expiry (short-lived: 1–24 hours) + expires_at: SystemTime, + /// Relay's signature over the above fields + signature: NostrSignature, +} +``` + +This gives relay-enforced membership (NIP-29's strength) applied to the libp2p data plane. The relay is the bouncer; libp2p is the venue. Attestation refresh is periodic — the peer re-authenticates to the relay before expiry. + +**Trade-off**: This makes the NIP-29 relay a liveness dependency for private swarm access (peers can't get fresh attestations if the relay is down). Mitigation: attestations are valid for hours, not seconds. The swarm continues operating during brief relay outages; only new joins are blocked. + +### 2.4 Dual-Rail Publication (Nostr + GossipSub) + +**Problem**: Community governance events need both durability (must be retrievable months later) and real-time propagation (active members should see them immediately). Nostr provides durability; GossipSub provides immediacy. Neither alone covers both. + +**Solution**: Dual-rail publish — send the event to both channels simultaneously. + +| Action | Nostr (durable rail) | GossipSub (real-time rail) | +|--------|---------------------|--------------------------| +| FLora checkpoint approved | kind 4550 approval event (relay-persisted) | Announcement to active swarm members | +| New index epoch published | kind 30078 with CID references | GossipSub notification + Bitswap for segment content | +| Moderation action | kind 9000-9009 (NIP-29) or kind 4550 | GossipSub blacklist propagation | +| Community manifest update | kind 34550 (replaceable event) | GossipSub to inform connected peers immediately | +| Member came online | kind 30078 (replaceable, self-advertisement) | GossipSub peer exchange | + +The pattern: **publish to Nostr for permanence, broadcast on GossipSub for immediacy**. Peers that were offline during the GossipSub broadcast catch up from Nostr when they rejoin. Deduplication is trivial — events have unique IDs. + +This is not double the bandwidth: the Nostr event is the metadata (< 1 KB), the GossipSub message is the same metadata or a pointer to it. Bulk content is always libp2p-only. + +### 2.5 NIP-90 DVMs as the Compute Bridge + +The DVM integration doc covers this comprehensively. The key synergy summarized: + +- **Job dispatch** (control plane): Nostr kind 5000+ events. Contain CID references to input data, not the data itself. +- **Input/output transfer** (data plane): DVM provider pulls input content from the libp2p swarm, pushes result content back as a VerseBlob. +- **Payment** (control plane): NIP-57 Lightning zaps on the result event. +- **Reputation** (control plane): Proof of Access receipt generated, stored as Nostr event or in the PoA ledger. + +Verse communities don't need to build compute infrastructure — they outsource it to the Nostr DVM marketplace while keeping content distribution on libp2p. + +### 2.6 The Embedded Relay as Unified Community Service + +The nostr_relay_spec Community mode makes a single Graphshell instance a combined Nostr relay + libp2p peer: + +**Nostr side** (relay): +- Stores governance events, membership, announcements +- Enforces NIP-29 group membership +- Provides NIP-46 bunker transport for community signing +- Caches member multiaddr events for bootstrap + +**libp2p side** (peer): +- Participates in GossipSub for content distribution +- Serves VerseBLOBs via Bitswap +- Routes DHT queries for content discovery + +Both share the same Ed25519 identity (via `P2PIdentitySecret`). A community member connects to one endpoint and gets access to the full stack. This makes "run a Verse community" a single toggle in Graphshell's settings, not two separate processes. + +--- + +## 3. What the Existing Docs Get Right + +The existing architecture is well-positioned. Specifically: + +- **network_architecture.md §4.4** — layer assignment table correctly separates Nostr (community definition, membership, host approval) from libp2p (peer discovery, state replication, blob transfer). +- **verse_tier2_architecture.md §8** — correctly positions Nostr as convenience signaling, not a dependency. +- **verse_nostr_dvm_integration.md §2–3** — comprehensive layer assignment and Verse-specific NIP-72 tag schema. +- **nostr_relay_spec.md §3.3** — Community relay mode covers the Nostr-side infrastructure for community operation. +- **The three-context + two-fabric model** — Nostr as a cross-cutting fabric rather than a competing substrate is the right framing. + +--- + +## 4. Gaps in Current Documentation + +### 4.1 No explicit control-plane / data-plane naming + +The separation exists in practice but isn't named as an architectural invariant. Design decisions have to re-derive the boundary each time. Adding a named principle ("Nostr is the control plane; libp2p is the data plane; they meet at CID references") to VERSE_AS_NETWORK.md or the network architecture doc would make the boundary self-documenting. + +**Scope**: One paragraph addition to an existing doc. + +### 4.2 NIP-29 → libp2p swarm gating not specified + +The relay-issued `SwarmAttestation` pattern (§2.3 above) closes the private-community story for the data plane. Currently, NIP-29 only gates Nostr event access; the libp2p swarm has no equivalent membership enforcement. + +**Scope**: New section in verse_tier2_architecture.md or a standalone spec. Medium complexity — requires defining the attestation event kind, expiry semantics, and the libp2p connection guard. + +### 4.3 GossipSub peer scoring from Nostr trust signals not documented + +The tier2 doc discusses moderation and curator signatures but doesn't connect them to GossipSub 1.1's native peer scoring API. The mapping (§2.2 above) is straightforward but needs to be specified so implementers know to wire it. + +**Scope**: New subsection in verse_tier2_architecture.md §4 (Community Model). Small addition. + +### 4.4 Member self-advertisement events not specified + +Tier2 §8 covers community-level kind 30078 announcements. Individual members publishing their own replaceable kind 30078 events with their multiaddr (§2.1 above) for self-healing bootstrap isn't specified. + +**Scope**: Extension to the existing kind 30078 usage in tier2 §8. Small addition. + +### 4.5 Dual-rail publication pattern not formalized + +The idea that governance events go to both Nostr and GossipSub simultaneously, with Nostr as the durable fallback, is implied but not stated as a formal pattern. Formalizing it prevents future specs from accidentally making GossipSub the sole distribution channel for durable content. + +**Scope**: New subsection or a "publication patterns" section in VERSE_AS_NETWORK.md. Small addition. + +--- + +## 5. Curve Mismatch Accommodation + +Nostr uses secp256k1; libp2p/iroh use Ed25519. The network_architecture.md §7 already addresses this with the signed presence-binding assertion. For the synergy patterns above, this means: + +- `SwarmAttestation` (§2.3) is signed with the relay's Nostr key (secp256k1). libp2p peers verify it using the relay's Nostr pubkey from the community kind 34550 event. +- GossipSub scoring (§2.2) maps `npub` (secp256k1) → `PeerId` (Ed25519) via the same binding assertion. The mapping is cached locally per community. +- Member self-advertisement (§2.1) events are signed with the member's Nostr key but contain the libp2p `PeerId` as a tag. Peers verify the Nostr signature and accept the `PeerId` claim. + +No new curve bridging is needed beyond what the existing binding assertion provides. + +--- + +## 6. Summary + +libp2p and Nostr compose naturally for Verse because they occupy non-overlapping roles: + +- **Nostr** tells you *what exists, who made it, who approved it, and where to find peers*. +- **libp2p** *moves the actual content between those peers*. + +The five integration patterns identified (bootstrap, trust scoring, swarm gating, dual-rail publication, compute bridging) all follow the control-plane / data-plane split without exception. The gaps in current documentation (§4) are additions to existing specs, not architectural changes — the foundation is sound. diff --git a/design_docs/mere_docs/research/2026-04-17_verse_distributed_index_protocol_v0_1.md b/design_docs/mere_docs/research/2026-04-17_verse_distributed_index_protocol_v0_1.md index a5367e47c..dd92d5e82 100644 --- a/design_docs/mere_docs/research/2026-04-17_verse_distributed_index_protocol_v0_1.md +++ b/design_docs/mere_docs/research/2026-04-17_verse_distributed_index_protocol_v0_1.md @@ -14,7 +14,7 @@ > donor docs that no longer exist locally. Read the text below for the ideas, > not the direction. > -> The filename was normalised from `..._v0.1.md` to `..._v0_1.md` to match its +> The filename was normalized from `..._v0.1.md` to `..._v0_1.md` to match its > successor. It was already archived in the donor: VGCP replaced it as protocol > authority the same day, and VGCP is recovered beside it as > `2026-04-17_verse_graph_contribution_protocol_v0_1.md`. From 00d1d168a7a871d3ad605de452982cfccc86d5e2 Mon Sep 17 00:00:00 2001 From: Markik <54276851+mark-ik@users.noreply.github.com> Date: Tue, 18 Aug 2026 02:51:32 -0400 Subject: [PATCH 5/5] Preserve Gemini certificate changes --- crates/system/fetch/src/lib.rs | 40 ++++++++++++++++++++++++++++++-- crates/system/fetch/src/tests.rs | 21 +++++++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/crates/system/fetch/src/lib.rs b/crates/system/fetch/src/lib.rs index 65d0410ff..bc0985fc6 100644 --- a/crates/system/fetch/src/lib.rs +++ b/crates/system/fetch/src/lib.rs @@ -36,6 +36,9 @@ use serde::{Deserialize, Serialize}; use tokio::runtime::Builder; use zeroize::Zeroizing; +/// Host-supplied durable trust storage for Gemini-style TLS. +pub use errand::TofuStore as SmolwebTofuStore; + /// The most redirects a smolweb fetch will follow before giving up. const MAX_REDIRECTS: usize = 5; @@ -80,6 +83,15 @@ pub enum FetchFailure { prompt: String, code: Option<u8>, }, + /// A Gemini capsule presented a certificate that differs from its durable + /// pin. The request was not sent; the host must ask a human before replacing + /// `pinned` with `seen` and retrying `url`. + CertificateChanged { + url: String, + target: String, + pinned: String, + seen: String, + }, /// A terminal transport, protocol, HTTP, or size-limit failure. Failed(String), } @@ -89,6 +101,9 @@ impl std::fmt::Display for FetchFailure { match self { Self::InputRequired { prompt, .. } => write!(f, "input required: {prompt}"), Self::ClientCertificateRequired { .. } => f.write_str("client certificate required"), + Self::CertificateChanged { target, .. } => { + write!(f, "certificate for {target} changed") + } Self::Failed(error) => f.write_str(error), } } @@ -413,7 +428,14 @@ pub async fn fetch_page_anonymous_capped(url: &str, max_bytes: usize) -> Result< /// durability claim; a host with durable trust state should install its own /// [`errand::TofuStore`] instead. pub fn install_in_memory_smolweb_tofu() { - errand::set_trust_store(Arc::new(errand::InMemoryTofu::new())); + install_smolweb_tofu(Arc::new(errand::InMemoryTofu::new())); +} + +/// Install a host-owned Gemini trust store for every smolweb request in this +/// process. The host keeps the concrete store so certificate-change approval +/// can replace one pin before retrying the refused request. +pub fn install_smolweb_tofu(store: Arc<dyn SmolwebTofuStore>) { + errand::set_trust_store(store); } /// Fetch a page as the **crawler**, identifying with [`CRAWLER_USER_AGENT`]. http(s) @@ -449,7 +471,7 @@ async fn smolweb_fetch( } None => errand::fetch_url_timeout(¤t, SMOLWEB_TIMEOUT).await, } - .map_err(|error| FetchFailure::Failed(error.to_string()))?; + .map_err(|error| smolweb_transport_failure(¤t, error))?; match response.status { errand::Status::Success => { let content_type = smolweb_content_type(¤t, &response); @@ -486,6 +508,20 @@ async fn smolweb_fetch( Err(FetchFailure::Failed("too many redirects".to_string())) } +fn smolweb_transport_failure(current: &url::Url, error: errand::Error) -> FetchFailure { + match error { + errand::Error::CertificateChanged { host, pinned, seen } => { + FetchFailure::CertificateChanged { + url: current.to_string(), + target: host, + pinned, + seen, + } + } + error => FetchFailure::Failed(error.to_string()), + } +} + fn smolweb_input_failure(current: &url::Url, response: &errand::Response) -> FetchFailure { FetchFailure::InputRequired { url: current.to_string(), diff --git a/crates/system/fetch/src/tests.rs b/crates/system/fetch/src/tests.rs index 4016f962e..52f61b3c8 100644 --- a/crates/system/fetch/src/tests.rs +++ b/crates/system/fetch/src/tests.rs @@ -122,6 +122,27 @@ fn gemini_identity_is_scoped_to_one_capsule_origin() { assert_eq!(identity.origin(), "gemini://capsule.example"); } +#[test] +fn certificate_change_keeps_the_target_and_both_fingerprints_typed() { + let current = url::Url::parse("gemini://capsule.example:1966/private").unwrap(); + assert_eq!( + smolweb_transport_failure( + ¤t, + errand::Error::CertificateChanged { + host: "capsule.example:1966".into(), + pinned: "11".repeat(32), + seen: "22".repeat(32), + }, + ), + FetchFailure::CertificateChanged { + url: current.to_string(), + target: "capsule.example:1966".into(), + pinned: "11".repeat(32), + seen: "22".repeat(32), + } + ); +} + #[test] fn state_tag_distinguishes_transitions() { let ready = ContentState::Ready(Fetched {