diff --git a/docs/site/docs/fundamentals/mcp.mdx b/docs/site/docs/fundamentals/mcp.mdx index 59120d41c0f..a31888e6122 100644 --- a/docs/site/docs/fundamentals/mcp.mdx +++ b/docs/site/docs/fundamentals/mcp.mdx @@ -307,3 +307,23 @@ In addition to tools, the MCP server exposes structured **resources** and **prom | `debug_logs` | Triage node issues from recent log output | | `torrent_status` | Snapshot download health check | | `sync_analysis` | Node sync progress and performance | + +--- + +## These Docs as a Single File + +MCP connects a model to your **running node**. To let a model answer questions about **Erigon itself** — flags, pruning modes, RPC namespaces, hardware sizing — point it at this documentation in machine-readable form, following the [llmstxt.org](https://llmstxt.org) convention: + +| File | Contents | Use it when | +|------|----------|-------------| +| [llms.txt](https://docs.erigon.tech/llms.txt) | Page index with titles and descriptions | The model fetches pages on demand, or its context window is small | +| [llms-full.txt](https://docs.erigon.tech/llms-full.txt) | Every documentation page, in full, as one plain-text file | You want the whole corpus in context, offline, with nothing scraped | + +Both are generated from the same Markdown source as this site by a script in the repository, and CI fails when a documentation change lands without regenerating them, so they cannot quietly go stale. + +```bash +# Give a local model the whole documentation, with no network access +curl -sO https://docs.erigon.tech/llms-full.txt +``` + +The full file is around 420 KB. If your model's context cannot hold it, use `llms.txt` to pick the pages you need, or slice the sections you want out of the full file — each page begins with a `# Title` heading. diff --git a/docs/site/docs/get-started/why-using-erigon.mdx b/docs/site/docs/get-started/why-using-erigon.mdx index c555338df50..d0d61a4e09a 100644 --- a/docs/site/docs/get-started/why-using-erigon.mdx +++ b/docs/site/docs/get-started/why-using-erigon.mdx @@ -179,3 +179,5 @@ The interface is **read-only** and localhost-bound — no risk of unintended sta # Register with Claude Code in one command claude mcp add --transport sse erigon http://127.0.0.1:8553/sse ``` + +Where MCP lets a model query your *node*, [llms.txt and llms-full.txt](../fundamentals/mcp#these-docs-as-a-single-file) let it read these *docs* — the whole documentation as one plain-text file, for offline use. diff --git a/docs/site/docusaurus.config.ts b/docs/site/docusaurus.config.ts index 2593cad2ce2..d9623d9770b 100644 --- a/docs/site/docusaurus.config.ts +++ b/docs/site/docusaurus.config.ts @@ -86,6 +86,27 @@ export default async function createConfig(): Promise { attributes: {}, innerHTML: 'window.plausible=window.plausible||function(){(plausible.q=plausible.q||[]).push(arguments)},plausible.init=plausible.init||function(i){plausible.o=i||{}};plausible.init()', }, + // Machine-readable copies of this site (llmstxt.org convention). Advertised + // on every page so crawlers and agents can find them without guessing the + // path — they are static files, so nothing else links to them. + { + tagName: 'link', + attributes: { + rel: 'alternate', + type: 'text/plain', + href: 'https://docs.erigon.tech/llms.txt', + title: 'Erigon Documentation — page index for LLMs', + }, + }, + { + tagName: 'link', + attributes: { + rel: 'alternate', + type: 'text/plain', + href: 'https://docs.erigon.tech/llms-full.txt', + title: 'Erigon Documentation — full text for LLMs', + }, + }, ], plugins: [ @@ -150,6 +171,15 @@ export default async function createConfig(): Promise { changefreq: 'weekly', priority: 0.5, ignorePatterns: ['/search'], + // The llms.txt artifacts live in static/, so Docusaurus never routes + // them and the default sitemap omits them. Nothing else on the web + // links to them either, which leaves them unindexable and unreachable + // by search — append them explicitly. + createSitemapItems: async ({defaultCreateSitemapItems, ...rest}) => [ + ...(await defaultCreateSitemapItems(rest)), + {url: 'https://docs.erigon.tech/llms.txt', changefreq: 'weekly', priority: 0.5}, + {url: 'https://docs.erigon.tech/llms-full.txt', changefreq: 'weekly', priority: 0.5}, + ], }, } satisfies Preset.Options], ], diff --git a/docs/site/scripts/generate-llms.py b/docs/site/scripts/generate-llms.py index 7ca09256013..e7a115c78e9 100644 --- a/docs/site/scripts/generate-llms.py +++ b/docs/site/scripts/generate-llms.py @@ -285,16 +285,25 @@ def strip_mdx(text): # index pages. After MDX stripping, they collapse to a structureless pile of # title/desc fragments. We detect the lp-card pattern and synthesize a clean # bullet list instead, keeping link structure intact for LLM consumption. -_LANDING_CARD_RE = re.compile( - r']*\bto=[\'"]([^\'"]+)[\'"][^>]*>' - r'(?:.*?)' - r']*\blp-card-title\b[^>]*>([^<]+)' - r'(?:.*?)' - r']*\blp-card-desc\b[^>]*>([^<]+)' - r'(?:.*?)' - r'', +# Parsed in two stages, deliberately. A single pattern spanning the whole card +# cannot express "and never cross into the next card": with `[^<]+` for the text +# and `.*?` for the gaps, a description containing inline markup (``, +# ``) fails to match locally, and the engine then scans forward and pairs +# the title with the *next* card's description — silently swallowing the card in +# between. Matching each block first makes a card boundary unrepresentable. +_LANDING_LINK_RE = re.compile( + r']*\bto=[\'"]([^\'"]+)[\'"][^>]*>(.*?)', re.DOTALL, ) +_CARD_TITLE_RE = re.compile( + r']*\blp-card-title\b[^>]*>(.*?)', re.DOTALL) +_CARD_DESC_RE = re.compile( + r']*\blp-card-desc\b[^>]*>(.*?)', re.DOTALL) + + +def _card_text(fragment): + """Flatten a card's inner HTML to plain text, collapsing whitespace.""" + return re.sub(r'\s+', ' ', re.sub(r'<[^>]+>', '', fragment)).strip() _LANDING_HERO_RE = re.compile( r']*\blp-hero\b[^>]*>' r'(?:.*?)' @@ -312,10 +321,28 @@ def synthesize_landing(raw_body): Card hrefs that start with `/` are rewritten to absolute BASE_URL form so bullets remain useful when the body is read out of context. """ - cards = list(_LANDING_CARD_RE.finditer(raw_body)) + cards = [] + for link in _LANDING_LINK_RE.finditer(raw_body): + href, block = link.group(1), link.group(2) + title = _CARD_TITLE_RE.search(block) + desc = _CARD_DESC_RE.search(block) + if not title or not desc: + continue + cards.append((href, _card_text(title.group(1)), + _card_text(desc.group(1)))) if not cards: return None + # A card that parses to nothing is invisible in the output, and `--check` + # only compares generated output against committed output — so a systematic + # parser regression stays green in CI while the corpus quietly loses pages. + expected = raw_body.count('lp-card-title') + if len(cards) != expected: + raise SystemExit( + f"synthesize_landing dropped cards: parsed {len(cards)} " + f"of {expected} lp-card-title occurrences" + ) + out = [] hero = _LANDING_HERO_RE.search(raw_body) if hero: @@ -326,12 +353,10 @@ def synthesize_landing(raw_body): out.append("## Sections") out.append("") - for m in cards: - href = m.group(1).strip() + for href, title, desc in cards: + href = href.strip() if href.startswith("/"): href = BASE_URL + href.rstrip("/") - title = m.group(2).strip() - desc = m.group(3).strip() out.append(f"- [{title}]({href}): {desc}") return "\n".join(out) diff --git a/docs/site/scripts/test_generate_llms.py b/docs/site/scripts/test_generate_llms.py index 187ec6c154c..b7cce696977 100644 --- a/docs/site/scripts/test_generate_llms.py +++ b/docs/site/scripts/test_generate_llms.py @@ -159,6 +159,68 @@ def test_no_cards_returns_none(self): out = g.synthesize_landing("# Hello\n\nProse only, no cards.") self.assertIsNone(out) + def test_desc_with_inline_markup_is_extracted(self): + """Inline markup in a description must not defeat the match.""" + body = """ +
+ +
Flexible Pruning
+
--prune.mode=minimal gives a smaller footprint.
+ +
+""" + out = g.synthesize_landing(body) + self.assertIsNotNone(out) + self.assertIn("--prune.mode=minimal gives a smaller footprint.", out) + self.assertNotIn("", out) + self.assertNotIn("", out) + + def test_markup_card_does_not_swallow_the_next_card(self): + """A card whose desc has markup must not steal the following card's desc.""" + body = """ +
+ +
Immutable Data
+
Historical data lives in immutable files.
+ + +
Staged Sync
+
Splits processing into specialised stages.
+ +
+""" + out = g.synthesize_landing(body) + self.assertIsNotNone(out) + bullets = [ln for ln in out.split("\n") if ln.startswith("- [")] + self.assertEqual(len(bullets), 2) + self.assertIn( + "- [Immutable Data](https://docs.erigon.tech/immutable): " + "Historical data lives in immutable files.", + bullets, + ) + self.assertIn( + "- [Staged Sync](https://docs.erigon.tech/staged-sync): " + "Splits processing into specialised stages.", + bullets, + ) + + def test_dropped_card_raises_instead_of_silently_shrinking(self): + """A card the parser cannot read must fail loudly — `--check` cannot see it.""" + body = """ +
+ +
Fine
+
Parses correctly.
+ + +
Missing Its Description
+ +
+""" + with self.assertRaises(SystemExit) as ctx: + g.synthesize_landing(body) + self.assertIn("dropped cards", str(ctx.exception)) + class LeadingH1StripTests(unittest.TestCase): """The build() body-prep step strips a leading H1 to avoid duplicate headings.""" diff --git a/docs/site/static/llms-full.txt b/docs/site/static/llms-full.txt index 63a5a94870a..b875b3d5a63 100644 --- a/docs/site/static/llms-full.txt +++ b/docs/site/static/llms-full.txt @@ -38,12 +38,15 @@ URL: https://docs.erigon.tech/get-started/why-using-erigon - [Integrated Consensus Layer (Caplin)](../fundamentals/caplin): Built-in consensus client — no need to run and manage separate software like Lighthouse. One binary handles both execution and consensus, simplifying setup for stakers and node runners. - [State Storage (Flat DB)](../fundamentals/optimizing-storage): Replaces the Merkle Patricia Trie with a flat key-value database. Faster reads and writes, and a smaller on-disk footprint for any node type. -- [Immutable, Decentralised Data](../fundamentals/modules/downloader): Splits data processing into specialised stages, minimising random disk I/O and write amplification. Efficiently integrates large datasets into Erigon's flat DB. +- [Immutable, Decentralised Data](../fundamentals/modules/downloader): Historical data lives in immutable files distributed via BitTorrent — up to 10× cheaper to backup, repair, and distribute. No double-disk usage, no complex serialisation. +- [Staged Sync](../fundamentals/pruning-modes): Splits data processing into specialised stages, minimising random disk I/O and write amplification. Efficiently integrates large datasets into Erigon's flat DB. - [Predictable RPC Performance](../fundamentals/modules/rpc-daemon): No background compaction means no unpredictable resource spikes. RPC throughput stays stable and consistent — critical for providers and validators who cannot afford surprises. - [Modularity](../fundamentals/modules/): Core node, RPC Daemon, and TxPool run as independent processes with per-component resource limits. If one crashes the rest stay alive. Run multiple instances without downtime. - [OtterSync](../fundamentals/pruning-modes): Shifts initial sync from CPU-intensive transaction re-execution to network download — similar to BitTorrent for state data. Reduces sync time for both full and Archive nodes. -- [Flexible Pruning](../fundamentals/optimizing-storage): Modular architecture and predictable performance handle high-volume traffic with low latency. Smaller database cuts hardware costs. Caplin reduces missed attestations and slashing risk. +- [Flexible Pruning](../fundamentals/optimizing-storage): --prune.mode=minimal gives a smaller-than-full-node footprint that still satisfies all validator requirements. Right-size your storage without sacrificing functionality. +- [RPC Providers & Large Stakers](../fundamentals/modules/rpc-daemon): Modular architecture and predictable performance handle high-volume traffic with low latency. Smaller database cuts hardware costs. Caplin reduces missed attestations and slashing risk. - [Home Users & Solo Stakers](../staking/): Run a full or archive node on a consumer SSD. Fast sync cuts setup from days to hours. No third-party servers — your node, your keys, your contribution to decentralisation. +- [Developers](../interacting-with-erigon/): Full trace_ namespace, historical eth_getProof, historical blobs, and gRPC API. Modular components let you update, extend, or replace parts without rebuilding the whole stack. --- @@ -3920,6 +3923,26 @@ In addition to tools, the MCP server exposes structured **resources** and **prom --- +## These Docs as a Single File + +MCP connects a model to your **running node**. To let a model answer questions about **Erigon itself** — flags, pruning modes, RPC namespaces, hardware sizing — point it at this documentation in machine-readable form, following the [llmstxt.org](https://llmstxt.org) convention: + +| File | Contents | Use it when | +|------|----------|-------------| +| [llms.txt](https://docs.erigon.tech/llms.txt) | Page index with titles and descriptions | The model fetches pages on demand, or its context window is small | +| [llms-full.txt](https://docs.erigon.tech/llms-full.txt) | Every documentation page, in full, as one plain-text file | You want the whole corpus in context, offline, with nothing scraped | + +Both are generated from the same Markdown source as this site by a script in the repository, and CI fails when a documentation change lands without regenerating them, so they cannot quietly go stale. + +```bash +# Give a local model the whole documentation, with no network access +curl -sO https://docs.erigon.tech/llms-full.txt +``` + +The full file is around 420 KB. If your model's context cannot hold it, use `llms.txt` to pick the pages you need, or slice the sections you want out of the full file — each page begins with a `# Title` heading. + +--- + # Otterscan URL: https://docs.erigon.tech/fundamentals/otterscan diff --git a/llms-full.txt b/llms-full.txt index 63a5a94870a..b875b3d5a63 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -38,12 +38,15 @@ URL: https://docs.erigon.tech/get-started/why-using-erigon - [Integrated Consensus Layer (Caplin)](../fundamentals/caplin): Built-in consensus client — no need to run and manage separate software like Lighthouse. One binary handles both execution and consensus, simplifying setup for stakers and node runners. - [State Storage (Flat DB)](../fundamentals/optimizing-storage): Replaces the Merkle Patricia Trie with a flat key-value database. Faster reads and writes, and a smaller on-disk footprint for any node type. -- [Immutable, Decentralised Data](../fundamentals/modules/downloader): Splits data processing into specialised stages, minimising random disk I/O and write amplification. Efficiently integrates large datasets into Erigon's flat DB. +- [Immutable, Decentralised Data](../fundamentals/modules/downloader): Historical data lives in immutable files distributed via BitTorrent — up to 10× cheaper to backup, repair, and distribute. No double-disk usage, no complex serialisation. +- [Staged Sync](../fundamentals/pruning-modes): Splits data processing into specialised stages, minimising random disk I/O and write amplification. Efficiently integrates large datasets into Erigon's flat DB. - [Predictable RPC Performance](../fundamentals/modules/rpc-daemon): No background compaction means no unpredictable resource spikes. RPC throughput stays stable and consistent — critical for providers and validators who cannot afford surprises. - [Modularity](../fundamentals/modules/): Core node, RPC Daemon, and TxPool run as independent processes with per-component resource limits. If one crashes the rest stay alive. Run multiple instances without downtime. - [OtterSync](../fundamentals/pruning-modes): Shifts initial sync from CPU-intensive transaction re-execution to network download — similar to BitTorrent for state data. Reduces sync time for both full and Archive nodes. -- [Flexible Pruning](../fundamentals/optimizing-storage): Modular architecture and predictable performance handle high-volume traffic with low latency. Smaller database cuts hardware costs. Caplin reduces missed attestations and slashing risk. +- [Flexible Pruning](../fundamentals/optimizing-storage): --prune.mode=minimal gives a smaller-than-full-node footprint that still satisfies all validator requirements. Right-size your storage without sacrificing functionality. +- [RPC Providers & Large Stakers](../fundamentals/modules/rpc-daemon): Modular architecture and predictable performance handle high-volume traffic with low latency. Smaller database cuts hardware costs. Caplin reduces missed attestations and slashing risk. - [Home Users & Solo Stakers](../staking/): Run a full or archive node on a consumer SSD. Fast sync cuts setup from days to hours. No third-party servers — your node, your keys, your contribution to decentralisation. +- [Developers](../interacting-with-erigon/): Full trace_ namespace, historical eth_getProof, historical blobs, and gRPC API. Modular components let you update, extend, or replace parts without rebuilding the whole stack. --- @@ -3920,6 +3923,26 @@ In addition to tools, the MCP server exposes structured **resources** and **prom --- +## These Docs as a Single File + +MCP connects a model to your **running node**. To let a model answer questions about **Erigon itself** — flags, pruning modes, RPC namespaces, hardware sizing — point it at this documentation in machine-readable form, following the [llmstxt.org](https://llmstxt.org) convention: + +| File | Contents | Use it when | +|------|----------|-------------| +| [llms.txt](https://docs.erigon.tech/llms.txt) | Page index with titles and descriptions | The model fetches pages on demand, or its context window is small | +| [llms-full.txt](https://docs.erigon.tech/llms-full.txt) | Every documentation page, in full, as one plain-text file | You want the whole corpus in context, offline, with nothing scraped | + +Both are generated from the same Markdown source as this site by a script in the repository, and CI fails when a documentation change lands without regenerating them, so they cannot quietly go stale. + +```bash +# Give a local model the whole documentation, with no network access +curl -sO https://docs.erigon.tech/llms-full.txt +``` + +The full file is around 420 KB. If your model's context cannot hold it, use `llms.txt` to pick the pages you need, or slice the sections you want out of the full file — each page begins with a `# Title` heading. + +--- + # Otterscan URL: https://docs.erigon.tech/fundamentals/otterscan