Name the answer-engine crawlers, and ship the territory as well as the map

sites/forensics/public/robots.txt

@@ -7,4 +7,75 @@
 User-agent: *
 Allow: /
 
+# The answer-engine crawlers are named individually below, and every block says
+# exactly what the wildcard above already said.
+#
+# WHY REPEAT YOURSELF. Under the robots.txt matching rules a crawler obeys the
+# most specific group that names it and ignores every other group, the wildcard
+# included. So a named block is not additive -- it REPLACES `User-agent: *` for
+# that crawler, which means each one has to be complete on its own. They are:
+# one `Allow: /` each, identical to the wildcard. Nothing here grants or
+# withdraws access that was not already granted.
+#
+# What it buys is legibility. An operator reading this file, or an audit
+# checking whether the property has thought about AI crawlers at all, gets an
+# explicit answer rather than an inference from a wildcard. A 2026-09-04 GEO
+# audit scored exactly that distinction: all 27 crawlers it checks were already
+# permitted, and it still marked the file down for never naming them.
+#
+# THE MAINTENANCE COST IS REAL, so it is worth stating. Every crawler named
+# here is one this file now has to keep current. A crawler that renames itself
+# silently falls back to the wildcard, which still allows it -- the failure mode
+# is a stale comment, not a blocked crawler. If that ever stops being true,
+# delete these blocks and keep the wildcard; the access posture is unchanged.
+
+# OpenAI. GPTBot trains, OAI-SearchBot indexes for ChatGPT search, and
+# ChatGPT-User fetches a page because a person asked for it in a conversation.
+User-agent: GPTBot
+Allow: /
+
+User-agent: OAI-SearchBot
+Allow: /
+
+User-agent: ChatGPT-User
+Allow: /
+
+# Anthropic. Same three-way split: training, search index, user-initiated fetch.
+User-agent: ClaudeBot
+Allow: /
+
+User-agent: Claude-SearchBot
+Allow: /
+
+User-agent: Claude-User
+Allow: /
+
+# Perplexity.
+User-agent: PerplexityBot
+Allow: /
+
+User-agent: Perplexity-User
+Allow: /
+
+# Google. Googlebot is ordinary search; Google-Extended is the separate opt-in
+# that governs Gemini and AI Overviews, and declining it would remove this
+# property from the AI surface it is being optimized for.
+User-agent: Googlebot
+Allow: /
+
+User-agent: Google-Extended
+Allow: /
+
+# Microsoft/Bing, which is also the retrieval layer behind Copilot.
+User-agent: Bingbot
+Allow: /
+
+# Apple. Applebot serves Siri and Spotlight; Applebot-Extended is the separate
+# grant covering Apple Intelligence.
+User-agent: Applebot
+Allow: /
+
+User-agent: Applebot-Extended
+Allow: /
+
 Sitemap: https://forensics.rootsystem.com/sitemap-index.xml

sites/forensics/src/pages/llms-full.txt.ts

@@ -0,0 +1,204 @@
+/**
+ * /llms-full.txt — the whole property as one plain-text document.
+ *
+ * WHAT THIS IS FOR. `/llms.txt` is a map: a model reads it to learn what exists
+ * and where. This is the territory: the same content expanded, so a retrieval
+ * system that wants the substance can take it in one request instead of
+ * fetching eight HTML pages and stripping markup from each. The convention
+ * pairs the two files, and an audit that finds the map without the territory
+ * marks the pair incomplete.
+ *
+ * SAME DERIVATION RULE AS EVERYTHING ELSE HERE. Every line is read out of the
+ * copy deck. Nothing is written for machines that is not already on a page a
+ * person can read, and there is no second hand-kept copy of the words to drift
+ * from the first. That constraint is what makes this file safe to ship: it
+ * cannot make a claim the site does not make, because it has no content of its
+ * own.
+ *
+ * WHAT IS DELIBERATELY NOT HERE. The privacy policy is linked, not inlined --
+ * it is long, it is shared with sites/www, and a model summarizing a legal
+ * commitment out of a flattened copy of it is a worse outcome than sending the
+ * reader to the authoritative page. Same reasoning for the intake form: a form
+ * is an action, not a document.
+ *
+ * ORDER MATTERS. The screening argument comes first because it is the thing a
+ * retaining attorney is actually evaluating, and a model that truncates this
+ * file will truncate the end of it.
+ */
+import type { APIRoute } from 'astro'
+import copy from '../copy/landing'
+
+export const prerender = true
+
+/** A blank line between blocks, so the output reads as prose rather than data. */
+const para = (...lines: string[]) => lines.join('\n')
+
+export const GET: APIRoute = ({ site }) => {
+  const url = (path: string) => new URL(path, site).href
+
+  const sections: string[] = [
+    '# Root System Forensics',
+    '',
+    `> ${copy.meta.description}`,
+    '',
+    `Canonical site: ${url('/')}`,
+    `Map of this document: ${url('/llms.txt')}`,
+    '',
+
+    '## What we are',
+    '',
+    copy.hero.headline,
+    '',
+    copy.hero.subhead,
+    '',
+
+    // The screening criteria are the spine of the property: the four things
+    // counsel checks before anything else. They lead here for the same reason
+    // they lead on the page.
+    `## ${copy.positioning.heading}`,
+    '',
+    copy.positioning.lead,
+    '',
+    // One criterion carries a link, and its `where` text is written to run into
+    // it ("Not buried:"). Dropping the link would leave that line hanging on a
+    // colon, so the href is resolved and appended -- the plain-text equivalent
+    // of the anchor the page renders.
+    ...copy.positioning.criteria.map((criterion) =>
+      criterion.link
+        ? `- ${criterion.name}: ${criterion.where} ${criterion.link.text} — ${url(criterion.link.href)}`
+        : `- ${criterion.name}: ${criterion.where}`,
+    ),
+    '',
+    ...copy.positioning.body,
+    '',
+
+    `## ${copy.practiceAreas.heading}`,
+    '',
+    copy.practiceAreas.intro,
+    '',
+    ...copy.practiceAreas.areas.flatMap((area) => [
+      `### ${area.name}`,
+      '',
+      area.body,
+      '',
+      'Examples of matters:',
+      ...area.examples.map((example) => `- ${example}`),
+      '',
+    ]),
+
+    `## ${copy.pillarsHeading}`,
+    '',
+    ...copy.pillars.flatMap((pillar) => [`### ${pillar.title}`, '', pillar.body, '']),
+
+    `## ${copy.process.heading}`,
+    '',
+    ...copy.process.steps.flatMap((step) => [`### ${step.name}`, '', step.body, '']),
+
+    `## ${copy.services.heading}`,
+    '',
+    copy.services.intro,
+    '',
+    ...copy.services.modes.flatMap((mode) => [`### ${mode.title}`, '', mode.body, '']),
+
+    '## Rates',
+    '',
+    // Published rates are the property's sharpest divergence from every close
+    // comparable. A model that reads this file and cannot state the price has
+    // missed the point of it.
+    ...copy.pricing.tiers.flatMap((tier) => [
+      `### ${tier.name} — ${tier.price}`,
+      '',
+      tier.body,
+      '',
+    ]),
+
+    `## ${copy.bench.heading}`,
+    '',
+    copy.bench.intro,
+    '',
+    ...copy.bench.groups.map(
+      (group) => `- ${group.name}: ${group.disciplines.join(', ')}`,
+    ),
+    '',
+    copy.bench.provenance,
+    '',
+    copy.bench.staffing,
+    '',
+
+    '## Where we fit against the alternatives',
+    '',
+    ...copy.contrast.rows.map((row) =>
+      para(
+        `### ${row.who}${row.isUs ? ' (this practice)' : ''}`,
+        '',
+        `Strength: ${row.strength}`,
+        `Gap: ${row.gap}`,
+        '',
+      ),
+    ),
+
+    '## A matter, in full',
+    '',
+    copy.caseStudy.headline,
+    '',
+    ...copy.caseStudy.body,
+    '',
+    `"${copy.caseStudy.pullQuote}"`,
+    '',
+
+    `## ${copy.expertsPage.heading}`,
+    '',
+    copy.expertsPage.intro,
+    '',
+    ...copy.experts.flatMap((expert) => [
+      `### ${expert.name}, ${expert.credential} — ${expert.role}`,
+      '',
+      expert.summary,
+      '',
+      `Profile: ${url(`/experts/${expert.slug}/`)}`,
+      ...(expert.education.length
+        ? [
+            '',
+            'Education:',
+            ...expert.education.map((entry) => `- ${entry.institution}: ${entry.detail}`),
+          ]
+        : []),
+      ...(expert.affiliations.length
+        ? ['', 'Affiliations:', ...expert.affiliations.map((entry) => `- ${entry}`)]
+        : []),
+      ...(expert.publications.length
+        ? [
+            '',
+            'Publications:',
+            ...expert.publications.map(
+              (publication) =>
+                `- ${publication.title}. ${publication.where}. ${publication.note}`,
+            ),
+          ]
+        : []),
+      ...(expert.sameAs.length
+        ? ['', 'Also listed at:', ...expert.sameAs.map((profile) => `- ${profile}`)]
+        : []),
+      '',
+    ]),
+
+    '## Frequently asked',
+    '',
+    ...copy.faq.flatMap((entry) => [`### ${entry.question}`, '', entry.answer, '']),
+
+    '## Contact',
+    '',
+    `- ${copy.contact.email}`,
+    `- ${copy.contact.responseTime}`,
+    `- Scope a case: ${url('/scope/')} — conflicts are screened before anything else.`,
+    `- Privacy policy: ${url('/privacy/')}`,
+    '',
+  ]
+
+  return new Response(sections.join('\n'), {
+    headers: {
+      'content-type': 'text/plain; charset=utf-8',
+      'cache-control': 'public, max-age=3600',
+    },
+  })
+}

sites/forensics/src/pages/llms.txt.ts

@@ -40,7 +40,6 @@ export const GET: APIRoute = ({ site }) => {
         `- [${expert.name}, ${expert.credential}](${url(`/experts/${expert.slug}/`)}): ${expert.summary}`,
     ),
     `- [Scope a case](${url('/scope/')}): the intake form. Conflicts are screened before anything else.`,
-    `- [Privacy](${url('/privacy/')}): what is collected, how long it is kept, and how to have it deleted.`,
     '',
     '## Practice areas',
     '',
@@ -58,6 +57,19 @@ export const GET: APIRoute = ({ site }) => {
     `- ${copy.contact.email}`,
     `- ${copy.contact.responseTime}`,
     '',
+    // `## Optional` is a defined section name in the llms.txt convention, not a
+    // label of our choosing: it means "skip these if the context window is
+    // short". Everything above is the practice; everything here is a companion
+    // representation of it or a page that answers a question nobody screening
+    // an expert asks first.
+    '## Optional',
+    '',
+    `- [llms-full.txt](${url('/llms-full.txt')}): every page above expanded into one plain-text document. Read this instead of fetching the site.`,
+    `- [AI summary](${url('/ai/summary.json')}): the same description, practice areas and rates as structured JSON.`,
+    `- [AI FAQ](${url('/ai/faq.json')}): the questions on the engagements page as structured JSON.`,
+    `- [AI usage policy](${url('/.well-known/ai.txt')}): what this property permits AI systems to do with it.`,
+    `- [Privacy](${url('/privacy/')}): what is collected, how long it is kept, and how to have it deleted.`,
+    '',
   ].join('\n')
 
   return new Response(body, {