diff --git a/sites/forensics/public/robots.txt b/sites/forensics/public/robots.txt index 2bbc022..ce01d5e 100644 --- a/sites/forensics/public/robots.txt +++ b/sites/forensics/public/robots.txt @@ -7,4 +7,75 @@ User-agent: * Allow: / +# The answer-engine crawlers are named individually below, and every block says +# exactly what the wildcard above already said. +# +# WHY REPEAT YOURSELF. Under the robots.txt matching rules a crawler obeys the +# most specific group that names it and ignores every other group, the wildcard +# included. So a named block is not additive -- it REPLACES `User-agent: *` for +# that crawler, which means each one has to be complete on its own. They are: +# one `Allow: /` each, identical to the wildcard. Nothing here grants or +# withdraws access that was not already granted. +# +# What it buys is legibility. An operator reading this file, or an audit +# checking whether the property has thought about AI crawlers at all, gets an +# explicit answer rather than an inference from a wildcard. A 2026-09-04 GEO +# audit scored exactly that distinction: all 27 crawlers it checks were already +# permitted, and it still marked the file down for never naming them. +# +# THE MAINTENANCE COST IS REAL, so it is worth stating. Every crawler named +# here is one this file now has to keep current. A crawler that renames itself +# silently falls back to the wildcard, which still allows it -- the failure mode +# is a stale comment, not a blocked crawler. If that ever stops being true, +# delete these blocks and keep the wildcard; the access posture is unchanged. + +# OpenAI. GPTBot trains, OAI-SearchBot indexes for ChatGPT search, and +# ChatGPT-User fetches a page because a person asked for it in a conversation. +User-agent: GPTBot +Allow: / + +User-agent: OAI-SearchBot +Allow: / + +User-agent: ChatGPT-User +Allow: / + +# Anthropic. Same three-way split: training, search index, user-initiated fetch. +User-agent: ClaudeBot +Allow: / + +User-agent: Claude-SearchBot +Allow: / + +User-agent: Claude-User +Allow: / + +# Perplexity. +User-agent: PerplexityBot +Allow: / + +User-agent: Perplexity-User +Allow: / + +# Google. Googlebot is ordinary search; Google-Extended is the separate opt-in +# that governs Gemini and AI Overviews, and declining it would remove this +# property from the AI surface it is being optimized for. +User-agent: Googlebot +Allow: / + +User-agent: Google-Extended +Allow: / + +# Microsoft/Bing, which is also the retrieval layer behind Copilot. +User-agent: Bingbot +Allow: / + +# Apple. Applebot serves Siri and Spotlight; Applebot-Extended is the separate +# grant covering Apple Intelligence. +User-agent: Applebot +Allow: / + +User-agent: Applebot-Extended +Allow: / + Sitemap: https://forensics.rootsystem.com/sitemap-index.xml diff --git a/sites/forensics/src/pages/llms-full.txt.ts b/sites/forensics/src/pages/llms-full.txt.ts new file mode 100644 index 0000000..a8cc620 --- /dev/null +++ b/sites/forensics/src/pages/llms-full.txt.ts @@ -0,0 +1,204 @@ +/** + * /llms-full.txt — the whole property as one plain-text document. + * + * WHAT THIS IS FOR. `/llms.txt` is a map: a model reads it to learn what exists + * and where. This is the territory: the same content expanded, so a retrieval + * system that wants the substance can take it in one request instead of + * fetching eight HTML pages and stripping markup from each. The convention + * pairs the two files, and an audit that finds the map without the territory + * marks the pair incomplete. + * + * SAME DERIVATION RULE AS EVERYTHING ELSE HERE. Every line is read out of the + * copy deck. Nothing is written for machines that is not already on a page a + * person can read, and there is no second hand-kept copy of the words to drift + * from the first. That constraint is what makes this file safe to ship: it + * cannot make a claim the site does not make, because it has no content of its + * own. + * + * WHAT IS DELIBERATELY NOT HERE. The privacy policy is linked, not inlined -- + * it is long, it is shared with sites/www, and a model summarizing a legal + * commitment out of a flattened copy of it is a worse outcome than sending the + * reader to the authoritative page. Same reasoning for the intake form: a form + * is an action, not a document. + * + * ORDER MATTERS. The screening argument comes first because it is the thing a + * retaining attorney is actually evaluating, and a model that truncates this + * file will truncate the end of it. + */ +import type { APIRoute } from 'astro' +import copy from '../copy/landing' + +export const prerender = true + +/** A blank line between blocks, so the output reads as prose rather than data. */ +const para = (...lines: string[]) => lines.join('\n') + +export const GET: APIRoute = ({ site }) => { + const url = (path: string) => new URL(path, site).href + + const sections: string[] = [ + '# Root System Forensics', + '', + `> ${copy.meta.description}`, + '', + `Canonical site: ${url('/')}`, + `Map of this document: ${url('/llms.txt')}`, + '', + + '## What we are', + '', + copy.hero.headline, + '', + copy.hero.subhead, + '', + + // The screening criteria are the spine of the property: the four things + // counsel checks before anything else. They lead here for the same reason + // they lead on the page. + `## ${copy.positioning.heading}`, + '', + copy.positioning.lead, + '', + // One criterion carries a link, and its `where` text is written to run into + // it ("Not buried:"). Dropping the link would leave that line hanging on a + // colon, so the href is resolved and appended -- the plain-text equivalent + // of the anchor the page renders. + ...copy.positioning.criteria.map((criterion) => + criterion.link + ? `- ${criterion.name}: ${criterion.where} ${criterion.link.text} — ${url(criterion.link.href)}` + : `- ${criterion.name}: ${criterion.where}`, + ), + '', + ...copy.positioning.body, + '', + + `## ${copy.practiceAreas.heading}`, + '', + copy.practiceAreas.intro, + '', + ...copy.practiceAreas.areas.flatMap((area) => [ + `### ${area.name}`, + '', + area.body, + '', + 'Examples of matters:', + ...area.examples.map((example) => `- ${example}`), + '', + ]), + + `## ${copy.pillarsHeading}`, + '', + ...copy.pillars.flatMap((pillar) => [`### ${pillar.title}`, '', pillar.body, '']), + + `## ${copy.process.heading}`, + '', + ...copy.process.steps.flatMap((step) => [`### ${step.name}`, '', step.body, '']), + + `## ${copy.services.heading}`, + '', + copy.services.intro, + '', + ...copy.services.modes.flatMap((mode) => [`### ${mode.title}`, '', mode.body, '']), + + '## Rates', + '', + // Published rates are the property's sharpest divergence from every close + // comparable. A model that reads this file and cannot state the price has + // missed the point of it. + ...copy.pricing.tiers.flatMap((tier) => [ + `### ${tier.name} — ${tier.price}`, + '', + tier.body, + '', + ]), + + `## ${copy.bench.heading}`, + '', + copy.bench.intro, + '', + ...copy.bench.groups.map( + (group) => `- ${group.name}: ${group.disciplines.join(', ')}`, + ), + '', + copy.bench.provenance, + '', + copy.bench.staffing, + '', + + '## Where we fit against the alternatives', + '', + ...copy.contrast.rows.map((row) => + para( + `### ${row.who}${row.isUs ? ' (this practice)' : ''}`, + '', + `Strength: ${row.strength}`, + `Gap: ${row.gap}`, + '', + ), + ), + + '## A matter, in full', + '', + copy.caseStudy.headline, + '', + ...copy.caseStudy.body, + '', + `"${copy.caseStudy.pullQuote}"`, + '', + + `## ${copy.expertsPage.heading}`, + '', + copy.expertsPage.intro, + '', + ...copy.experts.flatMap((expert) => [ + `### ${expert.name}, ${expert.credential} — ${expert.role}`, + '', + expert.summary, + '', + `Profile: ${url(`/experts/${expert.slug}/`)}`, + ...(expert.education.length + ? [ + '', + 'Education:', + ...expert.education.map((entry) => `- ${entry.institution}: ${entry.detail}`), + ] + : []), + ...(expert.affiliations.length + ? ['', 'Affiliations:', ...expert.affiliations.map((entry) => `- ${entry}`)] + : []), + ...(expert.publications.length + ? [ + '', + 'Publications:', + ...expert.publications.map( + (publication) => + `- ${publication.title}. ${publication.where}. ${publication.note}`, + ), + ] + : []), + ...(expert.sameAs.length + ? ['', 'Also listed at:', ...expert.sameAs.map((profile) => `- ${profile}`)] + : []), + '', + ]), + + '## Frequently asked', + '', + ...copy.faq.flatMap((entry) => [`### ${entry.question}`, '', entry.answer, '']), + + '## Contact', + '', + `- ${copy.contact.email}`, + `- ${copy.contact.responseTime}`, + `- Scope a case: ${url('/scope/')} — conflicts are screened before anything else.`, + `- Privacy policy: ${url('/privacy/')}`, + '', + ] + + return new Response(sections.join('\n'), { + headers: { + 'content-type': 'text/plain; charset=utf-8', + 'cache-control': 'public, max-age=3600', + }, + }) +} diff --git a/sites/forensics/src/pages/llms.txt.ts b/sites/forensics/src/pages/llms.txt.ts index d0b99ea..1317bff 100644 --- a/sites/forensics/src/pages/llms.txt.ts +++ b/sites/forensics/src/pages/llms.txt.ts @@ -40,7 +40,6 @@ export const GET: APIRoute = ({ site }) => { `- [${expert.name}, ${expert.credential}](${url(`/experts/${expert.slug}/`)}): ${expert.summary}`, ), `- [Scope a case](${url('/scope/')}): the intake form. Conflicts are screened before anything else.`, - `- [Privacy](${url('/privacy/')}): what is collected, how long it is kept, and how to have it deleted.`, '', '## Practice areas', '', @@ -58,6 +57,19 @@ export const GET: APIRoute = ({ site }) => { `- ${copy.contact.email}`, `- ${copy.contact.responseTime}`, '', + // `## Optional` is a defined section name in the llms.txt convention, not a + // label of our choosing: it means "skip these if the context window is + // short". Everything above is the practice; everything here is a companion + // representation of it or a page that answers a question nobody screening + // an expert asks first. + '## Optional', + '', + `- [llms-full.txt](${url('/llms-full.txt')}): every page above expanded into one plain-text document. Read this instead of fetching the site.`, + `- [AI summary](${url('/ai/summary.json')}): the same description, practice areas and rates as structured JSON.`, + `- [AI FAQ](${url('/ai/faq.json')}): the questions on the engagements page as structured JSON.`, + `- [AI usage policy](${url('/.well-known/ai.txt')}): what this property permits AI systems to do with it.`, + `- [Privacy](${url('/privacy/')}): what is collected, how long it is kept, and how to have it deleted.`, + '', ].join('\n') return new Response(body, {