diff --git a/apps/web/AGENTS.md b/apps/web/AGENTS.md index 52a7806..797b306 100644 --- a/apps/web/AGENTS.md +++ b/apps/web/AGENTS.md @@ -1,6 +1,7 @@ # Contents -- `src/` contains the static `atet.sh` homepage and documentation, visual system, appearance control, crawler files, favicons, and social preview. +- `src/` contains the static `atet.sh` homepage and documentation, visual system, appearance control, crawler files, favicons, social preview, and machine-readable page bodies. +- `src/negotiate.ts`, `src/negotiate-request.ts`, and `middleware.ts` select HTML or markdown from `Accept` for document routes. - `scripts/build.ts` renders fingerprinted local assets into `dist/` from an explicit allowlist and bundles the pinned PostHog browser client only for a configured Production build. - `site.test.ts`, `package.json`, and `vercel.json` define the content, identity, accessibility, performance, legacy-host, and deployment contracts. @@ -16,5 +17,7 @@ - Keep every page semantic, keyboard-operable, readable at 200% zoom, and free of remote fonts and client frameworks. Analytics may emit one anonymous cookieless `$pageview` from `https://atet.sh/` to `https://us.i.posthog.com`, tagged with `site_id=atet` and `analytics_schema_version=1`. Keep persons, persistence, autocapture, replay, flags, surveys, heatmaps, pageleave, web vitals, referrer, URL, query, hash, page text, content, and custom events disabled. Do not initialize analytics on Preview, staging, alternate hosts, or `404.html`. - Organize `/docs` by user intent: guided learning, goal-oriented how-to, factual reference, and conceptual explanation. Do not mix those modes into one undifferentiated command catalog. - Use the canonical Hraness footer lockup. Include every durable public route in crawler discovery. +- Publish `/llms.txt` as `text/plain` with an H1, summary, and a when-to-use section. Publish `/sitemap.md` and `/index.md` as markdown. Honor `Accept: text/markdown` on the homepage with `Content-Type: text/markdown; charset=utf-8` and `Vary: Accept`. Return `406` when the request rejects every produced type. Keep unknown routes as real `404` or `410` responses, and give markdown 404s recovery links to the home page, `llms.txt`, and sitemaps. +- Do not add a public API, OAuth, GraphQL, MCP, account, or commerce surface to the website. - Preserve permanent production and preview redirects for every reviewed predecessor host without redirecting canonical Atet hosts. - Run `bun run check` in this directory after a site change. diff --git a/apps/web/README.md b/apps/web/README.md index 3a78760..5971b7f 100644 --- a/apps/web/README.md +++ b/apps/web/README.md @@ -3,6 +3,7 @@ `atet.sh` is the static public site and documentation for Atet. It presents the SDK, Bun CLI, local runtime, and desktop capture shell without adding a server, account surface, API route, remote font, or browser credential path. +Agents can read `/llms.txt` and request `Accept: text/markdown` on the homepage. Generation runs from the local Atet SDK or CLI with the operator's Vercel AI Gateway access. diff --git a/apps/web/middleware.ts b/apps/web/middleware.ts new file mode 100644 index 0000000..45af047 --- /dev/null +++ b/apps/web/middleware.ts @@ -0,0 +1,12 @@ +import { negotiateSiteRequest } from "./src/negotiate-request" + +export default function middleware(request: Request): Response | undefined { + return negotiateSiteRequest(request) +} + +export const config = { + matcher: [ + "/", + "/((?!assets/).*)", + ], +} diff --git a/apps/web/scripts/build.ts b/apps/web/scripts/build.ts index 3a92e35..3db2d30 100644 --- a/apps/web/scripts/build.ts +++ b/apps/web/scripts/build.ts @@ -3,6 +3,8 @@ import { cp, mkdir, readFile, rm, stat, writeFile } from "node:fs/promises" import { basename, dirname, join } from "node:path" import { fileURLToPath } from "node:url" +import { homeMarkdown, llmsTxt, robotsTxt, sitemapMarkdown } from "../src/agent-pages" + const appDirectory = dirname(dirname(fileURLToPath(import.meta.url))) const sourceDirectory = join(appDirectory, "src") const defaultOutputDirectory = join(appDirectory, "dist") @@ -16,10 +18,16 @@ const copiedFiles = [ "apple-touch-icon.png", "icon.svg", "og.png", - "robots.txt", "sitemap.xml", ] as const +const generatedTextFiles = { + "index.md": homeMarkdown, + "llms.txt": llmsTxt, + "robots.txt": robotsTxt, + "sitemap.md": sitemapMarkdown, +} as const + function assetPath(name: string, bytes: Uint8Array): string { const digest = createHash("sha256").update(bytes).digest("hex").slice(0, 12) const extensionIndex = name.lastIndexOf(".") @@ -242,11 +250,18 @@ export async function buildWebsite(options: BuildOptions = {}): Promise ( + writeFile(join(outputDirectory, file), contents) + ))) + return { analyticsPath, stylesPath, themePath } } if (import.meta.main) { const result = await buildWebsite() - const generatedFiles = copiedFiles.length + 4 + (result.analyticsPath === null ? 0 : 1) + const generatedFiles = copiedFiles.length + + Object.keys(generatedTextFiles).length + + 4 + + (result.analyticsPath === null ? 0 : 1) console.log(`Built ${generatedFiles} static files in ${defaultOutputDirectory}`) } diff --git a/apps/web/site.test.ts b/apps/web/site.test.ts index de207c1..cdc5111 100644 --- a/apps/web/site.test.ts +++ b/apps/web/site.test.ts @@ -9,6 +9,21 @@ import { posthogCookielessDistinctId, sanitizePageview, } from "./src/analytics-contract" +import { + homeMarkdown, + llmsTxt, + notFoundMarkdown, + robotsTxt, + sitemapMarkdown, +} from "./src/agent-pages" +import { notAcceptableBody, preferredRepresentation } from "./src/negotiate" +import { + isHomePath, + isNegotiableDocumentPath, + isPreservedRedirectPath, + negotiateSiteRequest, +} from "./src/negotiate-request" +import middleware, { config as middlewareConfig } from "./middleware" import { buildWebsite } from "./scripts/build" const appDirectory = dirname(fileURLToPath(import.meta.url)) @@ -120,6 +135,8 @@ describe("static Atet site", () => { expect(html).toContain('') expect(html).not.toContain('') + expect(html).toContain('') + expect(html).toContain('') expect(html).toContain('') expect(html).toContain('') expect(html).toContain('') @@ -353,6 +370,10 @@ describe("static Atet site", () => { expect(notFound).toContain('
') expect(notFound).toContain('') expect(notFound).toContain('') + expect(notFound).toContain('href="/llms.txt"') + expect(notFound).toContain('href="/sitemap.md"') + expect(notFound).toContain('href="/sitemap.xml"') + expect(notFound).toContain("machine-readable site guide") }) test("uses a restrained editorial visual system", async () => { @@ -600,8 +621,11 @@ describe("static Atet site", () => { "assets", "icon.svg", "index.html", + "index.md", + "llms.txt", "og.png", "robots.txt", + "sitemap.md", "sitemap.xml", ]) expect(assetFiles.sort()).toEqual([ @@ -621,21 +645,38 @@ describe("static Atet site", () => { expect(themeAsset).not.toMatch(/fetch\(|XMLHttpRequest|WebSocket|EventSource|sendBeacon/) }) - test("publishes only the canonical page to crawler discovery", async () => { - const robots = await readSource("robots.txt") - const sitemap = await readSource("sitemap.xml") - const notFound = await readSource("404.html") + test("publishes crawler discovery for the home page and its markdown mirror", async () => { + const [robots, sitemap, notFound, builtRobots, builtLlms, builtHomeMarkdown, builtSitemapMarkdown] = await Promise.all([ + Promise.resolve(robotsTxt), + readSource("sitemap.xml"), + readSource("404.html"), + readBuilt("robots.txt"), + readBuilt("llms.txt"), + readBuilt("index.md"), + readBuilt("sitemap.md"), + ]) const locations = [...sitemap.matchAll(/([^<]+)<\/loc>/g)] .map(match => match[1]) expect(robots).toBe([ "User-agent: OAI-SearchBot", + "User-agent: ChatGPT-User", + "User-agent: GPTBot", "Allow: /", "", "User-agent: Claude-SearchBot", + "User-agent: Claude-User", + "User-agent: ClaudeBot", "Allow: /", "", - "User-agent: Claude-User", + "User-agent: PerplexityBot", + "User-agent: Perplexity-User", + "Allow: /", + "", + "User-agent: Google-Extended", + "Allow: /", + "", + "User-agent: CCBot", "Allow: /", "", "User-agent: *", @@ -644,8 +685,24 @@ describe("static Atet site", () => { "Sitemap: https://atet.sh/sitemap.xml", "", ].join("\n")) - expect(locations).toEqual(["https://atet.sh/"]) + expect(robots).not.toMatch(/^\s*Disallow:/mu) + expect(builtRobots).toBe(robotsTxt) + expect(locations).toEqual(["https://atet.sh/", "https://atet.sh/index.md"]) + expect(sitemap).toContain("2026-08-21") expect(notFound).toContain('') + expect(builtLlms).toBe(llmsTxt) + expect(builtHomeMarkdown).toBe(homeMarkdown) + expect(builtSitemapMarkdown).toBe(sitemapMarkdown) + expect(llmsTxt).toMatch(/^# Atet\n/u) + expect(llmsTxt).toContain("> Atet gives coding agents tools") + expect(llmsTxt).toContain("## When to use Atet") + expect(llmsTxt).toContain("https://atet.sh/index.md") + expect(sitemapMarkdown).toContain("# Sitemap") + expect(sitemapMarkdown).toContain("https://atet.sh/index.md") + expect(homeMarkdown).toContain("## Sitemap") + expect(homeMarkdown).toContain("https://atet.sh/sitemap.md") + expect(notFoundMarkdown).toContain("https://atet.sh/llms.txt") + expect(notFoundMarkdown).toContain("https://atet.sh/sitemap.xml") }) test("redirects the retired docs route and each reviewed predecessor host", async () => { @@ -705,6 +762,11 @@ describe("static Atet site", () => { await readFile(join(appDirectory, "vercel.json"), "utf8"), ) as { headers?: Array<{ source?: string; headers?: Array<{ key?: string; value?: string }> }> + rewrites?: Array<{ + source?: string + destination?: string + has?: Array<{ type?: string; key?: string; value?: string }> + }> } const global = vercel.headers?.find(entry => entry.source === "/(.*)")?.headers ?? [] const assets = vercel.headers?.find(entry => entry.source === "/assets/(.*)")?.headers ?? [] @@ -718,10 +780,34 @@ describe("static Atet site", () => { expect(csp).toContain("object-src 'none'") expect(byKey.get("Referrer-Policy")).toBe("no-referrer") expect(byKey.get("Strict-Transport-Security")).toContain("includeSubDomains") + expect(byKey.get("Vary")).toBe("Accept, Accept-Encoding") expect(assets).toContainEqual({ key: "Cache-Control", value: "public, max-age=31536000, immutable", }) + + const home = vercel.headers?.find(entry => entry.source === "/")?.headers ?? [] + const markdown = vercel.headers?.find(entry => entry.source === "/index.md")?.headers ?? [] + const llms = vercel.headers?.find(entry => entry.source === "/llms.txt")?.headers ?? [] + expect(home).toContainEqual({ + key: "Link", + value: '; rel="alternate"; type="text/markdown", ; rel="describedby"', + }) + expect(markdown).toContainEqual({ + key: "Content-Type", + value: "text/markdown; charset=utf-8", + }) + expect(llms).toContainEqual({ + key: "Content-Type", + value: "text/plain; charset=utf-8", + }) + expect(vercel.rewrites).toEqual([ + { + source: "/", + has: [{ type: "header", key: "accept", value: "^text/markdown" }], + destination: "/index.md", + }, + ]) }) test("preserves the canonical Hraness footer", async () => { @@ -732,4 +818,83 @@ describe("static Atet site", () => { expect(html).toContain('class="hraness-mark"') expect(html).toContain("Atet · MIT · AI media generation and video editing for coding agents.") }) + + test("selects markdown, HTML, and 406 from Accept quality values", () => { + expect(preferredRepresentation(null)).toBe("text/html") + expect(preferredRepresentation("")).toBe("text/html") + expect(preferredRepresentation("*/*")).toBe("text/html") + expect(preferredRepresentation("text/html")).toBe("text/html") + expect(preferredRepresentation("text/markdown")).toBe("text/markdown") + expect(preferredRepresentation("text/markdown, text/html, */*")).toBe("text/markdown") + expect(preferredRepresentation("text/html, text/markdown;q=0.9")).toBe("text/html") + expect(preferredRepresentation("text/html;q=0, */*;q=1")).toBe("text/markdown") + expect(preferredRepresentation("text/markdown;q=0, text/html;q=0")).toBeNull() + expect(preferredRepresentation("application/xml")).toBeNull() + expect(preferredRepresentation("application/json, image/png")).toBeNull() + }) + + test("negotiates homepage markdown, agent-friendly 404s, and 406 without an API route", async () => { + expect(isHomePath("/")).toBe(true) + expect(isHomePath("/index.html")).toBe(true) + expect(isPreservedRedirectPath("/docs")).toBe(true) + expect(isPreservedRedirectPath("/docs/install")).toBe(true) + expect(isNegotiableDocumentPath("/missing-route")).toBe(true) + expect(isNegotiableDocumentPath("/llms.txt")).toBe(false) + expect(isNegotiableDocumentPath("/index.md")).toBe(false) + expect(isNegotiableDocumentPath("/assets/styles.css")).toBe(false) + + const markdownHome = negotiateSiteRequest(new Request("https://atet.sh/", { + headers: { Accept: "text/markdown" }, + })) + expect(markdownHome).toBeDefined() + expect(markdownHome?.status).toBe(200) + expect(markdownHome?.headers.get("content-type")).toBe("text/markdown; charset=utf-8") + expect(markdownHome?.headers.get("vary")).toBe("Accept, Accept-Encoding") + expect(markdownHome?.headers.get("link")).toContain('rel="canonical"') + expect(await markdownHome?.text()).toBe(homeMarkdown) + + const htmlHome = negotiateSiteRequest(new Request("https://atet.sh/", { + headers: { Accept: "text/html" }, + })) + expect(htmlHome).toBeUndefined() + + const docsRedirect = negotiateSiteRequest(new Request("https://atet.sh/docs", { + headers: { Accept: "text/markdown" }, + })) + expect(docsRedirect).toBeUndefined() + + const markdownNotFound = negotiateSiteRequest(new Request("https://atet.sh/this-path-does-not-exist", { + headers: { Accept: "text/markdown" }, + })) + expect(markdownNotFound?.status).toBe(404) + expect(markdownNotFound?.headers.get("content-type")).toBe("text/markdown; charset=utf-8") + expect(markdownNotFound?.headers.get("vary")).toBe("Accept, Accept-Encoding") + expect(markdownNotFound?.headers.get("x-robots-tag")).toBe("noindex") + expect(await markdownNotFound?.text()).toBe(notFoundMarkdown) + + const notAcceptable = negotiateSiteRequest(new Request("https://atet.sh/", { + headers: { Accept: "application/xml" }, + })) + expect(notAcceptable?.status).toBe(406) + expect(notAcceptable?.headers.get("content-type")).toBe("text/plain; charset=utf-8") + expect(notAcceptable?.headers.get("vary")).toBe("Accept") + expect(await notAcceptable?.text()).toBe(notAcceptableBody) + + const staticFile = negotiateSiteRequest(new Request("https://atet.sh/llms.txt", { + headers: { Accept: "application/xml" }, + })) + expect(staticFile).toBeUndefined() + + const html = await readSource("index.html") + const build = await readFile(join(appDirectory, "scripts/build.ts"), "utf8") + expect(html).not.toMatch(/\/api\//) + expect(build).not.toMatch(/\/api\//) + expect(build).toContain('writeFile(join(outputDirectory, file), contents)') + expect(middlewareConfig.matcher).toContain("/") + const middlewareMarkdown = middleware(new Request("https://atet.sh/", { + headers: { Accept: "text/markdown" }, + })) + expect(middlewareMarkdown?.status).toBe(200) + expect(await middlewareMarkdown?.text()).toBe(homeMarkdown) + }) }) diff --git a/apps/web/src/404.html b/apps/web/src/404.html index 7a1a61e..75dc6e8 100644 --- a/apps/web/src/404.html +++ b/apps/web/src/404.html @@ -29,6 +29,12 @@

404 · Route not found

This passage ends here.

The address may have changed. Atet and its current installation guide remain at the canonical home.

+

+ Recover from the home page, the + machine-readable site guide, the + markdown sitemap, or the + XML sitemap. +

Return to atet.sh