feat: serve markdown to agents, and apply the _headers file sites ship

README.md

@@ -16,6 +16,48 @@ All configuration is via environment variables:
 
 > **`DEFAULT_EXTENSION` + `SPA_MODE`:** path resolution only tries one extension. If you set `DEFAULT_EXTENSION=.json`, a request for `/docs` looks for `docs.json` only — `docs.html` will not be found, and with `SPA_MODE=true` the request falls through to `index.html`. Leave `DEFAULT_EXTENSION=.html` unless every extensionless route on your site resolves to the same non-html file type.
 
+> A variable that is *set but empty* counts as unset and falls back to its default. The shipped `configs/.env` declares `STATIC_DIR_PATH=` and `DEFAULT_EXTENSION=` with empty values, so supplying them through the environment would otherwise resolve to `""`.
+
+## Content Negotiation
+
+A request whose `Accept` header names `text/markdown` is served the markdown sibling of the page, when one exists: `/about` returns `about.md` instead of `about/index.html`. Static site generators already emit these files, so **no build change is needed** — if the `.md` isn't there, the request falls through to HTML untouched.
+
+This exists because agents otherwise have to download the whole HTML document to discover the `<link rel="alternate">` inside it. Measured on a real site, the markdown ran 15–96× smaller than the page it replaces.
+
+| Behavior | Detail |
+|---|---|
+| What counts as asking | The media type must be **named**. Browsers send `*/*;q=0.8`, which matches `text/markdown` by the letter of RFC 9110 — matching wildcards would serve raw source to every visitor. `text/x-markdown` is accepted too. |
+| Preference is respected | `q`-values are honored: `text/html, text/markdown;q=0.1` still gets HTML. |
+| Which URLs negotiate | Extensionless routes only. `/` is always `index.html`, and a path with an extension (`/style.css`, `/about.md`) resolves the same way for every client. |
+| `Vary: Accept` | Set on responses that can depend on `Accept` — negotiable routes, the SPA fallback for them, and every miss. **Not** set on assets, so a CDN isn't asked to fragment its cache on a header that cannot change what it returns. A `Vary` your `_headers` declares is preserved, not replaced. |
+| Misses | A client that asked for markdown gets a small markdown 404 naming `/sitemap.xml` and `/llms.txt`, rather than an HTML error shell it cannot parse. |
+| `Content-Type` | Set explicitly for a **negotiated** response, since the distroless base image has no `/etc/mime.types`. A directly requested `.md` is left as-is, so existing links to `.md` files behave exactly as before. |
+
+## Response Headers (`_headers`)
+
+If the published directory contains a `_headers` file — the [Netlify](https://docs.netlify.com/routing/headers/) / Cloudflare Pages convention — its rules are parsed once at startup and applied to matching responses. **No file means no rules and no change**, so this is inert for sites that don't ship one.
+
+```
+/*
+  X-Frame-Options: DENY
+  X-Content-Type-Options: nosniff
+
+/_astro/*
+  Cache-Control: public, max-age=31536000, immutable
+```
+
+| Behavior | Detail |
+|---|---|
+| Matching | `*` matches any run of characters, including `/`. Patterns are anchored, so `/docs/*` does not match `/other/docs/x`. Netlify's `:placeholder` syntax is **not** supported. |
+| Precedence | Every matching rule contributes, in file order, so a later specific block overrides an earlier catch-all. |
+| Malformed lines | Skipped individually — one bad rule doesn't cost the file its other headers. |
+| Errors and misses | Rules apply to 404s and the SPA fallback too: a 404 that leaks framing protection is as exploitable as a 200 that does. **`Cache-Control` is the exception** — it is withdrawn from a miss, and from a delegated response that returns 4xx/5xx, so a `/*` cache rule cannot pin a file that is merely un-propagated mid-deploy. The SPA fallback keeps its caching, being a real route rather than a miss. |
+| `/.well-known/` | Delegated to the framework so ACME challenges resolve untouched, but the header rules still apply to it. |
+
+> **Reverse proxies win.** If something in front of this server sets the same header, its value is what reaches the client — ingress-nginx sends `Strict-Transport-Security` by default, for instance. Headers with no proxy counterpart take effect immediately.
+
+The startup log names the resolved directory alongside the rule count, since `0 rules` is normal for a site without the file and otherwise indistinguishable from a misrooted path.
+
 ## Usage
 
 ### Docker

handler.go

@@ -14,37 +14,137 @@ type staticFileHandler struct {
 	spaMode          bool
 	defaultExtension string
 	next             http.Handler
+
+	// Parsed once at startup from `_headers`; nil when the site has no such
+	// file, in which case nothing about the response changes.
+	headerRules headerRules
 }
 
 func (h *staticFileHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	// Applied before anything writes, so it covers hits, misses, the SPA
+	// fallback and the delegated paths below alike. Set first so the server's
+	// own headers further down still win where they are load-bearing.
+	h.headerRules.apply(w.Header(), r.URL.Path)
+
+	// .well-known is handed off untouched — ACME challenges and the like must
+	// not pick up an extension or the SPA fallback. The site's headers still
+	// apply to it: a `/*` block declaring X-Frame-Options means the whole site,
+	// and a path this server delegates is still a path it answers for.
+	//
+	// The status is not known here, because the delegate chooses it, so the
+	// cache directives are withdrawn on the way out instead of up front.
 	if strings.Contains(r.URL.Path, "/.well-known/") {
-		h.next.ServeHTTP(w, r)
+		h.next.ServeHTTP(&errorCacheScrubber{ResponseWriter: w}, r)
 
 		return
 	}
 
-	filePath, hasExtension := h.resolveFilePath(r.URL.Path)
+	wantsMarkdown := markdownPreferred(r.Header.Get("Accept"))
+
+	filePath, hasExtension := h.resolveFilePath(r.URL.Path, wantsMarkdown)
 
 	if _, err := h.fs.Stat(filePath); err != nil {
 		if h.spaMode && !hasExtension {
+			// The shell answers a route an agent may instead have been handed a
+			// .md sibling for: with `foo.md` on disk but no HTML page, markdown
+			// clients take the hit path above while browsers land here, so one
+			// URL yields two bodies. Saying so is what stops a shared cache
+			// handing the shell to the next client that asked for markdown.
+			if negotiable(r.URL.Path) {
+				advertiseAcceptVaries(w.Header())
+			}
+
 			http.ServeFile(w, r, filepath.Join(h.staticFilePath, indexHTML))
+
 			return
 		}
 
-		http.ServeFile(&statusOverrideWriter{ResponseWriter: w, status: http.StatusNotFound}, r,
-			filepath.Join(h.staticFilePath, "404.html"))
+		h.serveNotFound(w, r, wantsMarkdown)
 
 		return
 	}
 
+	// Only a negotiable route can resolve to a .md sibling, so only there can
+	// the body depend on Accept. Advertising Vary on everything else would
+	// fragment caches on a header that cannot change what they return — and
+	// those are exactly the hashed assets `_headers` marks immutable, so the
+	// cost would land on the responses this server most wants cached.
+	if negotiable(r.URL.Path) {
+		advertiseAcceptVaries(w.Header())
+	}
+
+	// http.ServeFile only sniffs a Content-Type when one is not already set,
+	// so setting it here wins. See labelAsMarkdown for why this is scoped to a
+	// negotiated response.
+	if labelAsMarkdown(wantsMarkdown, r.URL.Path, filePath) {
+		w.Header().Set("Content-Type", markdownContentType)
+	}
+
 	http.ServeFile(w, r, filePath)
 }
 
-func (h *staticFileHandler) resolveFilePath(urlPath string) (string, bool) {
+// serveNotFound answers a miss, in the type the client asked for.
+func (h *staticFileHandler) serveNotFound(w http.ResponseWriter, r *http.Request, wantsMarkdown bool) {
+	// A miss is answered in markdown whenever the client asked for it, so this
+	// response depends on Accept whatever the path looks like — including the
+	// extensions that never negotiate on a hit.
+	advertiseAcceptVaries(w.Header())
+
+	withdrawCacheDirectives(w.Header())
+
+	// A client that asked for markdown cannot use an HTML error shell — and
+	// those shells are not small. The 404 page of a real site measured 144 KB,
+	// sent in reply to a request the client could not parse. Answer in the type
+	// it asked for, at a size that suits an error.
+	if wantsMarkdown {
+		writeMarkdownNotFound(w)
+		return
+	}
+
+	http.ServeFile(&statusOverrideWriter{ResponseWriter: w, status: http.StatusNotFound}, r,
+		filepath.Join(h.staticFilePath, "404.html"))
+}
+
+// The requested path is deliberately not echoed back. Reflecting a
+// caller-controlled string into a response body is an injection sink even at
+// text/markdown, and the caller already knows which URL it asked for — the
+// recovery pointers are the part it does not have.
+const notFoundMarkdown = "# 404 Not Found\n\n" +
+	"The requested page does not exist on this server.\n\n" +
+	"See /sitemap.xml for the pages that do, or /llms.txt for an overview.\n"
+
+// writeMarkdownNotFound answers a miss in markdown and points the reader at
+// the two files that let it recover on its own rather than guessing at URLs.
+func writeMarkdownNotFound(w http.ResponseWriter) {
+	w.Header().Set("Content-Type", markdownContentType)
+	w.WriteHeader(http.StatusNotFound)
+
+	_, _ = w.Write([]byte(notFoundMarkdown))
+}
+
+func (h *staticFileHandler) resolveFilePath(urlPath string, wantsMarkdown bool) (string, bool) {
 	filePath := filepath.Join(h.staticFilePath, urlPath)
 
 	hasExtension := filepath.Ext(filePath) != ""
 
+	// Markdown content negotiation. Static site generators emit the markdown
+	// source of a page as a sibling of its directory index — `about.md` next
+	// to `about/index.html` — so when a client explicitly asks for markdown we
+	// can serve that file with no build changes and no extra round trip.
+	//
+	// This matters because the alternative costs the agent the whole HTML
+	// document first: it can only learn a .md exists by reading the
+	// <link rel="alternate"> inside the page it was trying to avoid
+	// downloading. On zop.dev the same page is 15-60x smaller as markdown.
+	//
+	// Falls through untouched when the client didn't ask or the file isn't
+	// there, so nothing an existing deployment serves today can change.
+	if wantsMarkdown && negotiable(urlPath) {
+		if _, err := h.fs.Stat(filePath + markdownExtension); err == nil {
+			return filePath + markdownExtension, true
+		}
+	}
+
 	if urlPath == rootPath {
 		filePath += indexHTML
 	} else if !hasExtension {
@@ -58,6 +158,41 @@ func (h *staticFileHandler) resolveFilePath(urlPath string) (string, bool) {
 	return filePath, hasExtension
 }
 
+// withdrawCacheDirectives removes the site's Cache-Control from a response that
+// turned out not to be a page it publishes.
+//
+// A `_headers` file describes what a site serves; an error is not that. Letting
+// a `/*.html` rule reach a miss would pin a file that is merely un-propagated
+// mid-deploy into every cache between this server and the reader for the rule's
+// full lifetime. The security headers still apply either way — a 404 that leaks
+// framing protection is as exploitable as a 200 that does.
+//
+// This is the one definition of that rule; both the miss path and the delegated
+// .well-known path go through it, so they cannot drift apart.
+func withdrawCacheDirectives(header http.Header) {
+	header.Del("Cache-Control")
+}
+
+// errorCacheScrubber applies withdrawCacheDirectives to a response whose status
+// is chosen by a handler this server delegated to, and is therefore not known
+// when the `_headers` rules are set.
+//
+// Deliberately not used on the main serving path: wrapping the writer there
+// would hide net/http's io.ReaderFrom from http.ServeFile and cost every static
+// file its sendfile fast path. The delegated paths are ACME challenges and the
+// like — small, rare, and not worth a special case to keep fast.
+type errorCacheScrubber struct {
+	http.ResponseWriter
+}
+
+func (w *errorCacheScrubber) WriteHeader(status int) {
+	if status >= http.StatusBadRequest {
+		withdrawCacheDirectives(w.Header())
+	}
+
+	w.ResponseWriter.WriteHeader(status)
+}
+
 type statusOverrideWriter struct {
 	http.ResponseWriter
 	status int

handler_test.go

@@ -65,7 +65,7 @@ func TestResolveFilePath(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			h := &staticFileHandler{fs: fs, staticFilePath: dir, defaultExtension: tt.defaultExtension}
 
-			path, hasExt := h.resolveFilePath(tt.urlPath)
+			path, hasExt := h.resolveFilePath(tt.urlPath, false)
 
 			assert.Equal(t, tt.wantPath, path)
 			assert.Equal(t, tt.wantHasExt, hasExt)

headers.go

@@ -0,0 +1,147 @@
+package main
+
+import (
+	"io"
+	"net/http"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"gofr.dev/pkg/gofr/datasource/file"
+)
+
+// headersFileName is the Netlify / Cloudflare Pages convention: a `_headers`
+// file at the root of the published directory, listing path patterns and the
+// response headers to send for them.
+//
+// Static site generators emit this file expecting the host to honor it. A
+// host that ignores it fails silently and in the worst way — the file looks
+// authoritative in the repo while the headers it declares were never sent. On
+// zop.dev the entire block (X-Frame-Options, X-Content-Type-Options,
+// Referrer-Policy, Permissions-Policy, and every Cache-Control rule including
+// `immutable` on hashed assets) had never once reached a browser.
+const headersFileName = "_headers"
+
+// headerPair is a single `Name: value` line.
+type headerPair struct {
+	name  string
+	value string
+}
+
+// headerRule is one block: a path pattern and the headers it contributes.
+type headerRule struct {
+	match   *regexp.Regexp
+	headers []headerPair
+}
+
+// headerRules is the parsed file, in source order.
+type headerRules []headerRule
+
+// apply writes every header whose pattern matches urlPath. All matching rules
+// contribute, in file order, so a later specific block overrides an earlier
+// catch-all — the same precedence the format has on Netlify and Cloudflare.
+func (r headerRules) apply(header http.Header, urlPath string) {
+	for i := range r {
+		if !r[i].match.MatchString(urlPath) {
+			continue
+		}
+
+		for _, pair := range r[i].headers {
+			header.Set(pair.name, pair.value)
+		}
+	}
+}
+
+// patternToRegexp converts a `_headers` path pattern to an anchored regexp.
+// `*` matches any run of characters including `/`, matching the upstream
+// behavior where `/*` covers the whole site.
+func patternToRegexp(pattern string) (*regexp.Regexp, error) {
+	var b strings.Builder
+
+	b.WriteString("^")
+
+	for i, part := range strings.Split(pattern, "*") {
+		if i > 0 {
+			b.WriteString(".*")
+		}
+
+		b.WriteString(regexp.QuoteMeta(part))
+	}
+
+	b.WriteString("$")
+
+	return regexp.Compile(b.String())
+}
+
+// isPatternLine reports whether a raw line opens a new block. Patterns sit at
+// column zero; header lines are indented, which is what separates them.
+func isPatternLine(raw string) bool {
+	return strings.HasPrefix(raw, "/")
+}
+
+// parseHeaderRules parses `_headers` content. Unparseable lines are skipped
+// rather than failing the whole file: a single malformed rule should not cost
+// a site every other header it declares.
+func parseHeaderRules(content string) headerRules {
+	var rules headerRules
+
+	for _, raw := range strings.Split(content, "\n") {
+		raw = strings.TrimRight(raw, "\r")
+		trimmed := strings.TrimSpace(raw)
+
+		if trimmed == "" || strings.HasPrefix(trimmed, "#") {
+			continue
+		}
+
+		if isPatternLine(raw) {
+			match, err := patternToRegexp(trimmed)
+			if err != nil {
+				continue
+			}
+
+			rules = append(rules, headerRule{match: match})
+
+			continue
+		}
+
+		if len(rules) == 0 {
+			continue
+		}
+
+		name, value, found := strings.Cut(trimmed, ":")
+		if !found {
+			continue
+		}
+
+		name = strings.TrimSpace(name)
+		value = strings.TrimSpace(value)
+
+		if name == "" || value == "" {
+			continue
+		}
+
+		last := len(rules) - 1
+		rules[last].headers = append(rules[last].headers, headerPair{name: name, value: value})
+	}
+
+	return rules
+}
+
+// loadHeaderRules reads `_headers` from the published directory. A missing or
+// unreadable file yields no rules, so a deployment without one behaves exactly
+// as it does today.
+func loadHeaderRules(fs file.FileSystem, staticFilePath string) headerRules {
+	f, err := fs.Open(filepath.Join(staticFilePath, headersFileName))
+	if err != nil {
+		return nil
+	}
+
+	defer func() { _ = f.Close() }()
+
+	content, err := io.ReadAll(f)
+	if err != nil {
+		return nil
+	}
+
+	return parseHeaderRules(string(content))
+}