From 26cbf930b4a729dfb38ff8078ecbfcb0264c5360 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Tue, 25 Aug 2026 23:54:07 +0000
Subject: [PATCH] Keep gathered sources verbatim

WebFetch and "cleaned source" were inviting a paraphrase. Save files
and pastes through save-source.mjs, strip page chrome only, and show
first/last lines so the author can confirm it is their writing.
---
 .github/workflows/test.yml                    |   2 +-
 README.md                                     |   2 +-
 package.json                                  |   2 +-
 skills/docwriter-style-generator/SKILL.md     |  46 +++-
 .../scripts/save-source.mjs                   | 219 ++++++++++++++++++
 tests/lexical-analysis.mjs                    |   9 +-
 tests/save-source.mjs                         |  94 ++++++++
 7 files changed, 359 insertions(+), 15 deletions(-)
 create mode 100644 skills/docwriter-style-generator/scripts/save-source.mjs
 create mode 100644 tests/save-source.mjs

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 44066c1..97a8e70 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -12,4 +12,4 @@ jobs:
       - uses: actions/setup-node@v4
         with:
           node-version: 22
-      - run: node tests/lexical-analysis.mjs
+      - run: npm test
diff --git a/README.md b/README.md
index 1f840e2..29f867a 100644
--- a/README.md
+++ b/README.md
@@ -88,7 +88,7 @@ A separate skill at `~/.claude/skills/my-writing-style/` (Claude Code) and
 ```
 my-writing-style/
   SKILL.md              ← "write like this person" instructions
-  sources/              ← cleaned writing samples
+  sources/              ← your writing, stored verbatim
   references/
     propositions.json   ← the habit data
     examples.md         ← your passages with key sentences highlighted
diff --git a/package.json b/package.json
index cdbd876..45aa1ca 100644
--- a/package.json
+++ b/package.json
@@ -3,6 +3,6 @@
   "private": true,
   "type": "module",
   "scripts": {
-    "test": "node tests/lexical-analysis.mjs"
+    "test": "node tests/lexical-analysis.mjs && node tests/save-source.mjs"
   }
 }
diff --git a/skills/docwriter-style-generator/SKILL.md b/skills/docwriter-style-generator/SKILL.md
index 13471d6..5990601 100644
--- a/skills/docwriter-style-generator/SKILL.md
+++ b/skills/docwriter-style-generator/SKILL.md
@@ -74,21 +74,44 @@ AskUserQuestion:
       description: "Some files, some URLs, some pasted"
 ```
 
-Based on their answer, ask for the paths/URLs/text. Read files with Read,
-fetch URLs with WebFetch (extract the article body, strip nav/menus/footers/
-cookie banners/bylines). Label each source by what the user calls it.
+Based on their answer, ask for the paths/URLs/text. Label each source by
+what the user calls it.
 
-Write each cleaned source into the style skill, not `/tmp`. Create the
-directories if they do not exist:
+**Sources must be the author's words, verbatim.** Do not paraphrase,
+summarize, tidy, or rewrite. The later analysis quotes these files word
+for word — a summary is the wrong sample.
+
+Do not use WebFetch to produce a stored source. WebFetch often returns a
+digest. This skill ships `scripts/save-source.mjs` next to this file.
+Run it once per piece so the write is a copy, not a rewrite:
+
+```bash
+# local file
+node scripts/save-source.mjs --label essay --file /path/to/essay.txt
+
+# pasted text
+node scripts/save-source.mjs --label talk --stdin
+# then pipe or type the paste; do not reword it
+
+# URL — fetches the page and strips nav/chrome only
+node scripts/save-source.mjs --label post --url https://example.com/essay
+```
+
+The script writes the same file to both homes:
 
 ```
 ~/.claude/skills/my-writing-style/sources/<label>.txt
 ~/.agents/skills/my-writing-style/sources/<label>.txt
 ```
 
-Use the user's label as the filename (safe slug). On a rebuild, replace
-`sources/` so leftover pieces from the last run do not stay in the sample.
-When adding a source, leave the existing files alone.
+After each save, show the user the script's `First:` / `Last:` lines and
+word count, and ask whether that is their writing. If they say no, or if
+a URL extract errors as too short, ask for a local file or a paste of
+the full piece. Do not invent the missing text.
+
+On a rebuild, replace `sources/` so leftover pieces from the last run do
+not stay in the sample. When adding a source, leave the existing files
+alone.
 
 After gathering, confirm:
 
@@ -173,8 +196,10 @@ Tell the user how many habits you found before moving to calibration.
 
 Rules that matter:
 
-- **Examples must be verbatim.** Copy word for word from the sources. No
-  invention. If you cannot find three real examples, drop the proposition.
+- **Examples must be verbatim.** Copy word for word from the `sources/`
+  files. Each example must appear as a contiguous substring of a saved
+  source. No invention, paraphrase, or tidy-up. If you cannot find three
+  real examples, drop the proposition.
 - **Examples are passages, not sentences.** Quote 3-4 consecutive sentences
   so the reader sees the habit in context. Set the focus to the one sentence
   the habit is actually about.
@@ -285,6 +310,7 @@ can re-run the word-level measurements:
 
 ```
 scripts/analyze-style.mjs
+scripts/save-source.mjs
 scripts/style-metrics.mjs
 scripts/style-metric-registry.mjs
 scripts/style-data.json
diff --git a/skills/docwriter-style-generator/scripts/save-source.mjs b/skills/docwriter-style-generator/scripts/save-source.mjs
new file mode 100644
index 0000000..b1a1fae
--- /dev/null
+++ b/skills/docwriter-style-generator/scripts/save-source.mjs
@@ -0,0 +1,219 @@
+#!/usr/bin/env node
+/**
+ * Write a source into the style skill homes without rewriting it.
+ * Local files and pasted text are copied as-is. URLs are fetched and
+ * reduced to visible article text (chrome stripped). Never summarizes.
+ */
+
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const DEFAULT_HOMES = [
+	join(homedir(), '.claude/skills/my-writing-style'),
+	join(homedir(), '.agents/skills/my-writing-style')
+];
+
+const CHROME = new Set(['script', 'style', 'noscript', 'template', 'svg', 'nav', 'footer', 'header', 'aside', 'form', 'iframe']);
+
+function flagValues(args, name) {
+	const values = [];
+	for (let index = 0; index < args.length; index += 1) {
+		if (args[index] === name && args[index + 1] && !String(args[index + 1]).startsWith('--')) {
+			values.push(args[index + 1]);
+			index += 1;
+		}
+	}
+	return values;
+}
+
+function slug(value) {
+	const safe = String(value)
+		.trim()
+		.toLowerCase()
+		.replace(/[^a-z0-9]+/g, '-')
+		.replace(/^-+|-+$/g, '');
+	if (!safe) throw new Error('label must contain letters or numbers');
+	return safe;
+}
+
+function wordCount(text) {
+	return text.trim() ? text.trim().split(/\s+/).length : 0;
+}
+
+function firstWords(text, count = 40) {
+	return text.trim().split(/\s+/).slice(0, count).join(' ');
+}
+
+function lastWords(text, count = 40) {
+	return text.trim().split(/\s+/).slice(-count).join(' ');
+}
+
+function decodeEntities(text) {
+	return text
+		.replace(/&#x([0-9a-f]+);/gi, (_, hex) => String.fromCodePoint(Number.parseInt(hex, 16)))
+		.replace(/&#(\d+);/g, (_, dec) => String.fromCodePoint(Number(dec)))
+		.replace(/&nbsp;/gi, ' ')
+		.replace(/&amp;/gi, '&')
+		.replace(/&lt;/gi, '<')
+		.replace(/&gt;/gi, '>')
+		.replace(/&quot;/gi, '"')
+		.replace(/&apos;/gi, "'");
+}
+
+function stripTagsKeepBreaks(html) {
+	return html
+		.replace(/<(br|hr)\b[^>]*>/gi, '\n')
+		.replace(/<\/(p|div|h[1-6]|li|tr|blockquote|section|article|pre)>/gi, '\n\n')
+		.replace(/<[^>]+>/g, '')
+		.replace(/[ \t]+\n/g, '\n')
+		.replace(/\n{3,}/g, '\n\n')
+		.trim();
+}
+
+function removeChrome(html) {
+	let next = html;
+	for (const tag of CHROME) {
+		next = next.replace(new RegExp(`<${tag}\\b[^>]*>[\\s\\S]*?<\\/${tag}>`, 'gi'), ' ');
+		next = next.replace(new RegExp(`<${tag}\\b[^>]*\\/?>`, 'gi'), ' ');
+	}
+	return next;
+}
+
+function innerByTag(html, tag) {
+	const match = html.match(new RegExp(`<${tag}\\b[^>]*>([\\s\\S]*?)<\\/${tag}>`, 'i'));
+	return match?.[1] ?? '';
+}
+
+function findArticleBody(node) {
+	if (!node || typeof node !== 'object') return '';
+	if (Array.isArray(node)) {
+		for (const item of node) {
+			const found = findArticleBody(item);
+			if (found) return found;
+		}
+		return '';
+	}
+	if (typeof node.articleBody === 'string' && node.articleBody.trim()) return node.articleBody;
+	if (Array.isArray(node['@graph'])) return findArticleBody(node['@graph']);
+	return findArticleBody(node.mainEntity);
+}
+
+function extractJsonLdArticle(html) {
+	for (const match of html.matchAll(/<script[^>]*type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi)) {
+		try {
+			const found = findArticleBody(JSON.parse(match[1]));
+			if (wordCount(found) >= 50) return found.trim();
+		} catch {
+			// ignore broken JSON-LD
+		}
+	}
+	return '';
+}
+
+export function extractVerbatimText(raw, { html = false } = {}) {
+	const source = String(raw ?? '').replace(/^\uFEFF/, '');
+	if (!html) return source.replace(/\r\n/g, '\n');
+	const fromLd = extractJsonLdArticle(source);
+	if (fromLd) return fromLd.replace(/\r\n/g, '\n');
+	const article = innerByTag(source, 'article') || innerByTag(source, 'main') || innerByTag(source, 'body') || source;
+	return decodeEntities(stripTagsKeepBreaks(removeChrome(article))).replace(/\r\n/g, '\n');
+}
+
+function looksLikeHtml(text, contentType = '') {
+	if (/html/i.test(contentType)) return true;
+	return /^\s*(<!doctype html|<html\b)/i.test(text);
+}
+
+async function readStdin() {
+	const chunks = [];
+	for await (const chunk of process.stdin) chunks.push(chunk);
+	return Buffer.concat(chunks).toString('utf8');
+}
+
+async function fetchUrl(url) {
+	const response = await fetch(url, {
+		headers: { 'user-agent': 'docwriter-style-generator/1.0 (verbatim source save)' },
+		redirect: 'follow'
+	});
+	if (!response.ok) throw new Error(`fetch failed ${response.status} for ${url}`);
+	const text = await response.text();
+	return { text, html: looksLikeHtml(text, response.headers.get('content-type') ?? '') };
+}
+
+export async function saveSource({ label, text, homes, html = false, minWords = 0 }) {
+	const name = slug(label);
+	const body = extractVerbatimText(text, { html: html || looksLikeHtml(text) });
+	if (!body.trim()) throw new Error('source is empty after chrome-strip; ask for a file or paste');
+	const count = wordCount(body);
+	if (minWords && count < minWords) {
+		throw new Error(
+			`Extract is only ${count} words — too short to trust. Ask the user for a local file or paste.`
+		);
+	}
+	const written = [];
+	for (const home of homes) {
+		const dest = join(home, 'sources', `${name}.txt`);
+		await mkdir(dirname(dest), { recursive: true });
+		await writeFile(dest, body.endsWith('\n') ? body : `${body}\n`, 'utf8');
+		written.push(dest);
+	}
+	return {
+		label: name,
+		wordCount: count,
+		first: firstWords(body),
+		last: lastWords(body),
+		written
+	};
+}
+
+async function cli() {
+	const args = process.argv.slice(2);
+	const label = flagValues(args, '--label')[0];
+	const file = flagValues(args, '--file')[0];
+	const url = flagValues(args, '--url')[0];
+	const homes = flagValues(args, '--home');
+	const useStdin = args.includes('--stdin');
+	if (!label || [file, url, useStdin].filter(Boolean).length !== 1) {
+		throw new Error(
+			'Usage: save-source.mjs --label <slug> (--file <path> | --url <url> | --stdin) [--home <dir> ...]'
+		);
+	}
+	let text;
+	let html = false;
+	if (file) {
+		text = await readFile(file, 'utf8');
+		html = looksLikeHtml(text);
+	} else if (url) {
+		const fetched = await fetchUrl(url);
+		text = fetched.text;
+		html = fetched.html;
+	} else {
+		text = await readStdin();
+		html = looksLikeHtml(text);
+	}
+	const result = await saveSource({
+		label,
+		text,
+		html,
+		minWords: url ? 200 : 0,
+		homes: homes.length ? homes : DEFAULT_HOMES
+	});
+	process.stdout.write(
+		[
+			`Saved ${result.label} (${result.wordCount} words)`,
+			`First: ${result.first}`,
+			`Last: ${result.last}`,
+			...result.written.map((path) => `Wrote ${path}`),
+			''
+		].join('\n')
+	);
+}
+
+if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) {
+	cli().catch((error) => {
+		process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
+		process.exitCode = 1;
+	});
+}
diff --git a/tests/lexical-analysis.mjs b/tests/lexical-analysis.mjs
index 6492aa8..25ff7f4 100644
--- a/tests/lexical-analysis.mjs
+++ b/tests/lexical-analysis.mjs
@@ -72,12 +72,17 @@ for (const needle of [
 	'~/.agents/skills/my-writing-style/',
 	'$docwriter-style-generator',
 	'Add a source',
-	'Update propositions'
+	'Update propositions',
+	'scripts/save-source.mjs',
+	'verbatim'
 ]) {
 	if (!skill.includes(needle)) throw new Error(`SKILL.md is missing ${needle}`);
 }
 if (skill.includes('/tmp/source-')) {
-	throw new Error('cleaned sources should live in the style skill, not /tmp');
+	throw new Error('sources should live in the style skill, not /tmp');
+}
+if (skill.includes('fetch URLs with WebFetch')) {
+	throw new Error('WebFetch paraphrases; sources must go through save-source.mjs');
 }
 
 const marketplace = JSON.parse(readFileSync(join(root, '.claude-plugin/marketplace.json'), 'utf8'));
diff --git a/tests/save-source.mjs b/tests/save-source.mjs
new file mode 100644
index 0000000..c17d6a0
--- /dev/null
+++ b/tests/save-source.mjs
@@ -0,0 +1,94 @@
+import { execFileSync } from 'node:child_process';
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { extractVerbatimText } from '../skills/docwriter-style-generator/scripts/save-source.mjs';
+
+const root = join(dirname(fileURLToPath(import.meta.url)), '..');
+const saver = join(root, 'skills/docwriter-style-generator/scripts/save-source.mjs');
+const fixture = readFileSync(join(root, 'tests/fixture-prose.txt'), 'utf8');
+const phrase = 'splendid and lonely courage';
+
+function save(args, input) {
+	return execFileSync(process.execPath, [saver, ...args], {
+		encoding: 'utf8',
+		input
+	});
+}
+
+const extractedFile = extractVerbatimText(fixture);
+if (!extractedFile.includes(phrase)) throw new Error('plain extract dropped a verbatim phrase');
+if (extractedFile.replace(/\n$/, '') !== fixture.replace(/\r\n/g, '\n').replace(/\n$/, '')) {
+	throw new Error('plain extract rewrote the file');
+}
+
+const html = `<!doctype html>
+<html>
+<body>
+<nav>Home About <a href="/">Subscribe</a></nav>
+<aside>In this essay, the author argues that Mallarme prefers motion to rest.</aside>
+<article>
+<p>It is in his moments of movement that Mallarme is exciting, and his moments of lingering are his moments of extreme conventionality.</p>
+<p>There is a splendid and lonely courage in the rejection of the accustomed word when the accustomed word is not the inevitable word.</p>
+</article>
+<footer>Cookie settings</footer>
+</body>
+</html>`;
+
+const fromHtml = extractVerbatimText(html, { html: true });
+if (!fromHtml.includes(phrase)) throw new Error('HTML extract lost the verbatim sentence');
+if (!fromHtml.includes('moments of movement that Mallarme is exciting')) {
+	throw new Error('HTML extract lost the opening sentence');
+}
+for (const leak of ['Subscribe', 'In this essay, the author argues', 'Cookie settings']) {
+	if (fromHtml.includes(leak)) throw new Error(`HTML extract kept chrome or paraphrase: ${leak}`);
+}
+
+const ld = `<html><body>
+<script type="application/ld+json">
+{"@type":"Article","articleBody":"There is a ${phrase}. He will not take the phrase that lies in wait. He will not have the easy metaphor, the ready epithet, the expected cadence. The language he wants is the language that has not yet been used for that thought, and he will wait, or he will invent, rather than accept a substitute that has served another man. The reader who will walk with him finds the path lit."}
+</script>
+<nav>Menu</nav>
+<p>Short teaser that is not the essay.</p>
+</body></html>`;
+const fromLd = extractVerbatimText(ld, { html: true });
+if (!fromLd.includes(phrase) || fromLd.includes('Short teaser')) {
+	throw new Error('JSON-LD extract should use articleBody, not the teaser');
+}
+
+const scratch = mkdtempSync(join(tmpdir(), 'style-save-'));
+const homeA = join(scratch, 'claude');
+const homeB = join(scratch, 'agents');
+try {
+	const sourceFile = join(scratch, 'essay.txt');
+	writeFileSync(sourceFile, fixture);
+	const receipt = save([
+		'--label', 'Essay Title!',
+		'--file', sourceFile,
+		'--home', homeA,
+		'--home', homeB
+	]);
+	if (!receipt.includes('Saved essay-title')) throw new Error(`bad receipt: ${receipt}`);
+	if (!receipt.includes('It is in his moments of movement')) {
+		throw new Error(`receipt First: line should quote the file: ${receipt}`);
+	}
+	const copyA = readFileSync(join(homeA, 'sources/essay-title.txt'), 'utf8');
+	const copyB = readFileSync(join(homeB, 'sources/essay-title.txt'), 'utf8');
+	if (copyA !== copyB) throw new Error('homes diverged');
+	if (!copyA.includes(phrase)) throw new Error('saved file paraphrased the fixture');
+	if (copyA.replace(/\n$/, '') !== fixture.replace(/\n$/, '')) {
+		throw new Error('saved file is not verbatim');
+	}
+
+	const pasted = 'He will not take the phrase that lies in wait.\n';
+	save(['--label', 'talk', '--stdin', '--home', homeA], pasted);
+	const talk = readFileSync(join(homeA, 'sources/talk.txt'), 'utf8');
+	if (talk.replace(/\n$/, '') !== pasted.replace(/\n$/, '')) {
+		throw new Error('stdin save rewrote the paste');
+	}
+} finally {
+	rmSync(scratch, { recursive: true, force: true });
+}
+
+process.stdout.write('ok: sources stay verbatim (file, paste, HTML chrome-strip)\n');