Keep gathered sources verbatim

.github/workflows/test.yml

@@ -12,4 +12,4 @@ jobs:
       - uses: actions/setup-node@v4
         with:
           node-version: 22
-      - run: node tests/lexical-analysis.mjs
+      - run: npm test

README.md

@@ -88,7 +88,7 @@ A separate skill at `~/.claude/skills/my-writing-style/` (Claude Code) and
 ```
 my-writing-style/
   SKILL.md              ← "write like this person" instructions
-  sources/              ← cleaned writing samples
+  sources/              ← your writing, stored verbatim
   references/
     propositions.json   ← the habit data
     examples.md         ← your passages with key sentences highlighted

package.json

@@ -3,6 +3,6 @@
   "private": true,
   "type": "module",
   "scripts": {
-    "test": "node tests/lexical-analysis.mjs"
+    "test": "node tests/lexical-analysis.mjs && node tests/save-source.mjs"
   }
 }

skills/docwriter-style-generator/SKILL.md

@@ -74,21 +74,44 @@ AskUserQuestion:
       description: "Some files, some URLs, some pasted"
 ```
 
-Based on their answer, ask for the paths/URLs/text. Read files with Read,
-fetch URLs with WebFetch (extract the article body, strip nav/menus/footers/
-cookie banners/bylines). Label each source by what the user calls it.
+Based on their answer, ask for the paths/URLs/text. Label each source by
+what the user calls it.
 
-Write each cleaned source into the style skill, not `/tmp`. Create the
-directories if they do not exist:
+**Sources must be the author's words, verbatim.** Do not paraphrase,
+summarize, tidy, or rewrite. The later analysis quotes these files word
+for word — a summary is the wrong sample.
+
+Do not use WebFetch to produce a stored source. WebFetch often returns a
+digest. This skill ships `scripts/save-source.mjs` next to this file.
+Run it once per piece so the write is a copy, not a rewrite:
+
+```bash
+# local file
+node scripts/save-source.mjs --label essay --file /path/to/essay.txt
+
+# pasted text
+node scripts/save-source.mjs --label talk --stdin
+# then pipe or type the paste; do not reword it
+
+# URL — fetches the page and strips nav/chrome only
+node scripts/save-source.mjs --label post --url https://example.com/essay
+```
+
+The script writes the same file to both homes:
 
 ```
 ~/.claude/skills/my-writing-style/sources/<label>.txt
 ~/.agents/skills/my-writing-style/sources/<label>.txt
 ```
 
-Use the user's label as the filename (safe slug). On a rebuild, replace
-`sources/` so leftover pieces from the last run do not stay in the sample.
-When adding a source, leave the existing files alone.
+After each save, show the user the script's `First:` / `Last:` lines and
+word count, and ask whether that is their writing. If they say no, or if
+a URL extract errors as too short, ask for a local file or a paste of
+the full piece. Do not invent the missing text.
+
+On a rebuild, replace `sources/` so leftover pieces from the last run do
+not stay in the sample. When adding a source, leave the existing files
+alone.
 
 After gathering, confirm:
 
@@ -173,8 +196,10 @@ Tell the user how many habits you found before moving to calibration.
 
 Rules that matter:
 
-- **Examples must be verbatim.** Copy word for word from the sources. No
-  invention. If you cannot find three real examples, drop the proposition.
+- **Examples must be verbatim.** Copy word for word from the `sources/`
+  files. Each example must appear as a contiguous substring of a saved
+  source. No invention, paraphrase, or tidy-up. If you cannot find three
+  real examples, drop the proposition.
 - **Examples are passages, not sentences.** Quote 3-4 consecutive sentences
   so the reader sees the habit in context. Set the focus to the one sentence
   the habit is actually about.
@@ -285,6 +310,7 @@ can re-run the word-level measurements:
 
 ```
 scripts/analyze-style.mjs
+scripts/save-source.mjs
 scripts/style-metrics.mjs
 scripts/style-metric-registry.mjs
 scripts/style-data.json

skills/docwriter-style-generator/scripts/save-source.mjs

@@ -0,0 +1,219 @@
+#!/usr/bin/env node
+/**
+ * Write a source into the style skill homes without rewriting it.
+ * Local files and pasted text are copied as-is. URLs are fetched and
+ * reduced to visible article text (chrome stripped). Never summarizes.
+ */
+
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const DEFAULT_HOMES = [
+	join(homedir(), '.claude/skills/my-writing-style'),
+	join(homedir(), '.agents/skills/my-writing-style')
+];
+
+const CHROME = new Set(['script', 'style', 'noscript', 'template', 'svg', 'nav', 'footer', 'header', 'aside', 'form', 'iframe']);
+
+function flagValues(args, name) {
+	const values = [];
+	for (let index = 0; index < args.length; index += 1) {
+		if (args[index] === name && args[index + 1] && !String(args[index + 1]).startsWith('--')) {
+			values.push(args[index + 1]);
+			index += 1;
+		}
+	}
+	return values;
+}
+
+function slug(value) {
+	const safe = String(value)
+		.trim()
+		.toLowerCase()
+		.replace(/[^a-z0-9]+/g, '-')
+		.replace(/^-+|-+$/g, '');
+	if (!safe) throw new Error('label must contain letters or numbers');
+	return safe;
+}
+
+function wordCount(text) {
+	return text.trim() ? text.trim().split(/\s+/).length : 0;
+}
+
+function firstWords(text, count = 40) {
+	return text.trim().split(/\s+/).slice(0, count).join(' ');
+}
+
+function lastWords(text, count = 40) {
+	return text.trim().split(/\s+/).slice(-count).join(' ');
+}
+
+function decodeEntities(text) {
+	return text
+		.replace(/&#x([0-9a-f]+);/gi, (_, hex) => String.fromCodePoint(Number.parseInt(hex, 16)))
+		.replace(/&#(\d+);/g, (_, dec) => String.fromCodePoint(Number(dec)))
+		.replace(/&nbsp;/gi, ' ')
+		.replace(/&amp;/gi, '&')
+		.replace(/&lt;/gi, '<')
+		.replace(/&gt;/gi, '>')
+		.replace(/&quot;/gi, '"')
+		.replace(/&apos;/gi, "'");
+}
+
+function stripTagsKeepBreaks(html) {
+	return html
+		.replace(/<(br|hr)\b[^>]*>/gi, '\n')
+		.replace(/<\/(p|div|h[1-6]|li|tr|blockquote|section|article|pre)>/gi, '\n\n')
+		.replace(/<[^>]+>/g, '')
+		.replace(/[ \t]+\n/g, '\n')
+		.replace(/\n{3,}/g, '\n\n')
+		.trim();
+}
+
+function removeChrome(html) {
+	let next = html;
+	for (const tag of CHROME) {
+		next = next.replace(new RegExp(`<${tag}\\b[^>]*>[\\s\\S]*?<\\/${tag}>`, 'gi'), ' ');
+		next = next.replace(new RegExp(`<${tag}\\b[^>]*\\/?>`, 'gi'), ' ');
+	}
+	return next;
+}
+
+function innerByTag(html, tag) {
+	const match = html.match(new RegExp(`<${tag}\\b[^>]*>([\\s\\S]*?)<\\/${tag}>`, 'i'));
+	return match?.[1] ?? '';
+}
+
+function findArticleBody(node) {
+	if (!node || typeof node !== 'object') return '';
+	if (Array.isArray(node)) {
+		for (const item of node) {
+			const found = findArticleBody(item);
+			if (found) return found;
+		}
+		return '';
+	}
+	if (typeof node.articleBody === 'string' && node.articleBody.trim()) return node.articleBody;
+	if (Array.isArray(node['@graph'])) return findArticleBody(node['@graph']);
+	return findArticleBody(node.mainEntity);
+}
+
+function extractJsonLdArticle(html) {
+	for (const match of html.matchAll(/<script[^>]*type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi)) {
+		try {
+			const found = findArticleBody(JSON.parse(match[1]));
+			if (wordCount(found) >= 50) return found.trim();
+		} catch {
+			// ignore broken JSON-LD
+		}
+	}
+	return '';
+}
+
+export function extractVerbatimText(raw, { html = false } = {}) {
+	const source = String(raw ?? '').replace(/^\uFEFF/, '');
+	if (!html) return source.replace(/\r\n/g, '\n');
+	const fromLd = extractJsonLdArticle(source);
+	if (fromLd) return fromLd.replace(/\r\n/g, '\n');
+	const article = innerByTag(source, 'article') || innerByTag(source, 'main') || innerByTag(source, 'body') || source;
+	return decodeEntities(stripTagsKeepBreaks(removeChrome(article))).replace(/\r\n/g, '\n');
+}
+
+function looksLikeHtml(text, contentType = '') {
+	if (/html/i.test(contentType)) return true;
+	return /^\s*(<!doctype html|<html\b)/i.test(text);
+}
+
+async function readStdin() {
+	const chunks = [];
+	for await (const chunk of process.stdin) chunks.push(chunk);
+	return Buffer.concat(chunks).toString('utf8');
+}
+
+async function fetchUrl(url) {
+	const response = await fetch(url, {
+		headers: { 'user-agent': 'docwriter-style-generator/1.0 (verbatim source save)' },
+		redirect: 'follow'
+	});
+	if (!response.ok) throw new Error(`fetch failed ${response.status} for ${url}`);
+	const text = await response.text();
+	return { text, html: looksLikeHtml(text, response.headers.get('content-type') ?? '') };
+}
+
+export async function saveSource({ label, text, homes, html = false, minWords = 0 }) {
+	const name = slug(label);
+	const body = extractVerbatimText(text, { html: html || looksLikeHtml(text) });
+	if (!body.trim()) throw new Error('source is empty after chrome-strip; ask for a file or paste');
+	const count = wordCount(body);
+	if (minWords && count < minWords) {
+		throw new Error(
+			`Extract is only ${count} words — too short to trust. Ask the user for a local file or paste.`
+		);
+	}
+	const written = [];
+	for (const home of homes) {
+		const dest = join(home, 'sources', `${name}.txt`);
+		await mkdir(dirname(dest), { recursive: true });
+		await writeFile(dest, body.endsWith('\n') ? body : `${body}\n`, 'utf8');
+		written.push(dest);
+	}
+	return {
+		label: name,
+		wordCount: count,
+		first: firstWords(body),
+		last: lastWords(body),
+		written
+	};
+}
+
+async function cli() {
+	const args = process.argv.slice(2);
+	const label = flagValues(args, '--label')[0];
+	const file = flagValues(args, '--file')[0];
+	const url = flagValues(args, '--url')[0];
+	const homes = flagValues(args, '--home');
+	const useStdin = args.includes('--stdin');
+	if (!label || [file, url, useStdin].filter(Boolean).length !== 1) {
+		throw new Error(
+			'Usage: save-source.mjs --label <slug> (--file <path> | --url <url> | --stdin) [--home <dir> ...]'
+		);
+	}
+	let text;
+	let html = false;
+	if (file) {
+		text = await readFile(file, 'utf8');
+		html = looksLikeHtml(text);
+	} else if (url) {
+		const fetched = await fetchUrl(url);
+		text = fetched.text;
+		html = fetched.html;
+	} else {
+		text = await readStdin();
+		html = looksLikeHtml(text);
+	}
+	const result = await saveSource({
+		label,
+		text,
+		html,
+		minWords: url ? 200 : 0,
+		homes: homes.length ? homes : DEFAULT_HOMES
+	});
+	process.stdout.write(
+		[
+			`Saved ${result.label} (${result.wordCount} words)`,
+			`First: ${result.first}`,
+			`Last: ${result.last}`,
+			...result.written.map((path) => `Wrote ${path}`),
+			''
+		].join('\n')
+	);
+}
+
+if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) {
+	cli().catch((error) => {
+		process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
+		process.exitCode = 1;
+	});
+}

tests/lexical-analysis.mjs

@@ -72,12 +72,17 @@ for (const needle of [
 	'~/.agents/skills/my-writing-style/',
 	'$docwriter-style-generator',
 	'Add a source',
-	'Update propositions'
+	'Update propositions',
+	'scripts/save-source.mjs',
+	'verbatim'
 ]) {
 	if (!skill.includes(needle)) throw new Error(`SKILL.md is missing ${needle}`);
 }
 if (skill.includes('/tmp/source-')) {
-	throw new Error('cleaned sources should live in the style skill, not /tmp');
+	throw new Error('sources should live in the style skill, not /tmp');
+}
+if (skill.includes('fetch URLs with WebFetch')) {
+	throw new Error('WebFetch paraphrases; sources must go through save-source.mjs');
 }
 
 const marketplace = JSON.parse(readFileSync(join(root, '.claude-plugin/marketplace.json'), 'utf8'));