From c4eeba3172bdc2d8d5706151c48eeca04c350447 Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Thu, 6 Aug 2026 21:48:36 -0700 Subject: [PATCH 1/5] Rework remend code-region detection and double-underscore counting A shared single-pass scanner (scan.ts) classifies fenced code and inline spans once per input, replacing the per-character rescans that made healing quadratic on delimiter-heavy input. Fence semantics now follow CommonMark: fences open only at line start with up to 3 spaces of indent, tilde fences are recognized, closers must be at least the opener's length, and info strings can neither open nor close emphasis. Inline code spans close on a backtick run of exactly the opener's length. Double underscores are counted per maximal run with flanking rules, so identifiers containing __ (snake__case) no longer invent or swallow emphasis closers. Healing is idempotent: incomplete link/image removal iterates to a fixed point and the trailing space exposed by a removal is stripped like any other, so healed output re-heals to itself. A fast-check property suite and an exhaustive prefix sweep enforce this along with a bounded- loss oracle, and size-scaled bench cases make the linear scaling visible. --- .changeset/spotty-eyes-brake.md | 11 + .../remend/__benchmarks__/remend.bench.ts | 22 + .../broken-markdown-variants.test.ts | 14 +- .../remend/__tests__/coverage-gaps.test.ts | 20 +- .../remend/__tests__/fence-semantics.test.ts | 78 ++++ packages/remend/__tests__/images.test.ts | 10 +- packages/remend/__tests__/links.test.ts | 15 +- .../__tests__/streaming-properties.test.ts | 151 +++++++ .../remend/__tests__/underscore-runs.test.ts | 51 +++ packages/remend/package.json | 1 + packages/remend/src/code-block-utils.ts | 99 +--- packages/remend/src/emphasis-handlers.ts | 332 +++++++------- packages/remend/src/index.ts | 7 + packages/remend/src/inline-code-handler.ts | 84 ++-- packages/remend/src/katex-handler.ts | 26 +- packages/remend/src/link-image-handler.ts | 29 +- packages/remend/src/patterns.ts | 6 - packages/remend/src/scan.ts | 424 ++++++++++++++++++ packages/remend/src/strikethrough-handler.ts | 29 +- packages/remend/src/utils.ts | 109 +---- pnpm-lock.yaml | 24 +- 21 files changed, 1068 insertions(+), 474 deletions(-) create mode 100644 .changeset/spotty-eyes-brake.md create mode 100644 packages/remend/__tests__/fence-semantics.test.ts create mode 100644 packages/remend/__tests__/streaming-properties.test.ts create mode 100644 packages/remend/__tests__/underscore-runs.test.ts create mode 100644 packages/remend/src/scan.ts diff --git a/.changeset/spotty-eyes-brake.md b/.changeset/spotty-eyes-brake.md new file mode 100644 index 00000000..99a29f99 --- /dev/null +++ b/.changeset/spotty-eyes-brake.md @@ -0,0 +1,11 @@ +--- +"remend": minor +--- + +Rework code-region detection and double-underscore counting. + +A shared single-pass scanner now classifies fences and inline code spans, replacing the per-character rescans that made healing quadratic on delimiter-heavy input. Fences follow CommonMark rules (line-start only, up to 3 spaces of indent, `~~~` supported, closers must match the opener's length, info strings excluded from emphasis), and inline code spans close on a backtick run of exactly the opener's length. + +Double underscores are counted per maximal run with flanking rules, so identifiers containing `__` (like `snake__case`) no longer invent or swallow emphasis closers. + +Healing is now idempotent: healed output re-heals to itself, including incomplete image removal and the trailing space it exposes. diff --git a/packages/remend/__benchmarks__/remend.bench.ts b/packages/remend/__benchmarks__/remend.bench.ts index d8ac7c3b..62ce9739 100644 --- a/packages/remend/__benchmarks__/remend.bench.ts +++ b/packages/remend/__benchmarks__/remend.bench.ts @@ -279,3 +279,25 @@ ${"Regular paragraph text with some [links](https://example.com) and more conten { iterations: 1000 } ); }); + +// Delimiter-heavy input at doubling sizes. Cost should grow in proportion to +// input size: a doubling that more than doubles the time signals a +// superlinear rescan in a handler. +describe("Scaling", () => { + const unit = + "word snake__case text __bold__ and _it_ plus `code` *star* ~~del~~ ".repeat( + 30 + ); + const sizes = [1, 2, 4, 8] as const; + + for (const mult of sizes) { + const doc = `${unit.repeat(mult)}__open`; + bench( + `delimiter-heavy ${doc.length} chars`, + () => { + remend(doc); + }, + { iterations: 200 } + ); + } +}); diff --git a/packages/remend/__tests__/broken-markdown-variants.test.ts b/packages/remend/__tests__/broken-markdown-variants.test.ts index 43d54a36..c39304e1 100644 --- a/packages/remend/__tests__/broken-markdown-variants.test.ts +++ b/packages/remend/__tests__/broken-markdown-variants.test.ts @@ -123,8 +123,10 @@ describe("multiple incomplete links", () => { }); it("should handle two incomplete links in text-only mode", () => { + // Healing runs to a fixed point, so both unmatched brackets are resolved + // in one call rather than one per call const result = remend("[link1 and [link2", { linkMode: "text-only" }); - expect(result).toBe("link1 and [link2"); + expect(result).toBe("link1 and link2"); }); }); @@ -586,11 +588,11 @@ describe("real-world AI streaming patterns", () => { ); }); - it("should handle incomplete image with partial URL (preserves trailing space)", () => { - // Image is removed, leaving "See " - the trailing space remains - // because remend only trims single trailing space at the very start - // before handlers run, and the handler produces new trailing space - expect(remend("See ![diagram](http://example.com/img")).toBe("See "); + it("should handle incomplete image with partial URL", () => { + // Removing the image exposes a trailing space, which is stripped the + // same way a trailing space in the input is, so healed output re-heals + // to itself + expect(remend("See ![diagram](http://example.com/img")).toBe("See"); }); it("should handle link with incomplete formatting after it", () => { diff --git a/packages/remend/__tests__/coverage-gaps.test.ts b/packages/remend/__tests__/coverage-gaps.test.ts index bdda90b1..6af65b2b 100644 --- a/packages/remend/__tests__/coverage-gaps.test.ts +++ b/packages/remend/__tests__/coverage-gaps.test.ts @@ -93,8 +93,12 @@ describe("link handler edge cases", () => { expect(remend("](partial")).toBe("](partial"); }); - it("should skip image brackets in text-only mode", () => { - expect(remend("![img [text", { linkMode: "text-only" })).toBe("![img text"); + it("should remove incomplete images in text-only mode", () => { + // The inner bracket is stripped first, exposing the incomplete image, + // which is removed like any other incomplete image. Previously the image + // survived one call only to be removed by the next, so healed output did + // not re-heal to itself. + expect(remend("![img [text", { linkMode: "text-only" })).toBe(""); }); it("should skip complete links in text-only mode", () => { @@ -196,9 +200,15 @@ describe("double underscore half-complete in code block", () => { }); }); -describe("double underscore half-complete with even pairs", () => { - it("should not complete when __ pairs are balanced", () => { - expect(remend("__a__ __b__content_")).toBe("__a__ __b__content_"); +describe("double underscore half-complete with word-internal run", () => { + it("should complete the opener left unmatched by a word-internal run", () => { + // The __ in b__content sits between word characters, so it is part of an + // identifier and cannot close emphasis. That leaves the __ before b as + // an unmatched opener, and the trailing _ as a half-typed closer to + // complete. Counting raw __ occurrences instead would pair the + // word-internal run against the opener and leave the underscores + // unhealed as literal text. + expect(remend("__a__ __b__content_")).toBe("__a__ __b__content__"); }); }); diff --git a/packages/remend/__tests__/fence-semantics.test.ts b/packages/remend/__tests__/fence-semantics.test.ts new file mode 100644 index 00000000..4ebc363c --- /dev/null +++ b/packages/remend/__tests__/fence-semantics.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from "vitest"; +import remend from "../src"; + +describe("tilde fences", () => { + it("should not heal emphasis inside a complete tilde fence", () => { + expect(remend("~~~\ncode with __stuff\n~~~\ndone")).toBe( + "~~~\ncode with __stuff\n~~~\ndone" + ); + }); + + it("should not heal emphasis inside an open tilde fence", () => { + expect(remend("~~~js\nx = a__b")).toBe("~~~js\nx = a__b"); + }); + + it("should heal strikethrough after a complete tilde fence", () => { + expect(remend("~~~\ncode\n~~~\nafter ~~open")).toBe( + "~~~\ncode\n~~~\nafter ~~open~~" + ); + }); + + it("should treat a mid-line tilde run as strikethrough context, not a fence", () => { + expect(remend("prose ~~struck~~ more prose __bold")).toBe( + "prose ~~struck~~ more prose __bold__" + ); + }); +}); + +describe("fence opener position", () => { + it("should recognize a fence indented up to three spaces", () => { + expect(remend(" ```\n__code\n ```\n__open")).toBe( + " ```\n__code\n ```\n__open__" + ); + }); + + it("should treat mid-line triple backticks as inline code", () => { + // A fence can only open at the start of a line, so a mid-line run is an + // inline code span and heals by completing its closing run + expect(remend("see ```inline code``")).toBe("see ```inline code```"); + }); +}); + +describe("fence closer length", () => { + it("should not close a fence with a shorter run", () => { + // The ``` run is shorter than the ```` opener, so the fence is still + // open and its content is not healed + expect(remend("````\ncode\n```\nstill __code")).toBe( + "````\ncode\n```\nstill __code" + ); + }); + + it("should close a fence with a longer run", () => { + expect(remend("```\ncode\n````\nafter __bold")).toBe( + "```\ncode\n````\nafter __bold__" + ); + }); +}); + +describe("fence info strings", () => { + it("should not heal emphasis in an info string", () => { + expect(remend("```python__hint\ncode")).toBe("```python__hint\ncode"); + }); +}); + +describe("inline code span run lengths", () => { + it("should complete a double-backtick span with a double run", () => { + expect(remend("``code`")).toBe("``code``"); + }); + + it("should complete only the missing part of the closing run", () => { + expect(remend("``code")).toBe("``code``"); + }); + + it("should leave a longer literal run inside an open span alone", () => { + // The trailing run is longer than the opener, so appending backticks + // could never close the span + expect(remend("`a``")).toBe("`a``"); + }); +}); diff --git a/packages/remend/__tests__/images.test.ts b/packages/remend/__tests__/images.test.ts index 0d4cdf02..6e1b8cbc 100644 --- a/packages/remend/__tests__/images.test.ts +++ b/packages/remend/__tests__/images.test.ts @@ -3,7 +3,9 @@ import remend from "../src"; describe("image handling", () => { it("should remove incomplete images", () => { - expect(remend("Text with ![incomplete image")).toBe("Text with "); + // The space exposed by removing the image is stripped like any other + // single trailing space + expect(remend("Text with ![incomplete image")).toBe("Text with"); expect(remend("![partial")).toBe(""); }); @@ -13,7 +15,7 @@ describe("image handling", () => { }); it("should handle partial image at chunk boundary", () => { - expect(remend("See ![the diag")).toBe("See "); + expect(remend("See ![the diag")).toBe("See"); // Images with partial URLs should be removed (images can't show skeleton) expect(remend("![logo](./assets/log")).toBe(""); }); @@ -21,9 +23,9 @@ describe("image handling", () => { it("should handle nested brackets in incomplete images", () => { // When findMatchingClosingBracket returns -1 for an image (lines 74-79) // For this to happen, we need an opening bracket with a ] but no proper matching - expect(remend("Text ![outer [inner]")).toBe("Text "); + expect(remend("Text ![outer [inner]")).toBe("Text"); expect(remend("![nested [brackets] text")).toBe(""); - expect(remend("Start ![foo [bar] baz")).toBe("Start "); + expect(remend("Start ![foo [bar] baz")).toBe("Start"); }); it("should not add trailing underscore for images with underscores in URL (#284)", () => { diff --git a/packages/remend/__tests__/links.test.ts b/packages/remend/__tests__/links.test.ts index bc0bb04e..7e8ec63f 100644 --- a/packages/remend/__tests__/links.test.ts +++ b/packages/remend/__tests__/links.test.ts @@ -115,10 +115,12 @@ describe("link handling with linkMode: text-only", () => { }); it("should handle nested brackets without matching closing bracket", () => { + // Healing runs to a fixed point, so every unmatched bracket is resolved + // in one call rather than one per call expect(remend("Text [outer [inner", textOnlyOptions)).toBe( - "Text outer [inner" + "Text outer inner" ); - expect(remend("[foo [bar [baz", textOnlyOptions)).toBe("foo [bar [baz"); + expect(remend("[foo [bar [baz", textOnlyOptions)).toBe("foo bar baz"); expect(remend("Text [outer [inner]", textOnlyOptions)).toBe( "Text outer [inner]" ); @@ -128,9 +130,10 @@ describe("link handling with linkMode: text-only", () => { }); it("should still remove incomplete images", () => { - // Images should still be removed entirely, regardless of linkMode - // Note: the space before the image is preserved - expect(remend("Text ![incomplete image", textOnlyOptions)).toBe("Text "); - expect(remend("Text ![alt](http://partial", textOnlyOptions)).toBe("Text "); + // Images should still be removed entirely, regardless of linkMode. + // The space exposed by the removal is stripped like any other single + // trailing space. + expect(remend("Text ![incomplete image", textOnlyOptions)).toBe("Text"); + expect(remend("Text ![alt](http://partial", textOnlyOptions)).toBe("Text"); }); }); diff --git a/packages/remend/__tests__/streaming-properties.test.ts b/packages/remend/__tests__/streaming-properties.test.ts new file mode 100644 index 00000000..1ee7c67f --- /dev/null +++ b/packages/remend/__tests__/streaming-properties.test.ts @@ -0,0 +1,151 @@ +import fc from "fast-check"; +import { describe, expect, it } from "vitest"; +import remend from "../src"; + +// A streaming consumer feeds every prefix of a document through remend, so +// these tests assemble documents from COMPLETE constructs and derive all +// truncation from the streaming cut. The atoms deliberately include +// identifiers with double-underscore runs (snake__case style), which look +// like emphasis delimiters to a context-free counter. + +const INLINE_ATOMS = [ + "plain words", + "**bold text**", + "*italic text*", + "_underscore italic_", + "__strong text__", + "___strong italic___", + "~~struck text~~", + "`inline code`", + "``double `tick` span``", + "snake__case", + "user__id", + "a_b_c", + "[label](https://example.com)", + "![alt](https://example.com/pic.png)", + "tag", + "value is 20~25 degrees", +]; + +const BLOCK_ATOMS = [ + "# heading", + "> a quote", + "- item one\n- item two", + "```js\nconst total__count = 1;\n```", + "~~~\ntilde fenced\n~~~", + "$$\nx^2\n$$", +]; + +// The longest fragment healing may legitimately drop: an incomplete +// construct cut mid-stream is at most one atom long, plus the stripped +// trailing space +const MAX_LOSS = + Math.max(...[...INLINE_ATOMS, ...BLOCK_ATOMS].map((a) => a.length)) + 2; + +// Length of the longest prefix of `input` that appears as a subsequence of +// `output`. Healing may insert characters (closers, escapes) and drop a +// trailing fragment, so authored text is preserved exactly when everything +// except a bounded tail survives as a subsequence. +const preservedPrefixLength = (input: string, output: string): number => { + let matched = 0; + for (let i = 0; i < output.length && matched < input.length; i += 1) { + if (output[i] === input[matched]) { + matched += 1; + } + } + return matched; +}; + +const assertStreamingSafe = (prefix: string): void => { + const healed = remend(prefix); + + const loss = prefix.length - preservedPrefixLength(prefix, healed); + if (loss > MAX_LOSS) { + throw new Error( + `healing dropped ${loss} chars of ${JSON.stringify(prefix)} -> ${JSON.stringify(healed)}` + ); + } + + const rehealed = remend(healed); + if (rehealed !== healed) { + throw new Error( + `healing is not idempotent: ${JSON.stringify(prefix)} -> ${JSON.stringify(healed)} -> ${JSON.stringify(rehealed)}` + ); + } +}; + +// A document is a sequence of atoms. Inline atoms join with spaces into +// paragraphs, block atoms stand alone, and everything joins with blank lines +// so fences and headings begin at a line start. +const documentArbitrary = fc + .array( + fc.oneof( + { weight: 3, arbitrary: fc.subarray(INLINE_ATOMS, { minLength: 1 }) }, + { weight: 1, arbitrary: fc.constantFrom(...BLOCK_ATOMS).map((a) => [a]) } + ), + { minLength: 1, maxLength: 6 } + ) + .map((groups) => groups.map((atoms) => atoms.join(" ")).join("\n\n")); + +describe("streaming properties", () => { + it("preserves authored text and re-heals to itself on every cut", () => { + fc.assert( + fc.property(documentArbitrary, fc.nat(), (doc, cutSeed) => { + const cut = cutSeed % (doc.length + 1); + assertStreamingSafe(doc.slice(0, cut)); + }), + { numRuns: 2000 } + ); + }); + + it("does not modify complete documents", () => { + // Escape-oriented handlers (comparison operators, single tilde between + // digits) intentionally rewrite complete text, so their trigger shapes + // are excluded here + const noOpAtoms = INLINE_ATOMS.filter((atom) => !atom.includes("20~25")); + const noOpDocArbitrary = fc + .array( + fc.oneof( + { weight: 3, arbitrary: fc.subarray(noOpAtoms, { minLength: 1 }) }, + { + weight: 1, + arbitrary: fc.constantFrom(...BLOCK_ATOMS).map((a) => [a]), + } + ), + { minLength: 1, maxLength: 6 } + ) + .map((groups) => groups.map((atoms) => atoms.join(" ")).join("\n\n")); + + fc.assert( + fc.property(noOpDocArbitrary, (doc) => { + expect(remend(doc)).toBe(doc); + }), + { numRuns: 1000 } + ); + }); +}); + +describe("exhaustive prefix sweep", () => { + // Every prefix of a fixed corpus, deterministically. The corpus mixes the + // constructs whose interactions have bitten before: identifiers with + // double underscores next to real emphasis, fences of both characters, + // spans with multi-backtick runs, and half-typed closers. + const corpus = [ + "Use snake__case for names and __bold text__ throughout.", + "The `obj__attr` field pairs with **bold** and _italic_ text.", + "```python\ndef f():\n return a__b\n```\n\nAfter the fence __open", + "~~~\ntilde __fence\n~~~\n\n~~struck~~ and more", + "A [link](https://example.com) and ![img](https://example.com/a.png) done.", + "Math $$\nx^2 + y^2\n$$ and `code` mixed with ___strong italic___.", + "| a | b |\n| - | - |\n| 1 | 2 |\n\nTable then **bold**", + "Nested **bold with *italic* inside** plus ``double `tick` span``.", + ]; + + it("preserves authored text and re-heals to itself on every prefix", () => { + for (const doc of corpus) { + for (let cut = 0; cut <= doc.length; cut += 1) { + assertStreamingSafe(doc.slice(0, cut)); + } + } + }); +}); diff --git a/packages/remend/__tests__/underscore-runs.test.ts b/packages/remend/__tests__/underscore-runs.test.ts new file mode 100644 index 00000000..a2648c9d --- /dev/null +++ b/packages/remend/__tests__/underscore-runs.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from "vitest"; +import remend from "../src"; + +// Double underscores are counted per maximal run with flanking rules, so +// identifiers containing __ (snake__case style) neither invent nor swallow +// emphasis delimiters. +describe("word-internal double underscores", () => { + it("should not treat an identifier's __ as a delimiter", () => { + expect(remend("fields user__id and org__id are join keys")).toBe( + "fields user__id and org__id are join keys" + ); + }); + + it("should still close an opener when an identifier follows", () => { + // Counting raw __ occurrences would pair the identifier's run against + // the opener and swallow the closer that is still needed + expect(remend("Use snake__case and __bold")).toBe( + "Use snake__case and __bold__" + ); + }); + + it("should not invent a closer for a lone identifier", () => { + expect(remend("the value of some__field is set")).toBe( + "the value of some__field is set" + ); + }); + + it("should ignore identifiers inside complete inline code", () => { + expect(remend("`obj__attr` and __bold")).toBe("`obj__attr` and __bold__"); + }); + + it("should ignore identifiers inside bold content", () => { + expect(remend("**bold snake__case text** and more")).toBe( + "**bold snake__case text** and more" + ); + }); +}); + +describe("underscore run lengths", () => { + it("should treat a run of four as balanced", () => { + expect(remend("a ____ b")).toBe("a ____ b"); + }); + + it("should not complete a thematic break line", () => { + expect(remend("text\n\n___\n")).toBe("text\n\n___\n"); + }); + + it("should keep ___text___ balanced", () => { + expect(remend("___both___ done")).toBe("___both___ done"); + }); +}); diff --git a/packages/remend/package.json b/packages/remend/package.json index 44366e90..9d2c5329 100644 --- a/packages/remend/package.json +++ b/packages/remend/package.json @@ -34,6 +34,7 @@ "description": "Self-healing markdown. Intelligently parses and styles incomplete Markdown blocks.", "devDependencies": { "@vitest/coverage-v8": "^4.0.15", + "fast-check": "^4.9.0", "mdast-util-from-markdown": "^2.0.2", "tsup": "^8.5.1", "vitest": "^4.0.15" diff --git a/packages/remend/src/code-block-utils.ts b/packages/remend/src/code-block-utils.ts index ca3de4bd..53938857 100644 --- a/packages/remend/src/code-block-utils.ts +++ b/packages/remend/src/code-block-utils.ts @@ -1,96 +1,13 @@ -// Check if a position is inside a code block (between ``` or `) -export const isInsideCodeBlock = (text: string, position: number): boolean => { - // Check for inline code (backticks) - let inInlineCode = false; - let inMultilineCode = false; +import { getScan, isCodeAt, isCompleteSpanAt } from "./scan"; - for (let i = 0; i < position; i += 1) { - // Skip escaped backticks - if (text[i] === "\\" && i + 1 < text.length && text[i + 1] === "`") { - i += 1; - continue; - } +// Check if a position is inside a code construct (fenced block or inline span) +export const isInsideCodeBlock = (text: string, position: number): boolean => + isCodeAt(getScan(text), position); - // Check for triple backticks (multiline code blocks) - if (text.substring(i, i + 3) === "```") { - inMultilineCode = !inMultilineCode; - i += 2; // Skip the next 2 backticks - continue; - } - - // Only check for inline code if not in multiline code - if (!inMultilineCode && text[i] === "`") { - inInlineCode = !inInlineCode; - } - } - - return inInlineCode || inMultilineCode; -}; - -// Checks if a backtick at position i is part of a triple backtick sequence -export const isPartOfTripleBacktick = (text: string, i: number): boolean => { - const isTripleStart = text.substring(i, i + 3) === "```"; - const isTripleMiddle = i > 0 && text.substring(i - 1, i + 2) === "```"; - const isTripleEnd = i > 1 && text.substring(i - 2, i + 1) === "```"; - - return isTripleStart || isTripleMiddle || isTripleEnd; -}; - -// Counts single backticks that are not part of triple backticks or escaped -export const countSingleBackticks = (text: string): number => { - let count = 0; - for (let i = 0; i < text.length; i += 1) { - // Skip escaped backticks - if (text[i] === "\\" && i + 1 < text.length && text[i + 1] === "`") { - i += 1; - continue; - } - if (text[i] === "`" && !isPartOfTripleBacktick(text, i)) { - count += 1; - } - } - return count; -}; - -// Check if a position is inside a COMPLETE inline code span (both opening and closing backtick present). -// Returns false for incomplete inline code spans (streaming) so emphasis markers can still be completed. +// Check if a position is within a COMPLETE inline code span (both opening and +// closing backtick runs present). Returns false for incomplete spans +// (streaming) so emphasis markers can still be completed. export const isWithinCompleteInlineCode = ( text: string, position: number -): boolean => { - let inInlineCode = false; - let inMultilineCode = false; - let inlineCodeStart = -1; - - for (let i = 0; i < text.length; i += 1) { - // Skip escaped backticks - if (text[i] === "\\" && i + 1 < text.length && text[i + 1] === "`") { - i += 1; - continue; - } - - // Check for triple backticks (multiline code blocks) - if (text.substring(i, i + 3) === "```") { - inMultilineCode = !inMultilineCode; - i += 2; - continue; - } - - // Only check for inline code if not in multiline code - if (!inMultilineCode && text[i] === "`") { - if (inInlineCode) { - // Found closing backtick — check if position is inside this complete span - if (inlineCodeStart < position && position < i) { - return true; - } - inInlineCode = false; - inlineCodeStart = -1; - } else { - inInlineCode = true; - inlineCodeStart = i; - } - } - } - - return false; -}; +): boolean => isCompleteSpanAt(getScan(text), position); diff --git a/packages/remend/src/emphasis-handlers.ts b/packages/remend/src/emphasis-handlers.ts index 4c863a46..05fec023 100644 --- a/packages/remend/src/emphasis-handlers.ts +++ b/packages/remend/src/emphasis-handlers.ts @@ -14,16 +14,18 @@ import { whitespaceOrMarkersPattern, } from "./patterns"; import { - isHorizontalRule, - isWithinHtmlTag, - isWithinLinkOrImageUrl, - isWithinMathBlock, - isWordChar, -} from "./utils"; + getScan, + inHtmlTagAt, + inLinkUrlAt, + inMathAt, + REGION, + type TextScan, +} from "./scan"; +import { isHorizontalRule, isWordChar } from "./utils"; // Helper function to check if an asterisk should be skipped const shouldSkipAsterisk = ( - text: string, + scan: TextScan, index: number, prevChar: string, nextChar: string @@ -33,9 +35,8 @@ const shouldSkipAsterisk = ( return true; } - // Skip if within math block (only check if text has dollar signs) - const hasMathBlocks = text.includes("$"); - if (hasMathBlocks && isWithinMathBlock(text, index)) { + // Skip if within math block + if (inMathAt(scan, index)) { return true; } @@ -43,7 +44,8 @@ const shouldSkipAsterisk = ( // If this is the first * in ***, don't skip it - it can close a single * italic // Example: **bold and *italic*** should count the first * of *** as closing the italic if (prevChar !== "*" && nextChar === "*") { - const nextNextChar = index < text.length - 2 ? text[index + 2] : ""; + const nextNextChar = + index < scan.text.length - 2 ? scan.text[index + 2] : ""; if (nextNextChar === "*") { // This is the first * in a *** sequence // Count it as a single asterisk for matching purposes @@ -76,40 +78,22 @@ const shouldSkipAsterisk = ( return false; }; -// OPTIMIZATION: Counts single asterisks without split("").reduce() -// Counts single asterisks that are not part of double asterisks, not escaped, not list markers, not word-internal, -// and not inside fenced code blocks +// Counts asterisks that can serve as single-emphasis delimiters, applying +// the skip rules in shouldSkipAsterisk and excluding code regions export const countSingleAsterisks = (text: string): number => { + const scan = getScan(text); let count = 0; - let inCodeBlock = false; const len = text.length; for (let index = 0; index < len; index += 1) { - // Track fenced code blocks (```) - if ( - text[index] === "`" && - index + 2 < len && - text[index + 1] === "`" && - text[index + 2] === "`" - ) { - inCodeBlock = !inCodeBlock; - index += 2; - continue; - } - - // Skip content inside fenced code blocks - if (inCodeBlock) { - continue; - } - - if (text[index] !== "*") { + if (text[index] !== "*" || scan.regions[index] !== REGION.PROSE) { continue; } const prevChar = index > 0 ? text[index - 1] : ""; const nextChar = index < len - 1 ? text[index + 1] : ""; - if (!shouldSkipAsterisk(text, index, prevChar, nextChar)) { + if (!shouldSkipAsterisk(scan, index, prevChar, nextChar)) { count += 1; } } @@ -119,7 +103,7 @@ export const countSingleAsterisks = (text: string): number => { // Helper function to check if an underscore should be skipped const shouldSkipUnderscore = ( - text: string, + scan: TextScan, index: number, prevChar: string, nextChar: string @@ -129,19 +113,18 @@ const shouldSkipUnderscore = ( return true; } - // Skip if within math block (only check if text has dollar signs) - const hasMathBlocks = text.includes("$"); - if (hasMathBlocks && isWithinMathBlock(text, index)) { + // Skip if within math block + if (inMathAt(scan, index)) { return true; } // Skip if within a link or image URL - if (isWithinLinkOrImageUrl(text, index)) { + if (inLinkUrlAt(scan, index)) { return true; } // Skip if within an HTML tag (e.g. ) - if (isWithinHtmlTag(text, index)) { + if (inHtmlTagAt(scan, index)) { return true; } @@ -158,40 +141,22 @@ const shouldSkipUnderscore = ( return false; }; -// OPTIMIZATION: Counts single underscores without split("").reduce() -// Counts single underscores that are not part of double underscores, not escaped, not in math blocks, -// and not inside fenced code blocks +// Counts underscores that can serve as single-emphasis delimiters, applying +// the skip rules in shouldSkipUnderscore and excluding code regions export const countSingleUnderscores = (text: string): number => { + const scan = getScan(text); let count = 0; - let inCodeBlock = false; const len = text.length; for (let index = 0; index < len; index += 1) { - // Track fenced code blocks (```) - if ( - text[index] === "`" && - index + 2 < len && - text[index + 1] === "`" && - text[index + 2] === "`" - ) { - inCodeBlock = !inCodeBlock; - index += 2; - continue; - } - - // Skip content inside fenced code blocks - if (inCodeBlock) { - continue; - } - - if (text[index] !== "_") { + if (text[index] !== "_" || scan.regions[index] !== REGION.PROSE) { continue; } const prevChar = index > 0 ? text[index - 1] : ""; const nextChar = index < len - 1 ? text[index + 1] : ""; - if (!shouldSkipUnderscore(text, index, prevChar, nextChar)) { + if (!shouldSkipUnderscore(scan, index, prevChar, nextChar)) { count += 1; } } @@ -200,37 +165,14 @@ export const countSingleUnderscores = (text: string): number => { }; // Counts triple asterisks that are not part of quadruple or more asterisks -// and not inside fenced code blocks -// OPTIMIZATION: Count *** without regex to avoid allocation +// and not inside code regions export const countTripleAsterisks = (text: string): number => { + const scan = getScan(text); let count = 0; let consecutiveAsterisks = 0; - let inCodeBlock = false; for (let i = 0; i < text.length; i += 1) { - // Track fenced code blocks (```) - if ( - text[i] === "`" && - i + 2 < text.length && - text[i + 1] === "`" && - text[i + 2] === "`" - ) { - // Flush any pending asterisks before toggling - if (consecutiveAsterisks >= 3) { - count += Math.floor(consecutiveAsterisks / 3); - } - consecutiveAsterisks = 0; - inCodeBlock = !inCodeBlock; - i += 2; - continue; - } - - // Skip content inside fenced code blocks - if (inCodeBlock) { - continue; - } - - if (text[i] === "*") { + if (text[i] === "*" && scan.regions[i] === REGION.PROSE) { consecutiveAsterisks += 1; } else { // End of asterisk sequence @@ -249,23 +191,13 @@ export const countTripleAsterisks = (text: string): number => { return count; }; -// Counts ** pairs outside fenced code blocks -const countDoubleAsterisksOutsideCodeBlocks = (text: string): number => { +// Counts ** pairs outside code regions +const countDoubleAsterisks = (text: string): number => { + const scan = getScan(text); let count = 0; - let inCodeBlock = false; for (let i = 0; i < text.length; i += 1) { - if ( - text[i] === "`" && - i + 2 < text.length && - text[i + 1] === "`" && - text[i + 2] === "`" - ) { - inCodeBlock = !inCodeBlock; - i += 2; - continue; - } - if (inCodeBlock) { + if (scan.regions[i] !== REGION.PROSE) { continue; } if (text[i] === "*" && i + 1 < text.length && text[i + 1] === "*") { @@ -276,31 +208,119 @@ const countDoubleAsterisksOutsideCodeBlocks = (text: string): number => { return count; }; -// Counts __ pairs outside fenced code blocks -const countDoubleUnderscoresOutsideCodeBlocks = (text: string): number => { - let count = 0; - let inCodeBlock = false; +// Whether the text has an unmatched __ delimiter, counted per maximal +// underscore run rather than per raw __ occurrence. +// +// Counting raw occurrences misreads identifiers: a name like snake__case +// contains __ but cannot open or close emphasis, and counting it either +// invents a closer (odd count) or pairs it against a real delimiter and +// swallows a closer that was needed (even count). Per run: +// +// - A run contributes floor(length / 2) pairs, and flips delimiter parity +// only when that is odd. __ and ___ flip; ____ does not. +// - A word-internal run (word characters on both sides) is part of an +// identifier, never a delimiter. +// - Runs inside code regions, math, link URLs, and HTML tags are skipped, +// matching the single-underscore handler's flanking rules. +// - A run alone on its line is a thematic break, not emphasis. +const isLineBoundaryChar = (char: string): boolean => + char === "" || char === " " || char === "\t" || char === "\n"; + +// isHorizontalRule scans the run's whole line, so its verdict is memoized +// per line to keep run counting linear when many runs share a line +interface ThematicBreakMemo { + lineEnd: number; + result: boolean; +} + +const isThematicBreakRun = ( + text: string, + runStart: number, + prevChar: string, + nextChar: string, + memo: ThematicBreakMemo +): boolean => { + // A thematic break line holds only markers and whitespace, so only runs + // flanked by whitespace or line boundaries can be part of one + if (!(isLineBoundaryChar(prevChar) && isLineBoundaryChar(nextChar))) { + return false; + } + if (runStart > memo.lineEnd) { + const lineEnd = text.indexOf("\n", runStart); + memo.lineEnd = lineEnd === -1 ? text.length : lineEnd; + memo.result = isHorizontalRule(text, runStart, "_"); + } + return memo.result; +}; - for (let i = 0; i < text.length; i += 1) { - if ( - text[i] === "`" && - i + 2 < text.length && - text[i + 1] === "`" && - text[i + 2] === "`" - ) { - inCodeBlock = !inCodeBlock; - i += 2; +// Whether an underscore run at [runStart, runEnd) flips delimiter parity +const doubleUnderscoreRunFlips = ( + scan: TextScan, + initialRunStart: number, + runEnd: number, + memo: ThematicBreakMemo +): boolean => { + const text = scan.text; + + // A backslash escapes the first underscore of the run + let runStart = initialRunStart; + if (runStart > 0 && text[runStart - 1] === "\\") { + runStart += 1; + } + const runLength = runEnd - runStart; + if (runLength < 2) { + return false; + } + + const prevChar = runStart > 0 ? text[runStart - 1] : ""; + const nextChar = runEnd < text.length ? text[runEnd] : ""; + if (isWordChar(prevChar) && isWordChar(nextChar)) { + return false; + } + if (isThematicBreakRun(text, runStart, prevChar, nextChar, memo)) { + return false; + } + if ( + inMathAt(scan, runStart) || + inLinkUrlAt(scan, runStart) || + inHtmlTagAt(scan, runStart) + ) { + return false; + } + + return Math.floor(runLength / 2) % 2 === 1; +}; + +const hasUnmatchedDoubleUnderscore = (text: string): boolean => { + const scan = getScan(text); + const n = text.length; + const memo: ThematicBreakMemo = { lineEnd: -1, result: false }; + let unmatched = false; + let i = 0; + + while (i < n) { + if (text[i] !== "_" || scan.regions[i] !== REGION.PROSE) { + i += 1; continue; } - if (inCodeBlock) { - continue; + + const runStart = i; + let runEnd = i + 1; + while ( + runEnd < n && + text[runEnd] === "_" && + scan.regions[runEnd] === REGION.PROSE + ) { + runEnd += 1; } - if (text[i] === "_" && i + 1 < text.length && text[i + 1] === "_") { - count += 1; - i += 1; + i = runEnd; + + if (doubleUnderscoreRunFlips(scan, runStart, runEnd, memo)) { + unmatched = !unmatched; } } - return count; + + return unmatched; }; // Helper to check if bold marker should not be completed @@ -355,7 +375,7 @@ export const handleIncompleteBold = (text: string): string => { return text; } - const asteriskPairs = countDoubleAsterisksOutsideCodeBlocks(text); + const asteriskPairs = countDoubleAsterisks(text); if (asteriskPairs % 2 === 1) { // Check for half-complete closing marker: **content* should become **content** // The trailing * is the first char of the closing ** being streamed @@ -414,12 +434,10 @@ export const handleIncompleteDoubleUnderscoreItalic = ( !( isInsideCodeBlock(text, markerIndex) || isWithinCompleteInlineCode(text, markerIndex) - ) + ) && + hasUnmatchedDoubleUnderscore(text) ) { - const underscorePairs = countDoubleUnderscoresOutsideCodeBlocks(text); - if (underscorePairs % 2 === 1) { - return `${text}_`; - } + return `${text}_`; } } return text; @@ -440,43 +458,25 @@ export const handleIncompleteDoubleUnderscoreItalic = ( return text; } - const underscorePairs = countDoubleUnderscoresOutsideCodeBlocks(text); - if (underscorePairs % 2 === 1) { + if (hasUnmatchedDoubleUnderscore(text)) { return `${text}__`; } return text; }; -// Helper function to find the first single asterisk index (skips fenced code blocks) -// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: asterisk detection requires many inline conditions +// Helper function to find the first single asterisk index (skips code regions) const findFirstSingleAsteriskIndex = (text: string): number => { - let inCodeBlock = false; + const scan = getScan(text); for (let i = 0; i < text.length; i += 1) { - // Track fenced code blocks (```) - if ( - text[i] === "`" && - i + 2 < text.length && - text[i + 1] === "`" && - text[i + 2] === "`" - ) { - inCodeBlock = !inCodeBlock; - i += 2; - continue; - } - - // Skip content inside fenced code blocks - if (inCodeBlock) { - continue; - } - if ( text[i] === "*" && + scan.regions[i] === REGION.PROSE && text[i - 1] !== "*" && text[i + 1] !== "*" && text[i - 1] !== "\\" && - !isWithinMathBlock(text, i) + !inMathAt(scan, i) ) { const prevChar = i > 0 ? text[i - 1] : ""; const nextChar = i < text.length - 1 ? text[i + 1] : ""; @@ -550,35 +550,19 @@ export const handleIncompleteSingleAsteriskItalic = (text: string): string => { return text; }; -// Helper function to find the first single underscore index (skips fenced code blocks) +// Helper function to find the first single underscore index (skips code regions) const findFirstSingleUnderscoreIndex = (text: string): number => { - let inCodeBlock = false; + const scan = getScan(text); for (let i = 0; i < text.length; i += 1) { - // Track fenced code blocks (```) - if ( - text[i] === "`" && - i + 2 < text.length && - text[i + 1] === "`" && - text[i + 2] === "`" - ) { - inCodeBlock = !inCodeBlock; - i += 2; - continue; - } - - // Skip content inside fenced code blocks - if (inCodeBlock) { - continue; - } - if ( text[i] === "_" && + scan.regions[i] === REGION.PROSE && text[i - 1] !== "_" && text[i + 1] !== "_" && text[i - 1] !== "\\" && - !isWithinMathBlock(text, i) && - !isWithinLinkOrImageUrl(text, i) + !inMathAt(scan, i) && + !inLinkUrlAt(scan, i) ) { // Check if underscore is word-internal (between word characters) const prevChar = i > 0 ? text[i - 1] : ""; @@ -621,7 +605,7 @@ const handleTrailingAsterisksForUnderscore = (text: string): string | null => { } const textWithoutTrailingAsterisks = text.slice(0, -2); - const asteriskPairsAfterRemoval = countDoubleAsterisksOutsideCodeBlocks( + const asteriskPairsAfterRemoval = countDoubleAsterisks( textWithoutTrailingAsterisks ); @@ -699,7 +683,7 @@ export const handleIncompleteSingleUnderscoreItalic = ( // Helper to check if bold-italic markers are already balanced const areBoldItalicMarkersBalanced = (text: string): boolean => { - const asteriskPairs = countDoubleAsterisksOutsideCodeBlocks(text); + const asteriskPairs = countDoubleAsterisks(text); const singleAsterisks = countSingleAsterisks(text); return asteriskPairs % 2 === 0 && singleAsterisks % 2 === 0; }; diff --git a/packages/remend/src/index.ts b/packages/remend/src/index.ts index 005c9fea..9d606d18 100644 --- a/packages/remend/src/index.ts +++ b/packages/remend/src/index.ts @@ -305,6 +305,13 @@ const remend = (text: string, options?: RemendOptions): string => { } } + // A handler that removes a trailing fragment can expose a trailing space + // (e.g. dropping an incomplete image). Strip it the same way the input + // was stripped, so healed output re-heals to itself. + if (result.endsWith(" ") && !result.endsWith(" ")) { + return result.slice(0, -1); + } + return result; }; diff --git a/packages/remend/src/inline-code-handler.ts b/packages/remend/src/inline-code-handler.ts index b77c2c32..ed079a79 100644 --- a/packages/remend/src/inline-code-handler.ts +++ b/packages/remend/src/inline-code-handler.ts @@ -1,61 +1,45 @@ -import { countSingleBackticks } from "./code-block-utils"; -import { - inlineCodePattern, - inlineTripleBacktickPattern, - whitespaceOrMarkersPattern, -} from "./patterns"; +import { whitespaceOrMarkersPattern } from "./patterns"; +import { getScan } from "./scan"; + +// Completes an unclosed inline code span (`) +// +// A span opened by a run of N backticks closes only on a run of exactly N, +// so the completion appends whatever remains of the closing run. If the text +// already ends with a partial closing run of k < N backticks, only N - k are +// appended. +export const handleIncompleteInlineCode = (text: string): string => { + const scan = getScan(text); -// Helper function to check for incomplete inline triple backticks -const handleInlineTripleBackticks = (text: string): string | null => { - const inlineTripleBacktickMatch = text.match(inlineTripleBacktickPattern); - if (!inlineTripleBacktickMatch || text.includes("\n")) { - return null; + // Inside an unterminated fenced code block, backticks are content and the + // block is left for the renderer to display as streaming code + if (scan.openFence) { + return text; } - // Check if it ends with exactly 2 backticks (incomplete) - if (text.endsWith("``") && !text.endsWith("```")) { - return `${text}\``; + const span = scan.openSpan; + if (!span) { + return text; } - // Already complete inline triple backticks - return text; -}; - -// Helper function to check if we're inside an incomplete code block -const isInsideIncompleteCodeBlock = (text: string): boolean => { - const allTripleBackticks = (text.match(/```/g) || []).length; - return allTripleBackticks % 2 === 1; -}; -// Completes incomplete inline code formatting (`) -// Avoids completing if inside an incomplete code block -export const handleIncompleteInlineCode = (text: string): string => { - // Check if we have inline triple backticks (starts with ``` and should end with ```) - // This pattern should ONLY match truly inline code (no newlines) - // Examples: ```code``` or ```python code``` - const inlineResult = handleInlineTripleBackticks(text); - if (inlineResult !== null) { - return inlineResult; + // Don't close if there's no meaningful content after the opening run + const content = text.slice(span.start + span.runLength); + if (!content || whitespaceOrMarkersPattern.test(content)) { + return text; } - const inlineCodeMatch = text.match(inlineCodePattern); - - if (inlineCodeMatch && !isInsideIncompleteCodeBlock(text)) { - // Don't close if there's no meaningful content after the opening marker - // inlineCodeMatch[2] contains the content after ` - // Check if content is only whitespace or other emphasis markers - const contentAfterMarker = inlineCodeMatch[2]; - if ( - !contentAfterMarker || - whitespaceOrMarkersPattern.test(contentAfterMarker) - ) { - return text; - } + // A trailing backtick run is the closing run being streamed + let trailingRun = 0; + let i = text.length - 1; + while (i >= 0 && text[i] === "`") { + trailingRun += 1; + i -= 1; + } - const singleBacktickCount = countSingleBackticks(text); - if (singleBacktickCount % 2 === 1) { - return `${text}\``; - } + // A trailing run at least as long as the opener is a literal run inside + // the span. Appending backticks would only extend it, never close the span. + if (trailingRun >= span.runLength) { + return text; } - return text; + return text + "`".repeat(span.runLength - trailingRun); }; diff --git a/packages/remend/src/katex-handler.ts b/packages/remend/src/katex-handler.ts index 765542f1..b550bf1d 100644 --- a/packages/remend/src/katex-handler.ts +++ b/packages/remend/src/katex-handler.ts @@ -1,20 +1,15 @@ -// Helper function to check if a backtick is part of a triple backtick -const isTripleBacktick = (text: string, index: number): boolean => - (index >= 2 && text.substring(index - 2, index + 1) === "```") || - (index >= 1 && text.substring(index - 1, index + 2) === "```") || - (index <= text.length - 3 && text.substring(index, index + 3) === "```"); +import { getScan, REGION } from "./scan"; -// Helper function to count $$ pairs outside of inline code blocks +// Helper function to count $$ pairs outside code regions const countDollarPairs = (text: string): number => { + const scan = getScan(text); let dollarPairs = 0; - let inInlineCode = false; for (let i = 0; i < text.length - 1; i += 1) { - if (text[i] === "`" && !isTripleBacktick(text, i)) { - inInlineCode = !inInlineCode; + if (scan.regions[i] !== REGION.PROSE) { + continue; } - - if (!inInlineCode && text[i] === "$" && text[i + 1] === "$") { + if (text[i] === "$" && text[i + 1] === "$") { dollarPairs += 1; i += 1; } @@ -23,10 +18,10 @@ const countDollarPairs = (text: string): number => { return dollarPairs; }; -// Helper function to count single $ signs (excluding $$) outside of code blocks +// Helper function to count single $ signs (excluding $$) outside code regions const countSingleDollars = (text: string): number => { + const scan = getScan(text); let count = 0; - let inInlineCode = false; for (let i = 0; i < text.length; i += 1) { if (text[i] === "\\") { @@ -34,12 +29,11 @@ const countSingleDollars = (text: string): number => { continue; } - if (text[i] === "`" && !isTripleBacktick(text, i)) { - inInlineCode = !inInlineCode; + if (scan.regions[i] !== REGION.PROSE) { continue; } - if (!inInlineCode && text[i] === "$") { + if (text[i] === "$") { if (i + 1 < text.length && text[i + 1] === "$") { i += 1; } else { diff --git a/packages/remend/src/link-image-handler.ts b/packages/remend/src/link-image-handler.ts index 067fd45d..df236370 100644 --- a/packages/remend/src/link-image-handler.ts +++ b/packages/remend/src/link-image-handler.ts @@ -131,11 +131,8 @@ const handleIncompleteText = ( return null; }; -// Handles incomplete links and images by preserving them with a special marker -export const handleIncompleteLinksAndImages = ( - text: string, - linkMode: LinkMode = "protocol" -): string => { +// One healing step over the trailing incomplete link or image, if any +const healTrailingLinkOrImage = (text: string, linkMode: LinkMode): string => { // Look for patterns like [text]( or ![text]( at the end of text // We need to handle nested brackets in the link text @@ -161,3 +158,25 @@ export const handleIncompleteLinksAndImages = ( return text; }; + +// Handles incomplete links and images by preserving them with a special marker +export const handleIncompleteLinksAndImages = ( + text: string, + linkMode: LinkMode = "protocol" +): string => { + let current = text; + + // Removing a trailing incomplete image can expose another incomplete + // construct that ended immediately before it (an input ending in "![![" + // heals to "![", which is itself incomplete). Iterate until healed text + // re-heals to itself, so healing is idempotent. A step that grows the text + // has completed the construct, and truncating steps strictly shorten, + // so the loop terminates. + for (;;) { + const next = healTrailingLinkOrImage(current, linkMode); + if (next.length >= current.length) { + return next; + } + current = next; + } +}; diff --git a/packages/remend/src/patterns.ts b/packages/remend/src/patterns.ts index 50700792..9e57b07e 100644 --- a/packages/remend/src/patterns.ts +++ b/packages/remend/src/patterns.ts @@ -3,16 +3,10 @@ export const italicPattern = /(__)([^_]*?)$/; export const boldItalicPattern = /(\*\*\*)([^*]*?)$/; export const singleAsteriskPattern = /(\*)([^*]*?)$/; export const singleUnderscorePattern = /(_)([^_]*?)$/; -export const inlineCodePattern = /(`)([^`]*?)$/; export const strikethroughPattern = /(~~)([^~]*?)$/; export const whitespaceOrMarkersPattern = /^[\s_~*`]*$/; export const listItemPattern = /^[\s]*[-*+][\s]+$/; export const letterNumberUnderscorePattern = /[\p{L}\p{N}_]/u; -export const inlineTripleBacktickPattern = /^```[^`\n]*```?$/; export const fourOrMoreAsterisksPattern = /^\*{4,}$/; -export const linkImagePattern = /(!?\[)([^\]]*?)$/; -export const incompleteLinkUrlPattern = /(!?)\[([^\]]+)\](\([^)]+)$/; export const halfCompleteUnderscorePattern = /(__)([^_]+)_$/; export const halfCompleteTildePattern = /(~~)([^~]+)~$/; -export const doubleUnderscoreGlobalPattern = /__/g; -export const doubleTildeGlobalPattern = /~~/g; diff --git a/packages/remend/src/scan.ts b/packages/remend/src/scan.ts new file mode 100644 index 00000000..5f7f1b6e --- /dev/null +++ b/packages/remend/src/scan.ts @@ -0,0 +1,424 @@ +// Single-pass classification of a text into code and prose regions. +// +// Handlers previously re-derived "am I inside code?" per candidate character +// with O(n) rescans, making remend quadratic on delimiter-heavy input. This +// module scans once per input string (memoized) so every position query is +// O(1), and centralizes fence semantics that were previously approximated +// with a context-free ``` toggle: +// +// - A fence only opens at the start of a line, indented at most 3 spaces. +// - Both ``` and ~~~ fences are recognized, with runs of 3 or more. +// - The info string of a backtick fence cannot contain a backtick +// (a line like ```code``` is inline code, not a fence). +// - A fence closes on a run of the same character at least as long as the +// opener, alone on its line. +// - An inline code span opened by a run of N backticks closes only on a run +// of exactly N backticks; other runs are literal inside the span. + +export const REGION = { + PROSE: 0, + /** The ``` or ~~~ run that opens or closes a fence */ + FENCE_MARKER: 1, + /** The info string on a fence opener line. Neither prose nor code body. */ + FENCE_INFO: 2, + /** Content of a fenced code block */ + FENCE_BODY: 3, + /** A complete inline code span, including its backtick markers */ + CODE_SPAN: 4, + /** An inline code span whose closing run has not arrived yet */ + CODE_SPAN_OPEN: 5, +} as const; + +export type Region = (typeof REGION)[keyof typeof REGION]; + +export interface OpenFence { + char: "`" | "~"; + /** Length of the opening run; a closer must be at least this long */ + length: number; +} + +export interface OpenSpan { + /** Length of the opening run; the closer must match it exactly */ + runLength: number; + /** Index of the first backtick of the opening run */ + start: number; +} + +export interface TextScan { + htmlTagMask: Uint8Array | null; + linkUrlMask: Uint8Array | null; + /** Lazily computed masks backing the inMathAt/inLinkUrlAt/inHtmlTagAt helpers */ + mathMask: Uint8Array | null; + /** Fence still open at end of text, if any */ + openFence: OpenFence | null; + /** Inline code span still open at end of text, if any */ + openSpan: OpenSpan | null; + regions: Uint8Array; + text: string; +} + +const FENCE_OPENER_PATTERN = /^( {0,3})(`{3,}|~{3,})(.*)$/; + +// Marks the fence regions of one opener line and returns the open fence state +const paintFenceOpener = ( + regions: Uint8Array, + lineStart: number, + lineEnd: number, + indentLength: number, + markerLength: number +): void => { + const markerStart = lineStart + indentLength; + regions.fill(REGION.FENCE_MARKER, markerStart, markerStart + markerLength); + // Info string plus the line terminator belong to the fence, not to prose + regions.fill( + REGION.FENCE_INFO, + markerStart + markerLength, + Math.min(lineEnd + 1, regions.length) + ); +}; + +// Whether a line inside an open fence closes it: at most 3 spaces of indent, +// then a run of the fence character at least as long as the opener, then +// only whitespace +const isFenceCloser = ( + text: string, + lineStart: number, + lineEnd: number, + fence: OpenFence +): boolean => { + let i = lineStart; + let indent = 0; + while (i < lineEnd && text[i] === " " && indent < 4) { + i += 1; + indent += 1; + } + if (indent > 3) { + return false; + } + let runLength = 0; + while (i < lineEnd && text[i] === fence.char) { + i += 1; + runLength += 1; + } + if (runLength < fence.length) { + return false; + } + while (i < lineEnd) { + if (text[i] !== " " && text[i] !== "\t") { + return false; + } + i += 1; + } + return true; +}; + +// First pass: paint fenced code blocks line by line +const paintFences = (text: string, regions: Uint8Array): OpenFence | null => { + const n = text.length; + let openFence: OpenFence | null = null; + let lineStart = 0; + + while (lineStart < n) { + let lineEnd = text.indexOf("\n", lineStart); + if (lineEnd === -1) { + lineEnd = n; + } + + if (openFence) { + if (isFenceCloser(text, lineStart, lineEnd, openFence)) { + regions.fill(REGION.FENCE_MARKER, lineStart, lineEnd); + openFence = null; + } else { + regions.fill(REGION.FENCE_BODY, lineStart, Math.min(lineEnd + 1, n)); + } + } else { + const line = text.slice(lineStart, lineEnd); + const opener = line.match(FENCE_OPENER_PATTERN); + if (opener) { + const markerChar = opener[2][0] as "`" | "~"; + // A backtick fence's info string cannot contain a backtick; such a + // line is inline code instead + if (markerChar === "~" || !opener[3].includes("`")) { + paintFenceOpener( + regions, + lineStart, + lineEnd, + opener[1].length, + opener[2].length + ); + openFence = { char: markerChar, length: opener[2].length }; + } + } + } + + lineStart = lineEnd + 1; + } + + return openFence; +}; + +const measureBacktickRun = (text: string, start: number): number => { + let end = start + 1; + while (end < text.length && text[end] === "`") { + end += 1; + } + return end; +}; + +// Second pass: paint inline code spans in the regions fences left as prose +const paintSpans = (text: string, regions: Uint8Array): OpenSpan | null => { + const n = text.length; + let spanStart = -1; + let spanRunLength = 0; + let i = 0; + + while (i < n) { + if (regions[i] !== REGION.PROSE) { + // A span cannot cross into a fence, so leave it marked open up to here + if (spanStart >= 0) { + regions.fill(REGION.CODE_SPAN_OPEN, spanStart, i); + spanStart = -1; + } + i += 1; + continue; + } + if (text[i] === "\\" && text[i + 1] === "`" && spanStart < 0) { + i += 2; + continue; + } + if (text[i] !== "`") { + i += 1; + continue; + } + + const runEnd = measureBacktickRun(text, i); + const runLength = runEnd - i; + if (spanStart < 0) { + spanStart = i; + spanRunLength = runLength; + } else if (runLength === spanRunLength) { + regions.fill(REGION.CODE_SPAN, spanStart, runEnd); + spanStart = -1; + } + // A run of a different length is literal inside the open span + i = runEnd; + } + + if (spanStart >= 0) { + regions.fill(REGION.CODE_SPAN_OPEN, spanStart, n); + return { start: spanStart, runLength: spanRunLength }; + } + return null; +}; + +const scanText = (text: string): TextScan => { + const regions = new Uint8Array(text.length); + const openFence = paintFences(text, regions); + const openSpan = paintSpans(text, regions); + return { + text, + regions, + openFence, + openSpan, + mathMask: null, + linkUrlMask: null, + htmlTagMask: null, + }; +}; + +// Memoize the most recent scan. Handlers query many positions of the same +// string, and remend's handler chain passes each handler's output to the +// next, so a single-entry cache gives O(1) queries within a handler while +// staying O(n) per handler overall. +let cachedText: string | null = null; +let cachedScan: TextScan | null = null; + +export const getScan = (text: string): TextScan => { + if (cachedScan !== null && text === cachedText) { + return cachedScan; + } + const scan = scanText(text); + cachedText = text; + cachedScan = scan; + return scan; +}; + +/** Whether the position is inside any code construct (fence or span) */ +export const isCodeAt = (scan: TextScan, position: number): boolean => { + if (position >= scan.regions.length) { + return scan.openFence !== null || scan.openSpan !== null; + } + if (position < 0) { + return false; + } + return scan.regions[position] !== REGION.PROSE; +}; + +/** Whether the position is inside a fenced code block (marker, info, or body) */ +export const isFenceAt = (scan: TextScan, position: number): boolean => { + if (position >= scan.regions.length) { + return scan.openFence !== null; + } + if (position < 0) { + return false; + } + const region = scan.regions[position]; + return ( + region === REGION.FENCE_MARKER || + region === REGION.FENCE_INFO || + region === REGION.FENCE_BODY + ); +}; + +/** Whether the position is inside a complete inline code span */ +export const isCompleteSpanAt = (scan: TextScan, position: number): boolean => + scan.regions[position] === REGION.CODE_SPAN; + +// Math mask: for each position, whether it is inside $...$ or $$...$$, +// mirroring the sequential toggle semantics the per-position helper used +const buildMathMask = (text: string): Uint8Array => { + const n = text.length; + const mask = new Uint8Array(n); + let inInlineMath = false; + let inBlockMath = false; + + let i = 0; + while (i < n) { + mask[i] = inInlineMath || inBlockMath ? 1 : 0; + if (text[i] === "\\" && text[i + 1] === "$") { + if (i + 1 < n) { + mask[i + 1] = mask[i]; + } + i += 2; + continue; + } + if (text[i] !== "$") { + i += 1; + continue; + } + if (text[i + 1] === "$") { + inBlockMath = !inBlockMath; + inInlineMath = false; + if (i + 1 < n) { + mask[i + 1] = 1; + } + i += 2; + continue; + } + if (!inBlockMath) { + inInlineMath = !inInlineMath; + } + i += 1; + } + + return mask; +}; + +export const inMathAt = (scan: TextScan, position: number): boolean => { + if (position < 0 || position >= scan.text.length) { + return false; + } + if (scan.mathMask === null) { + scan.mathMask = buildMathMask(scan.text); + } + return scan.mathMask[position] === 1; +}; + +// Marks the URL positions of one line: those between a "](" opener and the +// next ")" on the line. Two sub-passes: backward to know whether a ")" still +// follows a position, forward to know whether the nearest paren boundary +// before a position is a "](" opener. +const paintLinkUrlLine = ( + text: string, + lineStart: number, + lineEnd: number, + mask: Uint8Array +): void => { + // closerFollows[i - lineStart]: a ")" exists at or after i on this line + const closerFollows = new Uint8Array(lineEnd - lineStart); + let seenCloser = 0; + for (let i = lineEnd - 1; i >= lineStart; i -= 1) { + if (text[i] === ")") { + seenCloser = 1; + } + closerFollows[i - lineStart] = seenCloser; + } + + let inUrl = false; + for (let i = lineStart; i < lineEnd; i += 1) { + if (inUrl && closerFollows[i - lineStart] === 1) { + mask[i] = 1; + } + if (text[i] === ")") { + inUrl = false; + } else if (text[i] === "(") { + inUrl = i > 0 && text[i - 1] === "]"; + } + } +}; + +// Link/image URL mask: positions inside the (url) part of [text](url) +const buildLinkUrlMask = (text: string): Uint8Array => { + const n = text.length; + const mask = new Uint8Array(n); + let lineStart = 0; + + while (lineStart < n) { + let lineEnd = text.indexOf("\n", lineStart); + if (lineEnd === -1) { + lineEnd = n; + } + paintLinkUrlLine(text, lineStart, lineEnd, mask); + lineStart = lineEnd + 1; + } + + return mask; +}; + +export const inLinkUrlAt = (scan: TextScan, position: number): boolean => { + if (position < 0 || position >= scan.text.length) { + return false; + } + if (scan.linkUrlMask === null) { + scan.linkUrlMask = buildLinkUrlMask(scan.text); + } + return scan.linkUrlMask[position] === 1; +}; + +// HTML tag mask: positions after a "<" that begins a plausible tag (letter +// or /), through the closing ">" inclusive, within a single line +const buildHtmlTagMask = (text: string): Uint8Array => { + const n = text.length; + const mask = new Uint8Array(n); + let inTag = false; + + for (let i = 0; i < n; i += 1) { + if (text[i] === "\n") { + inTag = false; + continue; + } + mask[i] = inTag ? 1 : 0; + if (text[i] === ">") { + inTag = false; + } else if (text[i] === "<") { + const next = text[i + 1]; + inTag = + next !== undefined && + ((next >= "a" && next <= "z") || + (next >= "A" && next <= "Z") || + next === "/"); + } + } + + return mask; +}; + +export const inHtmlTagAt = (scan: TextScan, position: number): boolean => { + if (position < 0 || position >= scan.text.length) { + return false; + } + if (scan.htmlTagMask === null) { + scan.htmlTagMask = buildHtmlTagMask(scan.text); + } + return scan.htmlTagMask[position] === 1; +}; diff --git a/packages/remend/src/strikethrough-handler.ts b/packages/remend/src/strikethrough-handler.ts index bbb22dd4..e5a80910 100644 --- a/packages/remend/src/strikethrough-handler.ts +++ b/packages/remend/src/strikethrough-handler.ts @@ -3,11 +3,30 @@ import { isWithinCompleteInlineCode, } from "./code-block-utils"; import { - doubleTildeGlobalPattern, halfCompleteTildePattern, strikethroughPattern, whitespaceOrMarkersPattern, } from "./patterns"; +import { getScan, REGION } from "./scan"; + +// Counts ~~ pairs outside code regions. Tilde runs that open or close a +// fence are painted as fence regions by the scanner, so a line-start ~~~ +// fence never counts as strikethrough while a mid-line tilde run still does. +const countDoubleTildes = (text: string): number => { + const scan = getScan(text); + let count = 0; + + for (let i = 0; i < text.length; i += 1) { + if (scan.regions[i] !== REGION.PROSE) { + continue; + } + if (text[i] === "~" && i + 1 < text.length && text[i + 1] === "~") { + count += 1; + i += 1; + } + } + return count; +}; // Completes incomplete strikethrough formatting (~~) export const handleIncompleteStrikethrough = (text: string): string => { @@ -34,9 +53,7 @@ export const handleIncompleteStrikethrough = (text: string): string => { return text; } - // doubleTildeGlobalPattern always matches when strikethroughPattern matched - const tildePairs = text.match(doubleTildeGlobalPattern)?.length; - if (tildePairs % 2 === 1) { + if (countDoubleTildes(text) % 2 === 1) { return `${text}~~`; } } else { @@ -52,9 +69,7 @@ export const handleIncompleteStrikethrough = (text: string): string => { ) { return text; } - // doubleTildeGlobalPattern always matches when halfCompleteTildePattern matched - const tildePairs = text.match(doubleTildeGlobalPattern)?.length; - if (tildePairs % 2 === 1) { + if (countDoubleTildes(text) % 2 === 1) { return `${text}~`; } } diff --git a/packages/remend/src/utils.ts b/packages/remend/src/utils.ts index ac0a7d14..4d892fe1 100644 --- a/packages/remend/src/utils.ts +++ b/packages/remend/src/utils.ts @@ -1,4 +1,5 @@ import { letterNumberUnderscorePattern } from "./patterns"; +import { getScan, inHtmlTagAt, inLinkUrlAt, inMathAt, isFenceAt } from "./scan"; // OPTIMIZATION: Precompute which characters are word characters // Using ASCII fast path before falling back to Unicode regex @@ -20,20 +21,9 @@ export const isWordChar = (char: string): boolean => { return letterNumberUnderscorePattern.test(char); }; -// Check if a position is within a code block (between ``` markers) -export const isWithinCodeBlock = (text: string, position: number): boolean => { - let inCodeBlock = false; - - for (let i = 0; i < position; i += 1) { - // Check for triple backticks - if (text[i] === "`" && text[i + 1] === "`" && text[i + 2] === "`") { - inCodeBlock = !inCodeBlock; - i += 2; // Skip the next two backticks - } - } - - return inCodeBlock; -}; +// Check if a position is within a fenced code block +export const isWithinCodeBlock = (text: string, position: number): boolean => + isFenceAt(getScan(text), position); // Helper function to find the matching opening bracket for a closing bracket // Handles nested brackets correctly by searching backwards @@ -76,101 +66,20 @@ export const findMatchingClosingBracket = ( }; // Check if a position is within a math block (between $ or $$) -export const isWithinMathBlock = (text: string, position: number): boolean => { - // Count dollar signs before this position - let inInlineMath = false; - let inBlockMath = false; - - for (let i = 0; i < text.length && i < position; i += 1) { - // Skip escaped dollar signs - if (text[i] === "\\" && text[i + 1] === "$") { - i += 1; // Skip the next character - continue; - } - - if (text[i] === "$") { - // Check for block math ($$) - if (text[i + 1] === "$") { - inBlockMath = !inBlockMath; - i += 1; // Skip the second $ - inInlineMath = false; // Block math takes precedence - } else if (!inBlockMath) { - // Only toggle inline math if not in block math - inInlineMath = !inInlineMath; - } - } - } - - return inInlineMath || inBlockMath; -}; - -// Helper to check if position is before closing paren on same line -const isBeforeClosingParen = (text: string, position: number): boolean => { - for (let j = position; j < text.length; j += 1) { - if (text[j] === ")") { - return true; - } - if (text[j] === "\n") { - return false; - } - } - return false; -}; +export const isWithinMathBlock = (text: string, position: number): boolean => + inMathAt(getScan(text), position); // Check if a position is within a link or image URL // Links and images have the format [text](url) or ![alt](url) export const isWithinLinkOrImageUrl = ( text: string, position: number -): boolean => { - // Search backwards from position to find if we're inside a (url) part - for (let i = position - 1; i >= 0; i -= 1) { - if (text[i] === ")") { - return false; - } - if (text[i] === "(") { - // Check if there's a ] immediately before the ( - if (i > 0 && text[i - 1] === "]") { - // We're potentially inside a link/image URL - // Check if we're before the closing ) - return isBeforeClosingParen(text, position); - } - return false; - } - if (text[i] === "\n") { - return false; - } - } - - return false; -}; +): boolean => inLinkUrlAt(getScan(text), position); // Check if a position is within an HTML tag (between < and >) // e.g. — the underscore in _blank is inside the tag -export const isWithinHtmlTag = (text: string, position: number): boolean => { - // Search backwards from position to find < or > - for (let i = position - 1; i >= 0; i -= 1) { - if (text[i] === ">") { - return false; // Found closing > first — we're outside a tag - } - if (text[i] === "<") { - // Found opening < — check it starts a valid tag (followed by letter or /) - const nextChar = i + 1 < text.length ? text[i + 1] : ""; - if ( - (nextChar >= "a" && nextChar <= "z") || - (nextChar >= "A" && nextChar <= "Z") || - nextChar === "/" - ) { - return true; - } - return false; - } - if (text[i] === "\n") { - return false; // Tags don't span lines in this context - } - } - return false; -}; +export const isWithinHtmlTag = (text: string, position: number): boolean => + inHtmlTagAt(getScan(text), position); // Check if a marker sequence appears to be a horizontal rule // Horizontal rules must be on their own line with optional leading/trailing whitespace diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a055b22c..733ade41 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -158,7 +158,7 @@ importers: version: 1.6.1(next@16.2.6(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react@19.2.3) '@vercel/geistdocs': specifier: 1.11.0 - version: 1.11.0(@svta/cml-cta@1.0.1(@svta/cml-structured-field-values@1.0.1(@svta/cml-utils@1.0.1))(@svta/cml-utils@1.0.1))(@svta/cml-structured-field-values@1.0.1(@svta/cml-utils@1.0.1))(@svta/cml-utils@1.0.1)(@tanstack/react-router@1.151.6(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(@types/mdast@4.0.4)(@types/react-dom@19.2.3(@types/react@19.2.8))(@types/react@19.2.8)(micromark-util-types@2.0.2)(micromark@4.0.2)(next@16.2.6(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-dom@19.2.3(react@19.2.3))(react@19.2.3)(tailwindcss@4.1.18)(unified@11.0.5)(vite@7.3.1(@types/node@24.10.10)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)) + version: 1.11.0(@svta/cml-cta@1.0.1(@svta/cml-structured-field-values@1.0.1(@svta/cml-utils@1.0.1))(@svta/cml-utils@1.0.1))(@svta/cml-structured-field-values@1.0.1(@svta/cml-utils@1.0.1))(@svta/cml-utils@1.0.1)(@tanstack/react-router@1.151.6(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(@types/mdast@4.0.4)(@types/react-dom@19.2.3(@types/react@19.2.8))(@types/react@19.2.8)(next@16.2.6(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-dom@19.2.3(react@19.2.3))(react@19.2.3)(tailwindcss@4.1.18)(unified@11.0.5)(vite@7.3.1(@types/node@24.10.10)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)) '@vercel/speed-insights': specifier: ^1.3.1 version: 1.3.1(next@16.2.6(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react@19.2.3) @@ -268,6 +268,9 @@ importers: '@vitest/coverage-v8': specifier: ^4.0.15 version: 4.0.17(vitest@4.0.17(@opentelemetry/api@1.9.0)(@types/node@25.0.9)(jiti@2.6.1)(jsdom@27.4.0)(lightningcss@1.30.2)(tsx@4.21.0)) + fast-check: + specifier: ^4.9.0 + version: 4.9.0 mdast-util-from-markdown: specifier: ^2.0.2 version: 2.0.2 @@ -4241,6 +4244,10 @@ packages: extendable-error@0.1.7: resolution: {integrity: sha512-UOiS2in6/Q0FK0R0q6UY9vYpQ21mr/Qn1KOnte7vsACuNJf514WvCCUHSRCPcgjPT2bAhNIJdlE6bVap1GKmeg==} + fast-check@4.9.0: + resolution: {integrity: sha512-7ms6T7SybUev/PQITciI0yLM2pOSFy5zpG8Ty7tQofcVaQUvrMXp6CBwqF6fThLCLOrfBtuHAtwq6Yu4XPCllg==} + engines: {node: '>=12.17.0'} + fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} @@ -5348,6 +5355,9 @@ packages: resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} engines: {node: '>=6'} + pure-rand@8.4.2: + resolution: {integrity: sha512-vvuOGgcuPJAirlHvuQw1TrOiw7ptaIXXmIbNuiNOY6lNGJJH49PQ1Kj4nd783nPdQhQdicgOjVI2yI/9BD6/Ng==} + quansync@0.2.11: resolution: {integrity: sha512-AifT7QEbW9Nri4tAwR5M/uzpBuqfZf+zwaEM/QkzEjj7NBuFD2rBuy0K3dE+8wltbezDV7JMA0WfnCPYRSYbXA==} @@ -8933,7 +8943,7 @@ snapshots: '@standard-schema/spec@1.1.0': {} - '@streamdown/cjk@1.0.3(@types/mdast@4.0.4)(micromark-util-types@2.0.2)(micromark@4.0.2)(react@19.2.3)(unified@11.0.5)': + '@streamdown/cjk@1.0.3(@types/mdast@4.0.4)(react@19.2.3)(unified@11.0.5)': dependencies: react: 19.2.3 remark-cjk-friendly: 2.0.1(@types/mdast@4.0.4)(micromark-util-types@2.0.2)(micromark@4.0.2)(unified@11.0.5) @@ -9347,13 +9357,13 @@ snapshots: next: 16.2.6(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) react: 19.2.3 - '@vercel/geistdocs@1.11.0(@svta/cml-cta@1.0.1(@svta/cml-structured-field-values@1.0.1(@svta/cml-utils@1.0.1))(@svta/cml-utils@1.0.1))(@svta/cml-structured-field-values@1.0.1(@svta/cml-utils@1.0.1))(@svta/cml-utils@1.0.1)(@tanstack/react-router@1.151.6(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(@types/mdast@4.0.4)(@types/react-dom@19.2.3(@types/react@19.2.8))(@types/react@19.2.8)(micromark-util-types@2.0.2)(micromark@4.0.2)(next@16.2.6(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-dom@19.2.3(react@19.2.3))(react@19.2.3)(tailwindcss@4.1.18)(unified@11.0.5)(vite@7.3.1(@types/node@24.10.10)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0))': + '@vercel/geistdocs@1.11.0(@svta/cml-cta@1.0.1(@svta/cml-structured-field-values@1.0.1(@svta/cml-utils@1.0.1))(@svta/cml-utils@1.0.1))(@svta/cml-structured-field-values@1.0.1(@svta/cml-utils@1.0.1))(@svta/cml-utils@1.0.1)(@tanstack/react-router@1.151.6(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(@types/mdast@4.0.4)(@types/react-dom@19.2.3(@types/react@19.2.8))(@types/react@19.2.8)(next@16.2.6(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-dom@19.2.3(react@19.2.3))(react@19.2.3)(tailwindcss@4.1.18)(unified@11.0.5)(vite@7.3.1(@types/node@24.10.10)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0))': dependencies: '@ai-sdk/react': 3.0.201(react@19.2.3)(zod@4.3.5) '@clack/prompts': 0.11.0 '@icons-pack/react-simple-icons': 13.8.0(react@19.2.3) '@orama/tokenizers': 3.1.18 - '@streamdown/cjk': 1.0.3(@types/mdast@4.0.4)(micromark-util-types@2.0.2)(micromark@4.0.2)(react@19.2.3)(unified@11.0.5) + '@streamdown/cjk': 1.0.3(@types/mdast@4.0.4)(react@19.2.3)(unified@11.0.5) '@streamdown/code': 1.1.1(react@19.2.3) '@vercel/agent-readability': 0.2.1(next@16.2.6(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)) ai: 6.0.199(zod@4.3.5) @@ -10129,6 +10139,10 @@ snapshots: extendable-error@0.1.7: {} + fast-check@4.9.0: + dependencies: + pure-rand: 8.4.2 + fast-deep-equal@3.1.3: {} fast-glob@3.3.3: @@ -11601,6 +11615,8 @@ snapshots: punycode@2.3.1: {} + pure-rand@8.4.2: {} + quansync@0.2.11: {} queue-microtask@1.2.3: {} From 9a065ee567f6448e62b037e6c3e8cb6a40094ef5 Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Thu, 6 Aug 2026 21:59:06 -0700 Subject: [PATCH 2/5] Address review: slim test comments, forward-looking scan header Pin the boundary of the output-side trailing-space strip with a test showing a double-space hard break before a removed image survives. --- .../__tests__/broken-markdown-variants.test.ts | 8 +++----- packages/remend/__tests__/coverage-gaps.test.ts | 14 ++++---------- packages/remend/__tests__/images.test.ts | 9 +++++++-- packages/remend/__tests__/links.test.ts | 8 +++----- packages/remend/src/scan.ts | 17 ++++++++++------- 5 files changed, 27 insertions(+), 29 deletions(-) diff --git a/packages/remend/__tests__/broken-markdown-variants.test.ts b/packages/remend/__tests__/broken-markdown-variants.test.ts index c39304e1..48a22ebf 100644 --- a/packages/remend/__tests__/broken-markdown-variants.test.ts +++ b/packages/remend/__tests__/broken-markdown-variants.test.ts @@ -123,8 +123,7 @@ describe("multiple incomplete links", () => { }); it("should handle two incomplete links in text-only mode", () => { - // Healing runs to a fixed point, so both unmatched brackets are resolved - // in one call rather than one per call + // Fixed-point healing resolves both unmatched brackets in one call const result = remend("[link1 and [link2", { linkMode: "text-only" }); expect(result).toBe("link1 and link2"); }); @@ -589,9 +588,8 @@ describe("real-world AI streaming patterns", () => { }); it("should handle incomplete image with partial URL", () => { - // Removing the image exposes a trailing space, which is stripped the - // same way a trailing space in the input is, so healed output re-heals - // to itself + // The space exposed by the removal is stripped like an input trailing + // space, so healed output re-heals to itself expect(remend("See ![diagram](http://example.com/img")).toBe("See"); }); diff --git a/packages/remend/__tests__/coverage-gaps.test.ts b/packages/remend/__tests__/coverage-gaps.test.ts index 6af65b2b..8c91f147 100644 --- a/packages/remend/__tests__/coverage-gaps.test.ts +++ b/packages/remend/__tests__/coverage-gaps.test.ts @@ -94,10 +94,8 @@ describe("link handler edge cases", () => { }); it("should remove incomplete images in text-only mode", () => { - // The inner bracket is stripped first, exposing the incomplete image, - // which is removed like any other incomplete image. Previously the image - // survived one call only to be removed by the next, so healed output did - // not re-heal to itself. + // Stripping the inner bracket exposes an incomplete image, which is + // removed like any other expect(remend("![img [text", { linkMode: "text-only" })).toBe(""); }); @@ -202,12 +200,8 @@ describe("double underscore half-complete in code block", () => { describe("double underscore half-complete with word-internal run", () => { it("should complete the opener left unmatched by a word-internal run", () => { - // The __ in b__content sits between word characters, so it is part of an - // identifier and cannot close emphasis. That leaves the __ before b as - // an unmatched opener, and the trailing _ as a half-typed closer to - // complete. Counting raw __ occurrences instead would pair the - // word-internal run against the opener and leave the underscores - // unhealed as literal text. + // b__content is word-internal, so the __ before b is an unmatched + // opener and the trailing _ is its half-typed closer expect(remend("__a__ __b__content_")).toBe("__a__ __b__content__"); }); }); diff --git a/packages/remend/__tests__/images.test.ts b/packages/remend/__tests__/images.test.ts index 6e1b8cbc..0c412343 100644 --- a/packages/remend/__tests__/images.test.ts +++ b/packages/remend/__tests__/images.test.ts @@ -3,8 +3,7 @@ import remend from "../src"; describe("image handling", () => { it("should remove incomplete images", () => { - // The space exposed by removing the image is stripped like any other - // single trailing space + // The space exposed by the removal is stripped like an input trailing space expect(remend("Text with ![incomplete image")).toBe("Text with"); expect(remend("![partial")).toBe(""); }); @@ -14,6 +13,12 @@ describe("image handling", () => { expect(remend(text)).toBe(text); }); + it("should preserve a hard-break double space before a removed image", () => { + // Only a single exposed space is stripped. A double space is a markdown + // hard break and survives, matching the input-side rule. + expect(remend("line one ![partial")).toBe("line one "); + }); + it("should handle partial image at chunk boundary", () => { expect(remend("See ![the diag")).toBe("See"); // Images with partial URLs should be removed (images can't show skeleton) diff --git a/packages/remend/__tests__/links.test.ts b/packages/remend/__tests__/links.test.ts index 7e8ec63f..1c951580 100644 --- a/packages/remend/__tests__/links.test.ts +++ b/packages/remend/__tests__/links.test.ts @@ -115,8 +115,7 @@ describe("link handling with linkMode: text-only", () => { }); it("should handle nested brackets without matching closing bracket", () => { - // Healing runs to a fixed point, so every unmatched bracket is resolved - // in one call rather than one per call + // Fixed-point healing resolves every unmatched bracket in one call expect(remend("Text [outer [inner", textOnlyOptions)).toBe( "Text outer inner" ); @@ -130,9 +129,8 @@ describe("link handling with linkMode: text-only", () => { }); it("should still remove incomplete images", () => { - // Images should still be removed entirely, regardless of linkMode. - // The space exposed by the removal is stripped like any other single - // trailing space. + // Images are removed entirely regardless of linkMode, and the exposed + // trailing space is stripped expect(remend("Text ![incomplete image", textOnlyOptions)).toBe("Text"); expect(remend("Text ![alt](http://partial", textOnlyOptions)).toBe("Text"); }); diff --git a/packages/remend/src/scan.ts b/packages/remend/src/scan.ts index 5f7f1b6e..e29c484d 100644 --- a/packages/remend/src/scan.ts +++ b/packages/remend/src/scan.ts @@ -1,19 +1,22 @@ // Single-pass classification of a text into code and prose regions. // -// Handlers previously re-derived "am I inside code?" per candidate character -// with O(n) rescans, making remend quadratic on delimiter-heavy input. This -// module scans once per input string (memoized) so every position query is -// O(1), and centralizes fence semantics that were previously approximated -// with a context-free ``` toggle: +// Healing runs on every streaming token, and every handler needs to know +// whether a candidate delimiter sits in prose or in code. A single scan +// paints a region code for every position, memoized per input string, so +// each query is O(1) and healing stays linear in the input no matter how +// many delimiters it holds. The lazy masks below answer the same question +// for math, link URLs, and HTML tags. // -// - A fence only opens at the start of a line, indented at most 3 spaces. +// Fence and span semantics follow CommonMark: +// +// - A fence opens only at the start of a line, indented at most 3 spaces. // - Both ``` and ~~~ fences are recognized, with runs of 3 or more. // - The info string of a backtick fence cannot contain a backtick // (a line like ```code``` is inline code, not a fence). // - A fence closes on a run of the same character at least as long as the // opener, alone on its line. // - An inline code span opened by a run of N backticks closes only on a run -// of exactly N backticks; other runs are literal inside the span. +// of exactly N backticks. Other runs are literal inside the span. export const REGION = { PROSE: 0, From a362ba15f2a0895a8cf102ec193d173c7d2fffe9 Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Thu, 6 Aug 2026 22:20:55 -0700 Subject: [PATCH 3/5] comments: trim AI-slop comments --- packages/remend/__tests__/images.test.ts | 3 +-- .../remend/__tests__/streaming-properties.test.ts | 8 ++++---- packages/remend/src/code-block-utils.ts | 2 +- packages/remend/src/emphasis-handlers.ts | 10 ++-------- packages/remend/src/katex-handler.ts | 3 +-- packages/remend/src/link-image-handler.ts | 2 +- packages/remend/src/scan.ts | 11 +++-------- packages/remend/src/utils.ts | 1 - 8 files changed, 13 insertions(+), 27 deletions(-) diff --git a/packages/remend/__tests__/images.test.ts b/packages/remend/__tests__/images.test.ts index 0c412343..da2248e3 100644 --- a/packages/remend/__tests__/images.test.ts +++ b/packages/remend/__tests__/images.test.ts @@ -14,8 +14,7 @@ describe("image handling", () => { }); it("should preserve a hard-break double space before a removed image", () => { - // Only a single exposed space is stripped. A double space is a markdown - // hard break and survives, matching the input-side rule. + // Only a single exposed space is stripped. A double space is a markdown hard break and survives. expect(remend("line one ![partial")).toBe("line one "); }); diff --git a/packages/remend/__tests__/streaming-properties.test.ts b/packages/remend/__tests__/streaming-properties.test.ts index 1ee7c67f..435798c6 100644 --- a/packages/remend/__tests__/streaming-properties.test.ts +++ b/packages/remend/__tests__/streaming-properties.test.ts @@ -126,10 +126,10 @@ describe("streaming properties", () => { }); describe("exhaustive prefix sweep", () => { - // Every prefix of a fixed corpus, deterministically. The corpus mixes the - // constructs whose interactions have bitten before: identifiers with - // double underscores next to real emphasis, fences of both characters, - // spans with multi-backtick runs, and half-typed closers. + // Every prefix of a fixed corpus, deterministically. The corpus mixes + // constructs that interact: identifiers with double underscores next to + // real emphasis, fences of both characters, spans with multi-backtick runs, + // and half-typed closers. const corpus = [ "Use snake__case for names and __bold text__ throughout.", "The `obj__attr` field pairs with **bold** and _italic_ text.", diff --git a/packages/remend/src/code-block-utils.ts b/packages/remend/src/code-block-utils.ts index 53938857..499aa465 100644 --- a/packages/remend/src/code-block-utils.ts +++ b/packages/remend/src/code-block-utils.ts @@ -1,6 +1,6 @@ import { getScan, isCodeAt, isCompleteSpanAt } from "./scan"; -// Check if a position is inside a code construct (fenced block or inline span) +// A code construct here means a fenced block or an inline code span. export const isInsideCodeBlock = (text: string, position: number): boolean => isCodeAt(getScan(text), position); diff --git a/packages/remend/src/emphasis-handlers.ts b/packages/remend/src/emphasis-handlers.ts index 05fec023..82362f28 100644 --- a/packages/remend/src/emphasis-handlers.ts +++ b/packages/remend/src/emphasis-handlers.ts @@ -78,8 +78,6 @@ const shouldSkipAsterisk = ( return false; }; -// Counts asterisks that can serve as single-emphasis delimiters, applying -// the skip rules in shouldSkipAsterisk and excluding code regions export const countSingleAsterisks = (text: string): number => { const scan = getScan(text); let count = 0; @@ -141,8 +139,6 @@ const shouldSkipUnderscore = ( return false; }; -// Counts underscores that can serve as single-emphasis delimiters, applying -// the skip rules in shouldSkipUnderscore and excluding code regions export const countSingleUnderscores = (text: string): number => { const scan = getScan(text); let count = 0; @@ -191,7 +187,6 @@ export const countTripleAsterisks = (text: string): number => { return count; }; -// Counts ** pairs outside code regions const countDoubleAsterisks = (text: string): number => { const scan = getScan(text); let count = 0; @@ -209,7 +204,7 @@ const countDoubleAsterisks = (text: string): number => { }; // Whether the text has an unmatched __ delimiter, counted per maximal -// underscore run rather than per raw __ occurrence. +// underscore run. // // Counting raw occurrences misreads identifiers: a name like snake__case // contains __ but cannot open or close emphasis, and counting it either @@ -465,7 +460,7 @@ export const handleIncompleteDoubleUnderscoreItalic = ( return text; }; -// Helper function to find the first single asterisk index (skips code regions) +// Skips code regions when locating the asterisk. const findFirstSingleAsteriskIndex = (text: string): number => { const scan = getScan(text); @@ -550,7 +545,6 @@ export const handleIncompleteSingleAsteriskItalic = (text: string): string => { return text; }; -// Helper function to find the first single underscore index (skips code regions) const findFirstSingleUnderscoreIndex = (text: string): number => { const scan = getScan(text); diff --git a/packages/remend/src/katex-handler.ts b/packages/remend/src/katex-handler.ts index b550bf1d..17f53eba 100644 --- a/packages/remend/src/katex-handler.ts +++ b/packages/remend/src/katex-handler.ts @@ -1,6 +1,5 @@ import { getScan, REGION } from "./scan"; -// Helper function to count $$ pairs outside code regions const countDollarPairs = (text: string): number => { const scan = getScan(text); let dollarPairs = 0; @@ -18,7 +17,7 @@ const countDollarPairs = (text: string): number => { return dollarPairs; }; -// Helper function to count single $ signs (excluding $$) outside code regions +// Excludes $$ pairs and any $ inside code regions. const countSingleDollars = (text: string): number => { const scan = getScan(text); let count = 0; diff --git a/packages/remend/src/link-image-handler.ts b/packages/remend/src/link-image-handler.ts index df236370..7a592333 100644 --- a/packages/remend/src/link-image-handler.ts +++ b/packages/remend/src/link-image-handler.ts @@ -159,7 +159,7 @@ const healTrailingLinkOrImage = (text: string, linkMode: LinkMode): string => { return text; }; -// Handles incomplete links and images by preserving them with a special marker +// Preserves incomplete links and images with a special marker. export const handleIncompleteLinksAndImages = ( text: string, linkMode: LinkMode = "protocol" diff --git a/packages/remend/src/scan.ts b/packages/remend/src/scan.ts index e29c484d..57345fa9 100644 --- a/packages/remend/src/scan.ts +++ b/packages/remend/src/scan.ts @@ -24,7 +24,6 @@ export const REGION = { FENCE_MARKER: 1, /** The info string on a fence opener line. Neither prose nor code body. */ FENCE_INFO: 2, - /** Content of a fenced code block */ FENCE_BODY: 3, /** A complete inline code span, including its backtick markers */ CODE_SPAN: 4, @@ -62,7 +61,6 @@ export interface TextScan { const FENCE_OPENER_PATTERN = /^( {0,3})(`{3,}|~{3,})(.*)$/; -// Marks the fence regions of one opener line and returns the open fence state const paintFenceOpener = ( regions: Uint8Array, lineStart: number, @@ -72,7 +70,7 @@ const paintFenceOpener = ( ): void => { const markerStart = lineStart + indentLength; regions.fill(REGION.FENCE_MARKER, markerStart, markerStart + markerLength); - // Info string plus the line terminator belong to the fence, not to prose + // Info string plus the line terminator belong to the fence. regions.fill( REGION.FENCE_INFO, markerStart + markerLength, @@ -115,7 +113,6 @@ const isFenceCloser = ( return true; }; -// First pass: paint fenced code blocks line by line const paintFences = (text: string, regions: Uint8Array): OpenFence | null => { const n = text.length; let openFence: OpenFence | null = null; @@ -246,7 +243,7 @@ export const getScan = (text: string): TextScan => { return scan; }; -/** Whether the position is inside any code construct (fence or span) */ +/** A code construct here means a fence or inline span. */ export const isCodeAt = (scan: TextScan, position: number): boolean => { if (position >= scan.regions.length) { return scan.openFence !== null || scan.openSpan !== null; @@ -273,12 +270,10 @@ export const isFenceAt = (scan: TextScan, position: number): boolean => { ); }; -/** Whether the position is inside a complete inline code span */ export const isCompleteSpanAt = (scan: TextScan, position: number): boolean => scan.regions[position] === REGION.CODE_SPAN; -// Math mask: for each position, whether it is inside $...$ or $$...$$, -// mirroring the sequential toggle semantics the per-position helper used +// Math mask: for each position, whether it is inside $...$ or $$...$$ const buildMathMask = (text: string): Uint8Array => { const n = text.length; const mask = new Uint8Array(n); diff --git a/packages/remend/src/utils.ts b/packages/remend/src/utils.ts index 4d892fe1..06291fd0 100644 --- a/packages/remend/src/utils.ts +++ b/packages/remend/src/utils.ts @@ -21,7 +21,6 @@ export const isWordChar = (char: string): boolean => { return letterNumberUnderscorePattern.test(char); }; -// Check if a position is within a fenced code block export const isWithinCodeBlock = (text: string, position: number): boolean => isFenceAt(getScan(text), position); From 33f6fddacea1b5ae2dce4072f374c27a47339aab Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Thu, 6 Aug 2026 22:53:12 -0700 Subject: [PATCH 4/5] Fix streaming regressions and quadratic worst case found in review Recognize fences at any indent (list-nested fences carry deeper absolute indents than CommonMark's top-level 3-space cap) and on CRLF lines, so their content is no longer misread as an inline code span and corrupted with appended backticks. Stop inline code spans at blank lines, matching paragraph-scoped inline parsing, so one stray backtick run no longer disables healing for the rest of the stream. Treat the run after an escaped underscore as a delimiter again. Make the math, link-URL, and HTML masks region-aware so delimiters inside code cannot corrupt mask state for later prose, and skip building each mask when its trigger character is absent. Bound the link/image healing loop, which cost a full rescan per removed construct and turned healing quadratic on adversarial tails of nested incomplete constructs. Fold the three identical double-marker counting loops into one countDoublePairs helper on the scanner. --- .../remend/__tests__/fence-semantics.test.ts | 38 ++++++ packages/remend/__tests__/katex.test.ts | 12 ++ .../remend/__tests__/underscore-runs.test.ts | 6 + packages/remend/src/emphasis-handlers.ts | 27 ++-- packages/remend/src/katex-handler.ts | 19 +-- packages/remend/src/link-image-handler.ts | 8 +- packages/remend/src/scan.ts | 126 +++++++++++++----- packages/remend/src/strikethrough-handler.ts | 24 +--- 8 files changed, 173 insertions(+), 87 deletions(-) diff --git a/packages/remend/__tests__/fence-semantics.test.ts b/packages/remend/__tests__/fence-semantics.test.ts index 4ebc363c..37f8e025 100644 --- a/packages/remend/__tests__/fence-semantics.test.ts +++ b/packages/remend/__tests__/fence-semantics.test.ts @@ -76,3 +76,41 @@ describe("inline code span run lengths", () => { expect(remend("`a``")).toBe("`a``"); }); }); + +describe("list-indented fences", () => { + it("should recognize a fence indented inside a list item", () => { + expect(remend("1. Install:\n ```bash\n npm install foo")).toBe( + "1. Install:\n ```bash\n npm install foo" + ); + }); + + it("should not heal emphasis inside a list-indented fence", () => { + expect(remend("- step\n - nested\n ```js\n const x = a__b")).toBe( + "- step\n - nested\n ```js\n const x = a__b" + ); + }); +}); + +describe("CRLF line endings", () => { + it("should recognize a fence opener on a CRLF line", () => { + expect(remend("```js\r\nconst a = 1")).toBe("```js\r\nconst a = 1"); + }); + + it("should close a CRLF fence and heal after it", () => { + expect(remend("```\r\ncode\r\n```\r\n__open")).toBe( + "```\r\ncode\r\n```\r\n__open__" + ); + }); +}); + +describe("spans across paragraphs", () => { + it("should leave an unmatched run literal once its paragraph ends", () => { + expect(remend("use ``` to open a block\n\nmore **bold streaming")).toBe( + "use ``` to open a block\n\nmore **bold streaming**" + ); + }); + + it("should still complete an open span in the last paragraph", () => { + expect(remend("intro\n\nrun `npm i")).toBe("intro\n\nrun `npm i`"); + }); +}); diff --git a/packages/remend/__tests__/katex.test.ts b/packages/remend/__tests__/katex.test.ts index af5c375e..1095b0e5 100644 --- a/packages/remend/__tests__/katex.test.ts +++ b/packages/remend/__tests__/katex.test.ts @@ -286,3 +286,15 @@ describe("math blocks with asterisks", () => { expect(remend(text)).toBe("Start *italic with $$x^{*}$$*"); }); }); + +describe("dollar signs inside code", () => { + it("should not let a $ in inline code suppress later healing", () => { + expect(remend("`$` _hello")).toBe("`$` _hello_"); + }); + + it("should not let a $ in a fence suppress later healing", () => { + expect(remend("```\nprice = $5\n```\n_hello")).toBe( + "```\nprice = $5\n```\n_hello_" + ); + }); +}); diff --git a/packages/remend/__tests__/underscore-runs.test.ts b/packages/remend/__tests__/underscore-runs.test.ts index a2648c9d..120f6a29 100644 --- a/packages/remend/__tests__/underscore-runs.test.ts +++ b/packages/remend/__tests__/underscore-runs.test.ts @@ -49,3 +49,9 @@ describe("underscore run lengths", () => { expect(remend("___both___ done")).toBe("___both___ done"); }); }); + +describe("escaped underscores", () => { + it("should treat the run after an escaped underscore as a delimiter", () => { + expect(remend("\\___bold")).toBe("\\___bold__"); + }); +}); diff --git a/packages/remend/src/emphasis-handlers.ts b/packages/remend/src/emphasis-handlers.ts index 82362f28..37da522a 100644 --- a/packages/remend/src/emphasis-handlers.ts +++ b/packages/remend/src/emphasis-handlers.ts @@ -14,6 +14,7 @@ import { whitespaceOrMarkersPattern, } from "./patterns"; import { + countDoublePairs, getScan, inHtmlTagAt, inLinkUrlAt, @@ -187,21 +188,8 @@ export const countTripleAsterisks = (text: string): number => { return count; }; -const countDoubleAsterisks = (text: string): number => { - const scan = getScan(text); - let count = 0; - - for (let i = 0; i < text.length; i += 1) { - if (scan.regions[i] !== REGION.PROSE) { - continue; - } - if (text[i] === "*" && i + 1 < text.length && text[i + 1] === "*") { - count += 1; - i += 1; - } - } - return count; -}; +const countDoubleAsterisks = (text: string): number => + countDoublePairs(text, "*"); // Whether the text has an unmatched __ delimiter, counted per maximal // underscore run. @@ -257,17 +245,22 @@ const doubleUnderscoreRunFlips = ( ): boolean => { const text = scan.text; - // A backslash escapes the first underscore of the run + // A backslash escapes the first underscore of the run. The escaped + // underscore is literal punctuation, so the rest of the run still flanks + // as a delimiter. let runStart = initialRunStart; + let escaped = false; if (runStart > 0 && text[runStart - 1] === "\\") { runStart += 1; + escaped = true; } const runLength = runEnd - runStart; if (runLength < 2) { return false; } - const prevChar = runStart > 0 ? text[runStart - 1] : ""; + const beforeRun = runStart > 0 ? text[runStart - 1] : ""; + const prevChar = escaped ? "\\" : beforeRun; const nextChar = runEnd < text.length ? text[runEnd] : ""; if (isWordChar(prevChar) && isWordChar(nextChar)) { return false; diff --git a/packages/remend/src/katex-handler.ts b/packages/remend/src/katex-handler.ts index 17f53eba..c274fd70 100644 --- a/packages/remend/src/katex-handler.ts +++ b/packages/remend/src/katex-handler.ts @@ -1,21 +1,6 @@ -import { getScan, REGION } from "./scan"; +import { countDoublePairs, getScan, REGION } from "./scan"; -const countDollarPairs = (text: string): number => { - const scan = getScan(text); - let dollarPairs = 0; - - for (let i = 0; i < text.length - 1; i += 1) { - if (scan.regions[i] !== REGION.PROSE) { - continue; - } - if (text[i] === "$" && text[i + 1] === "$") { - dollarPairs += 1; - i += 1; - } - } - - return dollarPairs; -}; +const countDollarPairs = (text: string): number => countDoublePairs(text, "$"); // Excludes $$ pairs and any $ inside code regions. const countSingleDollars = (text: string): number => { diff --git a/packages/remend/src/link-image-handler.ts b/packages/remend/src/link-image-handler.ts index 7a592333..4fb75688 100644 --- a/packages/remend/src/link-image-handler.ts +++ b/packages/remend/src/link-image-handler.ts @@ -159,6 +159,11 @@ const healTrailingLinkOrImage = (text: string, linkMode: LinkMode): string => { return text; }; +// A pass costs a full rescan of the shortened text, so an adversarial tail +// of thousands of nested incomplete constructs would turn healing quadratic +// without a bound. Realistic nesting depth is single digits. +const MAX_HEAL_PASSES = 32; + // Preserves incomplete links and images with a special marker. export const handleIncompleteLinksAndImages = ( text: string, @@ -172,11 +177,12 @@ export const handleIncompleteLinksAndImages = ( // re-heals to itself, so healing is idempotent. A step that grows the text // has completed the construct, and truncating steps strictly shorten, // so the loop terminates. - for (;;) { + for (let pass = 0; pass < MAX_HEAL_PASSES; pass += 1) { const next = healTrailingLinkOrImage(current, linkMode); if (next.length >= current.length) { return next; } current = next; } + return current; }; diff --git a/packages/remend/src/scan.ts b/packages/remend/src/scan.ts index 57345fa9..3de71783 100644 --- a/packages/remend/src/scan.ts +++ b/packages/remend/src/scan.ts @@ -9,14 +9,20 @@ // // Fence and span semantics follow CommonMark: // -// - A fence opens only at the start of a line, indented at most 3 spaces. +// - A fence opens only at the start of a line, with any indentation. CommonMark +// caps a top-level fence at 3 spaces, but fences nested in list items carry +// deeper absolute indents and a line-based scan has no list context. Reading +// an indented line as code is the safe direction: healing then leaves it +// alone instead of corrupting it. // - Both ``` and ~~~ fences are recognized, with runs of 3 or more. // - The info string of a backtick fence cannot contain a backtick // (a line like ```code``` is inline code, not a fence). // - A fence closes on a run of the same character at least as long as the -// opener, alone on its line. +// opener, alone on its line. Lines may end in \n or \r\n. // - An inline code span opened by a run of N backticks closes only on a run // of exactly N backticks. Other runs are literal inside the span. +// - A span cannot cross a blank line: inline parsing is paragraph-scoped, so +// an unmatched run in a finished paragraph stays literal prose. export const REGION = { PROSE: 0, @@ -59,7 +65,7 @@ export interface TextScan { text: string; } -const FENCE_OPENER_PATTERN = /^( {0,3})(`{3,}|~{3,})(.*)$/; +const FENCE_OPENER_PATTERN = /^( *)(`{3,}|~{3,})(.*)$/; const paintFenceOpener = ( regions: Uint8Array, @@ -78,9 +84,8 @@ const paintFenceOpener = ( ); }; -// Whether a line inside an open fence closes it: at most 3 spaces of indent, -// then a run of the fence character at least as long as the opener, then -// only whitespace +// Whether a line inside an open fence closes it: optional indent, then a run +// of the fence character at least as long as the opener, then only whitespace const isFenceCloser = ( text: string, lineStart: number, @@ -88,13 +93,8 @@ const isFenceCloser = ( fence: OpenFence ): boolean => { let i = lineStart; - let indent = 0; - while (i < lineEnd && text[i] === " " && indent < 4) { + while (i < lineEnd && text[i] === " ") { i += 1; - indent += 1; - } - if (indent > 3) { - return false; } let runLength = 0; while (i < lineEnd && text[i] === fence.char) { @@ -105,7 +105,7 @@ const isFenceCloser = ( return false; } while (i < lineEnd) { - if (text[i] !== " " && text[i] !== "\t") { + if (text[i] !== " " && text[i] !== "\t" && text[i] !== "\r") { return false; } i += 1; @@ -132,7 +132,11 @@ const paintFences = (text: string, regions: Uint8Array): OpenFence | null => { regions.fill(REGION.FENCE_BODY, lineStart, Math.min(lineEnd + 1, n)); } } else { - const line = text.slice(lineStart, lineEnd); + const contentEnd = + lineEnd > lineStart && text[lineEnd - 1] === "\r" + ? lineEnd - 1 + : lineEnd; + const line = text.slice(lineStart, contentEnd); const opener = line.match(FENCE_OPENER_PATTERN); if (opener) { const markerChar = opener[2][0] as "`" | "~"; @@ -165,7 +169,19 @@ const measureBacktickRun = (text: string, start: number): number => { return end; }; -// Second pass: paint inline code spans in the regions fences left as prose +// A blank line ends the paragraph, and with it any chance of closing a span +const isParagraphBreakAt = (text: string, newlineIndex: number): boolean => { + let j = newlineIndex + 1; + while ( + j < text.length && + (text[j] === " " || text[j] === "\t" || text[j] === "\r") + ) { + j += 1; + } + return j < text.length && text[j] === "\n"; +}; + +// Paint inline code spans in the regions the fence pass left as prose const paintSpans = (text: string, regions: Uint8Array): OpenSpan | null => { const n = text.length; let spanStart = -1; @@ -182,6 +198,12 @@ const paintSpans = (text: string, regions: Uint8Array): OpenSpan | null => { i += 1; continue; } + if (text[i] === "\n" && spanStart >= 0 && isParagraphBreakAt(text, i)) { + // The unmatched opener stays literal prose in its finished paragraph + spanStart = -1; + i += 1; + continue; + } if (text[i] === "\\" && text[i + 1] === "`" && spanStart < 0) { i += 2; continue; @@ -273,8 +295,31 @@ export const isFenceAt = (scan: TextScan, position: number): boolean => { export const isCompleteSpanAt = (scan: TextScan, position: number): boolean => scan.regions[position] === REGION.CODE_SPAN; -// Math mask: for each position, whether it is inside $...$ or $$...$$ -const buildMathMask = (text: string): Uint8Array => { +/** Counts non-overlapping double-character pairs (**, ~~, $$) in prose */ +export const countDoublePairs = (text: string, char: string): number => { + const scan = getScan(text); + let count = 0; + + for (let i = 0; i < text.length; i += 1) { + if (scan.regions[i] !== REGION.PROSE) { + continue; + } + if (text[i] === char && i + 1 < text.length && text[i + 1] === char) { + count += 1; + i += 1; + } + } + return count; +}; + +// The masks share the empty array when their trigger character is absent, so +// plain prose skips three allocations and passes per scan +const EMPTY_MASK = new Uint8Array(0); + +// Math mask: for each position, whether it is inside $...$ or $$...$$. +// Delimiters inside code regions are literal and do not toggle math state. +const buildMathMask = (scan: TextScan): Uint8Array => { + const { text, regions } = scan; const n = text.length; const mask = new Uint8Array(n); let inInlineMath = false; @@ -283,10 +328,12 @@ const buildMathMask = (text: string): Uint8Array => { let i = 0; while (i < n) { mask[i] = inInlineMath || inBlockMath ? 1 : 0; + if (regions[i] !== REGION.PROSE) { + i += 1; + continue; + } if (text[i] === "\\" && text[i + 1] === "$") { - if (i + 1 < n) { - mask[i + 1] = mask[i]; - } + mask[i + 1] = mask[i]; i += 2; continue; } @@ -297,9 +344,7 @@ const buildMathMask = (text: string): Uint8Array => { if (text[i + 1] === "$") { inBlockMath = !inBlockMath; inInlineMath = false; - if (i + 1 < n) { - mask[i + 1] = 1; - } + mask[i + 1] = 1; i += 2; continue; } @@ -317,7 +362,7 @@ export const inMathAt = (scan: TextScan, position: number): boolean => { return false; } if (scan.mathMask === null) { - scan.mathMask = buildMathMask(scan.text); + scan.mathMask = scan.text.includes("$") ? buildMathMask(scan) : EMPTY_MASK; } return scan.mathMask[position] === 1; }; @@ -327,16 +372,17 @@ export const inMathAt = (scan: TextScan, position: number): boolean => { // follows a position, forward to know whether the nearest paren boundary // before a position is a "](" opener. const paintLinkUrlLine = ( - text: string, + scan: TextScan, lineStart: number, lineEnd: number, mask: Uint8Array ): void => { + const { text, regions } = scan; // closerFollows[i - lineStart]: a ")" exists at or after i on this line const closerFollows = new Uint8Array(lineEnd - lineStart); let seenCloser = 0; for (let i = lineEnd - 1; i >= lineStart; i -= 1) { - if (text[i] === ")") { + if (text[i] === ")" && regions[i] === REGION.PROSE) { seenCloser = 1; } closerFollows[i - lineStart] = seenCloser; @@ -347,6 +393,9 @@ const paintLinkUrlLine = ( if (inUrl && closerFollows[i - lineStart] === 1) { mask[i] = 1; } + if (regions[i] !== REGION.PROSE) { + continue; + } if (text[i] === ")") { inUrl = false; } else if (text[i] === "(") { @@ -355,8 +404,10 @@ const paintLinkUrlLine = ( } }; -// Link/image URL mask: positions inside the (url) part of [text](url) -const buildLinkUrlMask = (text: string): Uint8Array => { +// Link/image URL mask: positions inside the (url) part of [text](url). +// Delimiters inside code regions are literal and never open or close a URL. +const buildLinkUrlMask = (scan: TextScan): Uint8Array => { + const { text } = scan; const n = text.length; const mask = new Uint8Array(n); let lineStart = 0; @@ -366,7 +417,7 @@ const buildLinkUrlMask = (text: string): Uint8Array => { if (lineEnd === -1) { lineEnd = n; } - paintLinkUrlLine(text, lineStart, lineEnd, mask); + paintLinkUrlLine(scan, lineStart, lineEnd, mask); lineStart = lineEnd + 1; } @@ -378,14 +429,18 @@ export const inLinkUrlAt = (scan: TextScan, position: number): boolean => { return false; } if (scan.linkUrlMask === null) { - scan.linkUrlMask = buildLinkUrlMask(scan.text); + scan.linkUrlMask = scan.text.includes("](") + ? buildLinkUrlMask(scan) + : EMPTY_MASK; } return scan.linkUrlMask[position] === 1; }; // HTML tag mask: positions after a "<" that begins a plausible tag (letter -// or /), through the closing ">" inclusive, within a single line -const buildHtmlTagMask = (text: string): Uint8Array => { +// or /), through the closing ">" inclusive, within a single line. Angle +// brackets inside code regions are literal and never open or close a tag. +const buildHtmlTagMask = (scan: TextScan): Uint8Array => { + const { text, regions } = scan; const n = text.length; const mask = new Uint8Array(n); let inTag = false; @@ -396,6 +451,9 @@ const buildHtmlTagMask = (text: string): Uint8Array => { continue; } mask[i] = inTag ? 1 : 0; + if (regions[i] !== REGION.PROSE) { + continue; + } if (text[i] === ">") { inTag = false; } else if (text[i] === "<") { @@ -416,7 +474,9 @@ export const inHtmlTagAt = (scan: TextScan, position: number): boolean => { return false; } if (scan.htmlTagMask === null) { - scan.htmlTagMask = buildHtmlTagMask(scan.text); + scan.htmlTagMask = scan.text.includes("<") + ? buildHtmlTagMask(scan) + : EMPTY_MASK; } return scan.htmlTagMask[position] === 1; }; diff --git a/packages/remend/src/strikethrough-handler.ts b/packages/remend/src/strikethrough-handler.ts index e5a80910..61e9e2a8 100644 --- a/packages/remend/src/strikethrough-handler.ts +++ b/packages/remend/src/strikethrough-handler.ts @@ -7,26 +7,12 @@ import { strikethroughPattern, whitespaceOrMarkersPattern, } from "./patterns"; -import { getScan, REGION } from "./scan"; +import { countDoublePairs } from "./scan"; -// Counts ~~ pairs outside code regions. Tilde runs that open or close a -// fence are painted as fence regions by the scanner, so a line-start ~~~ -// fence never counts as strikethrough while a mid-line tilde run still does. -const countDoubleTildes = (text: string): number => { - const scan = getScan(text); - let count = 0; - - for (let i = 0; i < text.length; i += 1) { - if (scan.regions[i] !== REGION.PROSE) { - continue; - } - if (text[i] === "~" && i + 1 < text.length && text[i + 1] === "~") { - count += 1; - i += 1; - } - } - return count; -}; +// Tilde runs that open or close a fence are painted as fence regions by the +// scanner, so a line-start ~~~ fence never counts as strikethrough while a +// mid-line tilde run still does. +const countDoubleTildes = (text: string): number => countDoublePairs(text, "~"); // Completes incomplete strikethrough formatting (~~) export const handleIncompleteStrikethrough = (text: string): string => { From 2ba89e0c3700562b731e9228fd937deed98fb22f Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Thu, 6 Aug 2026 23:04:58 -0700 Subject: [PATCH 5/5] Correct changeset fence semantics after review fixes --- .changeset/spotty-eyes-brake.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.changeset/spotty-eyes-brake.md b/.changeset/spotty-eyes-brake.md index 99a29f99..1e6edb52 100644 --- a/.changeset/spotty-eyes-brake.md +++ b/.changeset/spotty-eyes-brake.md @@ -4,8 +4,8 @@ Rework code-region detection and double-underscore counting. -A shared single-pass scanner now classifies fences and inline code spans, replacing the per-character rescans that made healing quadratic on delimiter-heavy input. Fences follow CommonMark rules (line-start only, up to 3 spaces of indent, `~~~` supported, closers must match the opener's length, info strings excluded from emphasis), and inline code spans close on a backtick run of exactly the opener's length. +A shared single-pass scanner now classifies fences and inline code spans, replacing the per-character rescans that made healing quadratic on delimiter-heavy input. Fence and span detection follows CommonMark, so `~~~` fences, list-indented fences, CRLF line endings, and multi-backtick spans are all recognized, and content inside code is never healed as prose. Double underscores are counted per maximal run with flanking rules, so identifiers containing `__` (like `snake__case`) no longer invent or swallow emphasis closers. -Healing is now idempotent: healed output re-heals to itself, including incomplete image removal and the trailing space it exposes. +Healing is now idempotent. Healed output re-heals to itself, including incomplete image removal and the trailing space it exposes.