From df9bffca84c2b2a7986585e68d70b58fd3fb23d4 Mon Sep 17 00:00:00 2001 From: Dmitry Bondarenko Date: Fri, 11 Jul 2025 21:32:29 +0600 Subject: [PATCH 01/32] Tree-sitter with markdown in progress --- Dockerfile | 53 ++- .../char-streamer-tree-sitter.ts | 390 ++++++++++++++++++ package.json | 6 +- src/tree-sitter-markdown-stream-parser.ts | 163 ++++++++ 4 files changed, 601 insertions(+), 11 deletions(-) create mode 100644 demo/debug-scripts/char-streamer-tree-sitter.ts create mode 100644 src/tree-sitter-markdown-stream-parser.ts diff --git a/Dockerfile b/Dockerfile index b5791f5..0c16127 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,28 +1,63 @@ # Set Node.js version ARG NODE_VERSION=23 - +ARG TREE_SITTER_VERSION=0.25.5 # Stage 1: Build -FROM node:${NODE_VERSION}-alpine +# FROM node:${NODE_VERSION}-alpine +# FROM frolvlad/alpine-glibc +FROM node:20-slim # Install necessary packages -RUN apk add --update --no-cache curl +# RUN apk add --update --no-cache curl python3 make g++ gcc libc-dev + +RUN apt-get update && apt-get install -y \ + xz-utils \ + python3 \ + make \ + g++ \ + && rm -rf /var/lib/apt/lists/* + +# Download and install Node.js +# RUN curl -fsSL https://nodejs.org/dist/v${NODE_VERSION}.0.0/node-v${NODE_VERSION}.0.0-linux-x64.tar.xz | tar -xJ -C /usr/local --strip-components=1 + +ADD https://nodejs.org/dist/v23.0.0/node-v23.0.0-linux-x64.tar.gz /tmp/node.tar.gz +RUN tar -xzf /tmp/node.tar.gz -C /usr/local --strip-components=1 && rm /tmp/node.tar.gz + +ADD https://github.com/tree-sitter/tree-sitter/releases/download/v0.25.5/tree-sitter-linux-x64.gz /tmp/tree-sitter.gz +RUN gzip -d /tmp/tree-sitter.gz \ + && chmod +x /tmp/tree-sitter \ + && mv /tmp/tree-sitter /usr/local/bin/tree-sitter \ + && tree-sitter --version + +# Verify tree-sitter works +RUN tree-sitter --version -# Install pnpm globally RUN npm install -g pnpm +# Set environment variables for C++ compilation +ENV CXXFLAGS="-std=c++20 -fexceptions" +ENV CXX="g++ -std=c++20 -fexceptions" # Set the working directory WORKDIR /usr/src/service # Copy the rest of app's source code COPY . . -# Install dependencies +# Install dependencies with exception handling enabled RUN pnpm install --force && pnpm store prune && rm -rf ~/.pnpm-store +# Install dependencies +# CXXFLAGS="-fexceptions" added to make +# RUN CXXFLAGS="-std=c++20 -fexceptions" pnpm install --force && pnpm store prune && rm -rf ~/.pnpm-store + +# First ensure tsup is available at the root level +# RUN pnpm add -D tsup typescript ts-node @types/node + +# Build the demo +# WORKDIR /usr/src/service/demo/svelte-demo +# RUN pnpm install --force +# # Install ts-node in the demo directory as well +# RUN pnpm add -D ts-node @types/node typescript +# RUN pnpm run build -# Build the Svelte demo -WORKDIR /usr/src/service/demo/svelte-demo -RUN pnpm install --force -RUN pnpm run build WORKDIR /usr/src/service # Run the application diff --git a/demo/debug-scripts/char-streamer-tree-sitter.ts b/demo/debug-scripts/char-streamer-tree-sitter.ts new file mode 100644 index 0000000..07e867b --- /dev/null +++ b/demo/debug-scripts/char-streamer-tree-sitter.ts @@ -0,0 +1,390 @@ +import fs from 'fs' +import Parser from 'tree-sitter' +import Markdown from '@tree-sitter-grammars/tree-sitter-markdown'; +import { MarkdownStreamParser } from '../../src/tree-sitter-markdown-stream-parser.ts' + +import { log, info, infoStr, warn, err } from './debug-tools.ts' + +// Parse CLI arguments +const args = process.argv.slice(2); +let DELAY = 0; +let fileName = ''; + +for (const arg of args) { + if (arg.startsWith('--interval=')) { + const val = parseInt(arg.split('=')[1], 10); + if (!isNaN(val)) DELAY = val; + } + if (arg.startsWith('--file=')) { + fileName = arg.split('=')[1]; + } +} + +if (!fileName) { + throw new Error('Missing required argument: --file='); +} + +// const sourceFile = `/usr/src/service/demo/llm-streams-examples/${fileName}`; +const markdownLines = [ + "####", + " 🐾", + " **", + "Regex", + " Lesson", + ":", + " Evalu", + "ating", + " Cat", + " Bre", + "eds", + "**\n\n", + "Regex", + ",", + " or", + " **", + "regular", + " expressions", + "**,", + " are", + " a", + " powerful", + " textual", + " tool", + " often", + " used", + " in", + " programming", + " for", + " finding", + "```", + "python", + "\n", + "import", + " re", + "\n\n", + "#", + " List", + " of", + " cat", + " breeds", + "\n", + "cat", + "_b", + "re", + "eds", + " =", + " ['", + "S", + "iam", + "ese", + "',", + " '", + "Pers", + "ian", + "',", + " '", + "M", + "aine", + " C", + "oon", + "',", + " '", + "B", + "eng", + "al", + "',", + " '", + "S", + "ph", + "yn", + "x", + "']\n\n", + "#", + " Join", + " breeds", + " into", + " a", + " regex", + " pattern", + "\n", + "pattern", + " =", + " r", + "'\\", + "b", + "(?:", + "'", + " +", + " '|", + "'.", + "join", + "(map", + "(re", + ".escape", + ",", + " cat", + "_b", + "re", + "eds", + "))", + " +", + " r", + "')", + "\\", + "b", + "'\n\n", + "#", + " Sample", + " sentences", + "\n", + "text", + " =", + " \"", + "I", + " have", + " a", + " Bengal", + " and", + " a", + " Maine", + " C", + "oon", + ",", + " but", + " my", + " friend", + " prefers", + " S", + "ph", + "yn", + "x", + " cats", + ".\"\n\n", + "#", + " Search", + " for", + " matches", + "\n", + "matches", + " =", + " re", + ".findall", + "(pattern", + ",", + " text", + ",", + " flags", + "=re", + ".", + "IGNORE", + "CASE", + ")\n\n", + "print", + "(\"", + "Cat", + " breeds", + " found", + ":\",", + " matches", + ")\n", + "``", + "`\n\n", + "---\n\n", +] + +// Initialize tree-sitter parser +// const parser = new Parser(); +// parser.setLanguage(Markdown); + +// const treeSitter = parser.parse(sourceFile); + +const streamParser = new MarkdownStreamParser(); + +let accumulatedElements = []; + +markdownLines.forEach((chunk, index) => { + console.log(`\n=== Processing chunk ${index}: "${chunk.replace(/\n/g, '\\n')}" ===`); + + streamParser.processLine(chunk); + + const currentElements = streamParser.getCompletedElements(); + + // Show only newly completed elements + const newElements = currentElements.filter(elem => + !accumulatedElements.some(acc => + acc.type === elem.type && acc.text === elem.text + ) + ); + + if (newElements.length > 0) { + console.log('Newly completed elements:'); + newElements.forEach(elem => { + const preview = elem.text.replace(/\n/g, '\\n').substring(0, 60); + console.log(` - ${elem.type}${elem.level ? ` (h${elem.level})` : ''}: "${preview}${elem.text.length > 60 ? '...' : ''}"`); + }); + accumulatedElements = currentElements; + } else { + console.log('No newly completed elements yet'); + } + + // Debug tree for specific chunks + if (chunk.includes('\n\n')) { + streamParser.debugTree(); + } +}); + +console.log('\n=== Final completed elements ==='); +accumulatedElements.forEach(elem => { + console.log(`- ${elem.type}${elem.level ? ` (h${elem.level})` : ''}: "${elem.text.substring(0, 50)}${elem.text.length > 50 ? '...' : ''}"`); +}); + + +// console.log("\n\n\n TREE", treeSitter); +// const callExpression = treeSitter.rootNode.child(1).firstChild; +// console.log(callExpression); + +// Variables to maintain state +let content = ''; +// let tree = null; +let subscribers = []; +let isParsingActive = false; + +// Functions to mimic the original API +// const startParsing = () => { +// isParsingActive = true; +// content = ''; +// tree = null; +// }; + +// const parseToken = (chunk) => { +// if (!isParsingActive) return; + +// const textChunk = typeof chunk === 'string' ? chunk : JSON.stringify(chunk); +// content += textChunk; + +// // Parse the current content +// tree = parser.parse(content, tree); + +// // Create a simplified representation of the parse tree +// const parsedSegment = { +// ast: simplifyNode(tree.rootNode), +// content: content, +// status: 'PARSING' +// }; + +// // Notify subscribers +// notifySubscribers(parsedSegment); +// }; + +// const stopParsing = () => { +// isParsingActive = false; + +// if (content && tree) { +// // Final parse +// tree = parser.parse(content, tree); + +// const finalSegment = { +// ast: simplifyNode(tree.rootNode), +// content: content, +// status: 'END_STREAM' +// }; + +// notifySubscribers(finalSegment); +// } +// }; + +// // Helper to simplify the tree-sitter node structure +// const simplifyNode = (node) => { +// if (!node) return null; + +// const result = { +// type: node.type, +// text: node.text, +// startPosition: node.startPosition, +// endPosition: node.endPosition +// }; + +// if (node.childCount > 0) { +// result.children = []; +// for (let i = 0; i < node.childCount; i++) { +// const child = node.child(i); +// if (child) { +// result.children.push(simplifyNode(child)); +// } +// } +// } + +// return result; +// }; + +// // Subscription management +// const subscribeToTokenParse = (callback) => { +// const subscriber = { callback }; +// subscribers.push(subscriber); + +// const unsubscribe = () => { +// const index = subscribers.indexOf(subscriber); +// if (index !== -1) { +// subscribers.splice(index, 1); +// } +// }; + +// return unsubscribe; +// }; + +// const notifySubscribers = (parsedSegment) => { +// for (const subscriber of subscribers) { +// const unsubscribe = () => { +// const index = subscribers.indexOf(subscriber); +// if (index !== -1) { +// subscribers.splice(index, 1); +// } +// }; + +// subscriber.callback(parsedSegment, unsubscribe); +// } +// }; + +// // Stream JSON chunks (unchanged from your original code) +// type JSONChunk = string | object; + +// async function* streamJSONinChunks(jsonArray: JSONChunk[]): AsyncGenerator { +// const delay = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); + +// for (const item of jsonArray) { +// if (item !== '') { +// yield item; +// await delay(DELAY); +// } +// } +// } + +// // Main execution +// (async () => { +// console.log('\n'); + +// const jsonContent: string = fs.readFileSync(sourceFile, { encoding: 'utf-8' }); +// const parsedJson: JSONChunk[] = JSON.parse(jsonContent); +// const textStream = streamJSONinChunks(parsedJson); + +// startParsing(); // Start parsing before creating the stream + +// for await (const chunk of textStream) { +// parseToken(chunk); +// } + +// stopParsing(); // Flush any remaining content at the end +// })(); + +// // Subscribe to parse events +// const unsubscribe = subscribeToTokenParse( +// (parsedSegment, unsubscribe) => { +// console.log('parsedSegment', parsedSegment); + +// // Unsubscribe at the end of the stream +// if (parsedSegment.status === 'END_STREAM') { +// unsubscribe(); +// // No need to remove instance as we're not using the class approach +// } +// } +// ); diff --git a/package.json b/package.json index a2da289..26a17c4 100644 --- a/package.json +++ b/package.json @@ -24,14 +24,16 @@ ], "scripts": { "build": "tsup src/markdown-stream-parser.ts --dts --format esm,cjs --minify --out-dir build --no-sourcemap", - "debug-parser": "node ./demo/debug-scripts/char-streamer.ts", + "debug-parser": "node ./demo/debug-scripts/char-streamer-tree-sitter.ts", "prepare": "pnpm run build" }, "devDependencies": { "chalk": "*", "typescript": "*", "tsup": "^8.0.0", - "semver": "^7.5.4" + "semver": "^7.5.4", + "tree-sitter": "^0.21.1", + "@tree-sitter-grammars/tree-sitter-markdown": "*" }, "dependencies": { }, diff --git a/src/tree-sitter-markdown-stream-parser.ts b/src/tree-sitter-markdown-stream-parser.ts new file mode 100644 index 0000000..c4108f6 --- /dev/null +++ b/src/tree-sitter-markdown-stream-parser.ts @@ -0,0 +1,163 @@ +import Parser from 'tree-sitter' +import Markdown from '@tree-sitter-grammars/tree-sitter-markdown'; + +export interface CompletedElement { + type: string; + text: string; + level?: string; +} + +export class MarkdownStreamParser { + private parser: Parser; + private currentTree: Parser.Tree | null = null; + private content: string = ''; + private lineCount: number = 0; + + constructor() { + this.parser = new Parser(); + this.parser.setLanguage(Markdown); + } + + processLine(chunk: string): Parser.Tree { + const oldContent = this.content; + const oldLength = oldContent.length; + + // Don't add newlines automatically - let the content flow naturally + this.content += chunk; + + // Track if this chunk ends with newline + this.lastWasNewline = chunk.endsWith('\n'); + + // Incremental parsing + if (this.currentTree) { + // Calculate positions based on actual content + const lines = oldContent.split('\n'); + const startRow = lines.length - 1; + const startCol = lines[lines.length - 1].length; + + const newLines = this.content.split('\n'); + const endRow = newLines.length - 1; + const endCol = newLines[newLines.length - 1].length; + + this.currentTree.edit({ + startIndex: oldLength, + oldEndIndex: oldLength, + newEndIndex: this.content.length, + startPosition: { row: startRow, column: startCol }, + oldEndPosition: { row: startRow, column: startCol }, + newEndPosition: { row: endRow, column: endCol } + }); + + this.currentTree = this.parser.parse(this.content, this.currentTree); + } else { + this.currentTree = this.parser.parse(this.content); + } + + return this.currentTree; + } + + private hasErrorInSubtree(node: Parser.SyntaxNode): boolean { + if (node.type === 'ERROR' || node.type.includes('MISSING')) { + return true; + } + + for (const child of node.children) { + if (this.hasErrorInSubtree(child)) { + return true; + } + } + + return false; + } + + private isNodeComplete(node: Parser.SyntaxNode): boolean { + // Check if node has any ERROR nodes in its subtree + const hasErrors = this.hasErrorInSubtree(node); + + // For code blocks, check if we have both delimiters + if (node.type === 'fenced_code_block') { + const delimiters = node.children.filter(child => + child.type === 'fenced_code_block_delimiter' + ); + return delimiters.length >= 2 && !hasErrors; + } + + // For paragraphs, they're complete if followed by a blank line or end of content + if (node.type === 'paragraph') { + // Check if there's a double newline after this paragraph + const nodeEndIndex = node.endIndex; + const afterNode = this.content.substring(nodeEndIndex); + return !hasErrors && (afterNode.startsWith('\n\n') || afterNode === '' || afterNode === '\n'); + } + + // For other nodes, they're complete if no errors + return !hasErrors; + } + + getCompletedElements(): CompletedElement[] { + if (!this.currentTree) return []; + + const completed: CompletedElement[] = []; + const rootNode = this.currentTree.rootNode; + + // Find all headings + const headings = rootNode.descendantsOfType('atx_heading'); + headings.forEach(heading => { + if (this.isNodeComplete(heading)) { + const inline = heading.children.find(c => c.type === 'inline'); + if (inline && inline.text) { + // Extract heading level from marker + const marker = heading.children.find(c => + c.type.startsWith('atx_h') && c.type.includes('_marker') + ); + const level = marker ? marker.type.match(/h(\d)/)?.[1] : '1'; + + completed.push({ + type: 'heading', + level: level, + text: inline.text + }); + } + } + }); + + // Find completed paragraphs + const paragraphs = rootNode.descendantsOfType('paragraph'); + paragraphs.forEach(para => { + if (this.isNodeComplete(para) && para.text) { + completed.push({ + type: 'paragraph', + text: para.text + }); + } + }); + + // Find completed code blocks + const codeBlocks = rootNode.descendantsOfType('fenced_code_block'); + codeBlocks.forEach(block => { + if (this.isNodeComplete(block) && block.text) { + completed.push({ + type: 'code_block', + text: block.text + }); + } + }); + + return completed; + } + + getCurrentTree(): Parser.Tree | null { + return this.currentTree; + } + + getContent(): string { + return this.content; + } + + debugTree(): void { + if (!this.currentTree) return; + + console.log('\nCurrent tree structure:'); + console.log(this.currentTree.rootNode.toString()); + } +} From 40c6d7da866d4d9e45f5dc4aa74d40c998bb794b Mon Sep 17 00:00:00 2001 From: Dmitry Bondarenko Date: Tue, 21 Oct 2025 21:34:32 +0600 Subject: [PATCH 02/32] LIX-MDSP-5 # initial attempt to make web tree sitter working while maintaining compatibility with old api --- .../char-streamer-tree-sitter.ts | 312 +++------ demo/svelte-demo/package.json | 6 +- demo/svelte-demo/pnpm-lock.yaml | 13 + .../assets/parsers/tree-sitter-markdown.wasm | Bin 0 -> 379874 bytes demo/svelte-demo/src/routes/+page copy.svelte | 308 +++++++++ demo/svelte-demo/src/routes/+page.svelte | 115 +++- .../static/tree-sitter-markdown.wasm | Bin 0 -> 379874 bytes demo/svelte-demo/static/tree-sitter.wasm | Bin 0 -> 205488 bytes demo/svelte-demo/vite.config.ts | 15 +- package.json | 2 +- src/tree-sitter-markdown-stream-parser.ts | 630 +++++++++++++++--- ...ee-sitter-markdown-stream-parser_nodejs.ts | 481 +++++++++++++ 12 files changed, 1520 insertions(+), 362 deletions(-) create mode 100755 demo/svelte-demo/src/assets/parsers/tree-sitter-markdown.wasm create mode 100644 demo/svelte-demo/src/routes/+page copy.svelte create mode 100755 demo/svelte-demo/static/tree-sitter-markdown.wasm create mode 100755 demo/svelte-demo/static/tree-sitter.wasm create mode 100644 src/tree-sitter-markdown-stream-parser_nodejs.ts diff --git a/demo/debug-scripts/char-streamer-tree-sitter.ts b/demo/debug-scripts/char-streamer-tree-sitter.ts index 07e867b..02eb5e2 100644 --- a/demo/debug-scripts/char-streamer-tree-sitter.ts +++ b/demo/debug-scripts/char-streamer-tree-sitter.ts @@ -1,14 +1,15 @@ import fs from 'fs' import Parser from 'tree-sitter' import Markdown from '@tree-sitter-grammars/tree-sitter-markdown'; -import { MarkdownStreamParser } from '../../src/tree-sitter-markdown-stream-parser.ts' +import { MarkdownStreamParser, type StreamingChunk } from '../../src/tree-sitter-markdown-stream-parser.ts' import { log, info, infoStr, warn, err } from './debug-tools.ts' // Parse CLI arguments const args = process.argv.slice(2); let DELAY = 0; -let fileName = ''; +let filePath = ''; +let CHUNK_LIMIT: number | null = null; for (const arg of args) { if (arg.startsWith('--interval=')) { @@ -16,227 +17,114 @@ for (const arg of args) { if (!isNaN(val)) DELAY = val; } if (arg.startsWith('--file=')) { - fileName = arg.split('=')[1]; + filePath = arg.split('=')[1]; + } + if (arg.startsWith('--limit=')) { + const val = parseInt(arg.split('=')[1], 10); + if (!isNaN(val)) CHUNK_LIMIT = val; } } -if (!fileName) { +if (!filePath) { throw new Error('Missing required argument: --file='); } -// const sourceFile = `/usr/src/service/demo/llm-streams-examples/${fileName}`; -const markdownLines = [ - "####", - " 🐾", - " **", - "Regex", - " Lesson", - ":", - " Evalu", - "ating", - " Cat", - " Bre", - "eds", - "**\n\n", - "Regex", - ",", - " or", - " **", - "regular", - " expressions", - "**,", - " are", - " a", - " powerful", - " textual", - " tool", - " often", - " used", - " in", - " programming", - " for", - " finding", - "```", - "python", - "\n", - "import", - " re", - "\n\n", - "#", - " List", - " of", - " cat", - " breeds", - "\n", - "cat", - "_b", - "re", - "eds", - " =", - " ['", - "S", - "iam", - "ese", - "',", - " '", - "Pers", - "ian", - "',", - " '", - "M", - "aine", - " C", - "oon", - "',", - " '", - "B", - "eng", - "al", - "',", - " '", - "S", - "ph", - "yn", - "x", - "']\n\n", - "#", - " Join", - " breeds", - " into", - " a", - " regex", - " pattern", - "\n", - "pattern", - " =", - " r", - "'\\", - "b", - "(?:", - "'", - " +", - " '|", - "'.", - "join", - "(map", - "(re", - ".escape", - ",", - " cat", - "_b", - "re", - "eds", - "))", - " +", - " r", - "')", - "\\", - "b", - "'\n\n", - "#", - " Sample", - " sentences", - "\n", - "text", - " =", - " \"", - "I", - " have", - " a", - " Bengal", - " and", - " a", - " Maine", - " C", - "oon", - ",", - " but", - " my", - " friend", - " prefers", - " S", - "ph", - "yn", - "x", - " cats", - ".\"\n\n", - "#", - " Search", - " for", - " matches", - "\n", - "matches", - " =", - " re", - ".findall", - "(pattern", - ",", - " text", - ",", - " flags", - "=re", - ".", - "IGNORE", - "CASE", - ")\n\n", - "print", - "(\"", - "Cat", - " breeds", - " found", - ":\",", - " matches", - ")\n", - "``", - "`\n\n", - "---\n\n", -] - -// Initialize tree-sitter parser -// const parser = new Parser(); -// parser.setLanguage(Markdown); - -// const treeSitter = parser.parse(sourceFile); - -const streamParser = new MarkdownStreamParser(); - -let accumulatedElements = []; - -markdownLines.forEach((chunk, index) => { - console.log(`\n=== Processing chunk ${index}: "${chunk.replace(/\n/g, '\\n')}" ===`); - - streamParser.processLine(chunk); - - const currentElements = streamParser.getCompletedElements(); +const sourceFile = `/usr/src/service/demo/llm-streams-examples/${filePath}`; + +// Get parser instance with unique ID +const markdownStreamParser = MarkdownStreamParser.getInstance(filePath); + +type JSONChunk = string | object; + +async function* streamJSONinChunks( + jsonArray: JSONChunk[], + limit?: number | null +): AsyncGenerator { + const delay = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); - // Show only newly completed elements - const newElements = currentElements.filter(elem => - !accumulatedElements.some(acc => - acc.type === elem.type && acc.text === elem.text - ) - ); + const itemsToProcess = limit ? jsonArray.slice(0, limit) : jsonArray; - if (newElements.length > 0) { - console.log('Newly completed elements:'); - newElements.forEach(elem => { - const preview = elem.text.replace(/\n/g, '\\n').substring(0, 60); - console.log(` - ${elem.type}${elem.level ? ` (h${elem.level})` : ''}: "${preview}${elem.text.length > 60 ? '...' : ''}"`); - }); - accumulatedElements = currentElements; - } else { - console.log('No newly completed elements yet'); + for (const item of itemsToProcess) { + if (item !== '') { + yield item; + await delay(DELAY); + } } - - // Debug tree for specific chunks - if (chunk.includes('\n\n')) { - streamParser.debugTree(); +} + +(async () => { + console.log('\n'); + console.log(`Loading file: ${sourceFile}`); + console.log(`Delay between chunks: ${DELAY}ms`); + if (CHUNK_LIMIT) { + console.log(`Chunk limit: ${CHUNK_LIMIT}`); } -}); + console.log('\n'); + + try { + const jsonContent: string = fs.readFileSync(sourceFile, { encoding: 'utf-8' }); + const parsedJson: JSONChunk[] = JSON.parse(jsonContent); + + console.log(`Total chunks in file: ${parsedJson.length}`); + console.log('Starting parser...\n'); + + let chunkCounter = 0; + + // Subscribe to parsed segments + const unsubscribe = markdownStreamParser.subscribeToTokenParse((chunk: StreamingChunk) => { + if (chunk.status === 'START_STREAM') { + console.log('=== Stream Started ===\n'); + } else if (chunk.status === 'END_STREAM') { + console.log('\n=== Stream Ended ==='); + } else if (chunk.status === 'STREAMING' && chunk.segment) { + console.log(`Segment:`, JSON.stringify(chunk, null, 2)); + } + }); + + // Start the parser + markdownStreamParser.startParsing(); + + // Process chunks using async generator + for await (const chunk of streamJSONinChunks(parsedJson, CHUNK_LIMIT)) { + chunkCounter++; + + const chunkStr = typeof chunk === 'string' ? chunk : JSON.stringify(chunk); + console.log(`\nProcessing chunk ${chunkCounter}: "${chunkStr}"`); + + // Send chunk to parser + const error = markdownStreamParser.parseToken(chunkStr); + if (error) { + console.error('Error parsing token:', error.message); + break; + } + } + + // Stop the parser + markdownStreamParser.stopParsing(); + + // Display final summary + console.log(`\nTotal chunks processed: ${chunkCounter}`); + + const summary = markdownStreamParser.getSegmentsSummary(); + console.log('\nSegments by type:', summary.byType); + console.log('Total segments generated:', summary.total); + console.log('Final content length:', markdownStreamParser.getCurrentContent().length, 'characters'); + + // Cleanup + unsubscribe(); + MarkdownStreamParser.removeInstance(filePath); + + } catch (error) { + console.error('Error processing file:', error); + process.exit(1); + } +})(); + -console.log('\n=== Final completed elements ==='); -accumulatedElements.forEach(elem => { - console.log(`- ${elem.type}${elem.level ? ` (h${elem.level})` : ''}: "${elem.text.substring(0, 50)}${elem.text.length > 50 ? '...' : ''}"`); -}); +// console.log('\n=== Final completed elements ==='); +// accumulatedElements.forEach(elem => { +// console.log(`- ${elem.type}${elem.level ? ` (h${elem.level})` : ''}: "${elem.text.substring(0, 50)}${elem.text.length > 50 ? '...' : ''}"`); +// }); // console.log("\n\n\n TREE", treeSitter); diff --git a/demo/svelte-demo/package.json b/demo/svelte-demo/package.json index 2071606..2d1da56 100644 --- a/demo/svelte-demo/package.json +++ b/demo/svelte-demo/package.json @@ -14,7 +14,8 @@ "generate-manifest": "ts-node scripts/generate-llm-examples-manifest.ts", "copy-llm-examples": "ts-node scripts/copy-llm-examples.ts", "predev": "pnpm copy-llm-examples && pnpm generate-manifest", - "prebuild": "pnpm copy-llm-examples && pnpm generate-manifest" + "prebuild": "pnpm copy-llm-examples && pnpm generate-manifest", + "postinstall": "cp node_modules/web-tree-sitter/tree-sitter.wasm static/ && cp src/assets/parsers/tree-sitter-markdown.wasm static/" }, "files": [ "dist", @@ -52,7 +53,8 @@ "tailwindcss": "^4.0.0", "ts-node": "^10.9.2", "typescript": "^5.0.0", - "vite": "^6.2.6" + "vite": "^6.2.6", + "web-tree-sitter": "*" }, "keywords": [ "svelte" diff --git a/demo/svelte-demo/pnpm-lock.yaml b/demo/svelte-demo/pnpm-lock.yaml index 4909255..71cdb65 100644 --- a/demo/svelte-demo/pnpm-lock.yaml +++ b/demo/svelte-demo/pnpm-lock.yaml @@ -60,6 +60,9 @@ importers: vite: specifier: ^6.2.6 version: 6.3.3(@types/node@22.15.3)(jiti@2.4.2)(lightningcss@1.29.2) + web-tree-sitter: + specifier: '*' + version: 0.25.10 packages: @@ -1083,6 +1086,14 @@ packages: vite: optional: true + web-tree-sitter@0.25.10: + resolution: {integrity: sha512-Y09sF44/13XvgVKgO2cNDw5rGk6s26MgoZPXLESvMXeefBf7i6/73eFurre0IsTW6E14Y0ArIzhUMmjoc7xyzA==} + peerDependencies: + '@types/emscripten': ^1.40.0 + peerDependenciesMeta: + '@types/emscripten': + optional: true + whatwg-encoding@2.0.0: resolution: {integrity: sha512-p41ogyeMUrw3jWclHWTQg1k05DSVXPLcVxRTYsXUk+ZooOCZLcoYgPZ/HL/D/N+uQPOtcp1me1WhBEaX02mhWg==} engines: {node: '>=12'} @@ -1950,6 +1961,8 @@ snapshots: optionalDependencies: vite: 6.3.3(@types/node@22.15.3)(jiti@2.4.2)(lightningcss@1.29.2) + web-tree-sitter@0.25.10: {} + whatwg-encoding@2.0.0: dependencies: iconv-lite: 0.6.3 diff --git a/demo/svelte-demo/src/assets/parsers/tree-sitter-markdown.wasm b/demo/svelte-demo/src/assets/parsers/tree-sitter-markdown.wasm new file mode 100755 index 0000000000000000000000000000000000000000..e6aee074c4e3891221edec770620eb5919bf38ae GIT binary patch literal 379874 zcmeFacbr|vbtd@ky)Vb6L|e2d+YN|g%eH9A9@!()VfbcEQnKY4@2+>(yX#$wBXL#& zslYG}yWU|VXP}WY&`3lE8ad|-G;+>4r$)}eoI0n@J@4FlkM_?$|4H&k_N)5pe)XMD z;nuzVI`+;-Kk;Jl&kN}v{=@q}{^@(a{Y@{q_iiDcjCtSAe&7G(li2q?{LlNul;Hot zC-@^4`+f{caVPBsMW7<__+(BDtq{mIWh{D-&y z^E)5C8+;Noz2x71`{Q^1^ZV~s)+MQ#AB&q7zx4LoKmO?tfBfOQKLDZs^`}4h_{Tr{ zDP80JJeCZKg+wA5kH=%_Amb&2)LX#|Z@>L_;NQ26&^FR5jS3Y^A&<=jg%(we}KL#JpQh7~a%NMCUykN_%RGy!( zZFwM-r)O+= zD3u?kY!{q!-EE?BPv&o-@<8_HPAZ>e{=HOQ_9ff@K`QTI_Z_A3;W~?-r1BUp zpQUmqE?=bbrIKxbmCEOW?82yRN*NMuV+5O$9 z+?cif_o8wa=4+(#hM2_%+V+eOrSclaM^d>D>uaL&3~oPx%B^hAWGeUMcul8r1DBhr zJf5$gN98r?c5OWxKZ~g5C|kUY%1p3|iW?bTM`iZ^CMu3(|FlqfGmn;?RPM(9+DqlT z-2NbyCo=s}Di2}%Pg40f`}ZuBr?5SZ*D*e;`f`T@siNs6>+qqr!D8Q5jHHUzi!7;$ zDjIJT+O>_5PbN@R%UzZ=*|Kb9mg!W{(83kXmL;s1XIt#z7K^B&X)jkSql(tK+{-Fk z5w=)oDtywklfcHq+5MKcVucTyAK^cunAP zCn}HQayKgT2<$~gPLxI}a#jt*B4y}Mf{w6ZBdL6d%S}{X&mB#m@)|Brrt%6dPp9%M zzP_2t8+fglN9Aoi7cHW4BPZrEDzE2(zKY6|xx9|bgE(Z)){Rs?$mM}l9?R{AQu!g{BdI)?%S}{X zk(7BBYU=HrJDJe)?A_@VSaZeadDN~0vo50Y^f2mFzRuI&Dk>l5@;WMS;PNIa&*5?l zl}B@VCzX3}c`ucxvDXh$`5LcCN2xrHqj-|a!?}Fcw&yIoNafC)^;fApkjt%9e!zaY zL**rW{R1k`R+KQ@MflOrUa4)<2obU0MHhDtG5RZKm>8&hvRx9?E&Sh{_My zKg+1Rhi9`@RNld}(K;%x=j%67c>zhLoYRGz~5yo$=r+k?h z9>V1YxH*C7-(bE@RNlztZdBgHdU{d$I+q)5ne`8(@?kCyrSb@l&qykdWW0&W?HQjy zh(HYva|&%CHm3*T?=t?BDsZ`j}a>(x&pYSY3TA_X1Rz z&o3CH-mfd<4)!^XeKG{0%^M6N#eH9ywYf1?nf^g_W08GhM& z4|d-9IQB<9cpt<~&v;>?{M`GXEnZM)8~jBfRv)xU8exHH0%Y&iC*OivBdu+`T=&5n zP)L=%51_vo;5XpvBnZwYK?!ID(sX^Y?0?)2WYrbpuY;TvAaH#F?mF8Buf+T2@Cv4IBKBU<4?w@s{IXGRw7qP6_9FPKd=e~&Y_2Qi{7*=2iTpz@ ziGllE7eN9VmtywjUjTEB3Bl`SuT21H14-8=Wm3%i%c01;S4?;b-%p^#`ZvLQiSlp0 zRj}6Ar`~`ddvAS6&V7O043WT39!A#+gS0RqgqVl`<5gF8n?%{qfh6cE5@pCnyrgZK zjBS%NoS0rerGo0Ron@(mNaBDe%D3BLQG@e$@|@aOQ!D;F`* zoS2x5r`)X%-k`9&@nM?)LXv{m0pl;0Kl!LW_z=eX8!$d$ltJAKVI7P@Xqqrnf(e7^ z$;1-Fjor6_DG01E(@0wz_HyF8ZR6mzB3^={uHFaH(CX!OCNIiQg2Q z!6C*gv0MrMXGXla)D2}Mej&eT0{4M!Chh8DG@>Z=;$=*|Adv_l_2Okry*P}Ew+b-E zA^avm!Y)l#NeRt^(B>lDLn{;50LDFs=ARJ>PzY0c0!A9TKW=6soPKqWX6~wzPd1!n zly9as-IPKc5Ptvny`WczXX+Km{{wBkIMrZ8scUN`!ZyzSuELORkV7!ilnnP=Bj>ne6{RMg#w8)BN%u!(5kxMahn4h>e*F{dnAmd^CY9z|gC75t{5yDA;owA9K$u z=fOae@)!w|dvIaRu}Uy9KrK8vh_~TXs<*5xR)hcbOMV?!Vk7FwP)4PCdU|2zLp# zd88leBVNwFWj0Z@M?Q8D9r<{18<@3dXvB?mvA~!kOws;qTHBHFTP7XMJ3C3^=MJefjH{Z?GpN zy&$A=vt!(1wy}^Il?rO7lJz4cGhYpfm4jsyyn)*Z>lf=c95r#@kF#@(8PI|8J8bo6 z;sf`*i7SxGELgV_X5EgzTpz%6Z#sqenc3g$k70=maAeRnyUxrU5VH5$!tM_iG&AWy zA4RwhrXAR>RIJgoj{d?Mg+%!mfYq|VPC9421wze*&?EGQPWNDGEQeco<13V4S49&+ zTnOXWd?`7#VE zv&G;U9`;zUct9fHjnW&1EZP7r^58%4`%7;W@ZJYLezB0q<2`_dz63j_JO=y=HV$8a zv0)}5w9C+%Jzrq#3&RI6`(`b$);3B0vs>*X?RS#pS8*0bIq5gRVAyKo{tWuDU79guyrX8su>Z2JFz~49&CEtP zMH&}F1>QkHMuz)q8?H8HEQR}P4>rvZJrAQ-aU~>0f+Og?T%T&!2JQ^oz}D1k3&Ex4 zGMfGLMds!b_u8h9Hd*j{8vZjeFt>w1VRqcC0E%$W8m535m~%qpS;-Iv4t zZNklr&3p;V5!{&7n>i9jwQ=}YW97Q{+9qFtxeR>ow@thn`xTnHewBXI(~mav zqb>dTD*bqce*7Cd<3Ok3WCNo+fZ*@|hrUeN%qKMEA=)-pZu1rlkbjRk{_kN*gP{Qf z`_)(*LkTxF6^teYDrn$y8}OfL@B-ZV#mir;E16@X&%DH>OAOM5A`yypsBJql$AXx8 z5CbY-h0VITzcqRHnY!w$>57$C$KEUd(qG_wfV6Fmjs8Eu)wVjd8;og64!+(;od?W@LV)6;1ho*C$eD$B>!>@)uH0K&7Ps|-!`5WLu;7NUU z1fWyANvnq+I7PwfnmI**l4mH-n8LI(chBABGiWt!#s5U5o z9qxIdueyZUl0gYZW2~Gt_vXkQYiCY>Xp~wLXL#TNgL&T)Fnj8of)3325v2LA8ef8n@~?uwjDyWNG1UKxsfXcP{uNtK zV=iHRX>Mx3R)~z5eoc4go))hECsG@XH3C0#FgfDQ%b!|22z-SFzM=%GV530j4kF^i z9s%3|7vjVfo-P^JVF+M-w=RAa&zb|Ax5L>2qF6PAbS=b@B8d|XobADu6&ws4<|4QW ziNPvlPUItrLA$2_$bkp{Z z+j+_O7yMAE5RAj^PLvr{K^hndAgLY(R{;YYy}FwB^C^47PDX-7zh=|my|Vw^f^n2F zHxW;OeX;Vd!(A^-5s)F~fV^CQ^#=E!_1?$j_6-fG5AfLxJT!p$0uC|QD4ZAd@C9p( z@G9#$)|)8dUAXZ>(%ec(sNhn(&gQboXJu+N2SHohC+pdPnIc!v5V1DG7&pg>VOp7c zha6^0j5c8D$ki}Y8S!R}L^Fp@_UHh|4BLX)Uxdu8rcD|yhw1U()R=vMSw?XVrs2AF zuRz#gAC@RT1JC1ABYMn}h{enU71)QrYUVT;Adp|Nwn3E>95a9umH`^xR2aHy8B5PzoXOOetF31eC zxWTbX*sIB}&rkt_(bBhh5ARP(=BPb}=QV!$wfD-Y@7DXUvcCtX#4umj{vJcwJY;~A z6TEkUQ66^4&d11A)Ym7gp6;6%fCzIm04}TdKltzsI zO}yL0fhLYC<7apU(gV^0&%5A<8AUv2(lWzK70fdDWVihFKZA82M^wb$;EiyOsmNk8 zxd8AgrRHn&!=xh_gR@06ZpIgtBY|QE=O%Hp9Q{*4hIr6jrEwORXKtCyItgj}8JC9p z72^X+d3crx&j)-jnKUa9Ji|6yGn0jID+&uSJa{xy9ejdE_i$HMewAhs^A>>W2|QEv zv83G|#Bma=kHKTNFPhUzJfv!4oNTUxqXQp)n8CzH2PR;qk-2#?HbGLErgVPL4v#SL zJpq`a3}&7v!_?(}01lvc7vL&$?1Cq&@YVwIL4R;~V!~p!txxcE05HUV-ArlbG#P$) z^y9b91P60GIMz4|n#dFdzlLNU5E!!0IP_P6skZ!8!P%3HSz&5zWL3JC}md zWkv{|8iOWqM#`)u;9>)$0jn$8<1KnLR4^}mz{7R>kOH3LwKK1D6u{eNIGfWh`1zmA z=&~I3>2U1BF_h^pWU5c&?i4SD^A`$&aX@(N;KBVu>zCTqnL|yK4DUODN6l*?}t;+h9;f|=F& zhI@b{qCAVeWnZ>|_IMj^7Gv|28UFKfbK+j`Nf{0`D4D=Xe%XJoT=$>wO$)#1`(|o~ zD4G{M0=Q-4Q2Qp-=Go8(xO=jZ4(x0Uz-K&pg%!}w#^zm_A~_#lAEC2lGd#?uF%EMx z41O49^rA=^7JpDkYM^tRiN7m5P09&+LXt|EO9Qj}Ft6?WOH;8zV7+<21-G8?oW#7h zV&st5@uPUgWI4#g9lLbxTi-bEQ-FcSBp3kTdAJF(0|2Huyc0Fip(Zs3HD*|mMqou) z%*2b#@~+QkhXp3+_uvXzB#m*n<@C#6z&$c<^~|!9>+JE{DmV&Z()yYo$F&8<9zItB zxnS+bV2Sq;REg`Yst-KYvCJfXg|kWFVnc*!$;A^ z>G<#-7Q@Pi3*l=>F#ceN4^tkj^RGX0-X7hh~$7-PS=5kdIIbUZ=5y!eH*B z-7r1phIOpkbMA13s!(j#ZkX@tz>u$$f0^XqD@%BXdJ>jG6BHxwmp~K=FmJ{b{4a<5 zJ9w{#5&th>Px@upat7vUjPV4HvLF^SuY{l%a0bIqW_C5!!`lyVcVv8Jv(I$Mw=)sB z=8V)Y|56#=JTTW9siBi&-K@_}7u0{VgHnIHmx9*=*_r^0J7kLrrA#b83j z-d~HE#TL-I*J7AY=8Zs@`{6|r=+ytXtq+lc*UcbRFxQ@BVHXcbqFwHp2a}xXy7Ik7 zvy#-|-0j1%0492`#-J)spJ3rJlQq=&_`JY;!YB1uXT%$7*dCZ`q5Hoz%Q1W~QzQzo zOzTcrFuz)#)Pq{$TF@#58yxj`#rQJ1`1AjN?t#xe z@VN&*_rT{K_}l}Zd*E{qeC~nIJ@B~)KKH=q9{Ah?pL?L12kQNAz@bEOOkK?LgLopD zN@udUe4$vXd*KUTeDN24@k=lL(wATU$}j)QuhzF|`_)(e&9D9X*M8$Sf9vb7{`NP% z`8&V+Z@=|>zyI&P{Rgl8;UE3+pZxpR%m3kj_|rGueCs>k{j)#+-e3I3|MZvtp-+$*n|KNx3{+IvfM?Y@&-uplK>Cb-t!G|Ax{9pg} z@BaQD{@?%aAOHXV_DO@+-s|9X^g4N+y)Ir?ubbE1>*4kEdU?IQK3-q1(d*~+_Xc;GtZ-ckd+vIKbws>2; z7H^xk-P_^q^mci>y*=JuZ=bi{JK!Dk4ta;YBi>Q(n0MSe;hpqOd8fTI-dXRQciy|; zUGy$_m%S_ARqvX2-MitndN;jW-fi!Wch|e;-S-}N54}g;WABOg)O+SV_Zs~6eh0s! z-^uUnck#RW-Tdx;55K41%kS;?@%#FXem}pzKfoX85Ap~5L;RusFn_o|0uF{o`J??Y zev?1eALozvC-@WnPyI=7ZZyT8>QD2h`!oER{w%-QpY6}_=lb*f`Thcbp})vq>@V?` z`pf*~{tADkzsg_jukqLV>-_cp27jZ!$=~d6@wfUd{x*NRzr)|@@A7y1d;GorK7YS| zz(438@(=q*{GJ)lLC>I9&^zc8 z^bHz=enJ0WKrk>E6bufA1Ve*i!SG;2Fftewj1I;GO~Ke;TrfVE5KIg{4JHMXgDJt( zU|KLem=VkjW(Cc`>|jnXH<%a94;BOqgGIsOU`envSQab~Rs<`9Rl(|DO|Ujt7pxC9 z1RH}*!RBB~ur+82wguaR9l_3ESFk(S6YLH41^a^o!NK5Aa5y*;91V^I$Ac5W$>3CQ zIye)Y4bBDUgA2jM;8JioxDs3qt_9bF8$oMuGq@Gp4(KaTA>J|GDc(8WCEhjOE#5ueBi=LKE8aWaC*C*S81EPFA0H4O7#|cL93K)N z8Xp!P9v=}O86OoN9Ul{KijR$ti;s^_h);}v8lM!O9G?=O8lM)Q9-k4P8J`tzj?a$I ziO-GCi_ecQh%byUiZ6~Yi7$;Wi!YC_h_8&Vim#5ZiLZ^Xi?5Gwh;NK;&VfVs>IqVs2ty zVt!&lVqs!YVsTaUyXtaVl{-aVBv#aV~K_aUpRr zaVc>*aV2pzaV>E@aU;>1xS6<>xShC@xSP0_xSx2Cc$j#Uc$|2Wc$#>Yc%EoTwoi6Q zc1(6kc20Ilc1?Cmc2D+5_DuFl_D=Rm_Dwb>`z8A)2P6k32PFq5ha`t4hb4z6MysOj8&E&1*?c|;0-Q>OG{p5q>!{npnllXFV#OaAT=;GC^a}WBsDZOEHykeA~iBKDm6Ma zCe@T0n;MrIpPG=GnEEs|DK$AYB{elQEj2wgBQ-NME7hEuotl%Ho0^xJpIVSwm|B!t zoLZ7vnp&1xo?4MwnOc=vom!Jxn_8DzpW2YxnA()uoZ6DwnrcaHOKnf>NbOARO6^YV zN$pMTOYKh`NF7WaN*zueNgYicOC3+0NS#cbN}W!fNu5odOPx<$NL@@_N?lG}NnK4{ zOI=UhNVTSJrf#Kfr|zWgrtYQgryisprXHmpr=Fyqrk>BX)4kHY(|yu?(~aqV>Hg^f>4E7%>A~qC>7nUi>EY=S>5=JC>Cx#i>8AA9 z^tkl+^n~=p^rz`b>B;FS>8a^y>FMbi>6z(S>E`t8^qlnE^t|-^^n&!l^rH0Q^pf<_ z^s@Bw^osP#^s4mg^qTbA^t$x=^oI1t^rrOY^p^D2bW3_$dV6|DdS`l9dUtwHdT)AP zdVl&r`e6D{`f&P4`e^!C`gr<8`egc4`gHnC`fU1K`h5CA`eOQ0`f~b8`fB=G`g;0C zx;1?>eJg!CeJ6c4eJ_1K{UH4?{V4r7{UrS~{Ve@F-H>UY>5%D|>6Gc5>5}Q1>6Yo9 z>5=J~>6Pi7>67W3Y0UJ?^v?{)49pD549*P649yJ749|?njLeM6jLwY7G-bwS#%0E5 zCS)dNKFv(ZOwLTnOwCNoOwY{7%*@QnG-qaK=49q(=4Iw*7GxG?7G)M^mSmP@mSvV_ zR%BLYR%KRa)@0UZ)@9abHe@ztHf1(vwq&+uS~A-*+cP^dJ2Sg7yEA(-do%ko`!feJ z2Q!BEGV$1^7~Co`uqr!!|VXEWzA=Q9^F7c-YKmorx~S2Nc#*E2UVt(lvd zTbbLLJDIzgdzt&02bqVNN14Z&Cz+?2XPM`jhHU$6hiu1er)=kJmu%N;w`})pk8ICu zuWavZpKRZ3W42$me|A82V0KV;aCS&`Xm(h3cy>f~WOh_`baqU(DLXbhE;~LuAv-bq zX?9X}a&}5~YIa(7dUi&3W_DJ#IXgQ$Cp$MgFFQZGAiFTTD7!ejB)c@bEW13rBD*rX zD!V$nCc8GfF1tRvA-gfVDZ4qlCA&4-lHHcwp52k%ncbD$o!yh&o86b)pFNO0m_3v| zoIR2~nmv|1o;{I0nLU*~ojsF1n?093pS_U1n7x#}oV}90n!T32p1qN6&ECx3%HGc2 z$==Q0%ihmE$Ue+I%0A9M$v(|K%RbLGYGN+^*d2+@9Rt+`ioY+=1M|+@ajz+>zYT z+_Bv8+=<-D+^O8@+?m|j+_~KO+=bl5+@;**+?Cwb+_l{G+>Kmo?q=>*?so1@?r!d0 z?tbn;?qTjx?s4u(?rH8>?s={u-#*_V-!b1Q-#OnU-!FU~K?FU>E@FVC;YugtH?ugVNqsc^Y)rEs-yt#G|?qtIHoS-4fWUAR-YTew%aUwBY>Sa?); zTzFD=T6k7?UT7$`FLo$)EOsh(E_Nw)Ep{t*FZL+*EcPn)F7_$*EjAYW75f(l6bBXu z6$clG6o(dv6^9o`6h{_E6-O7x6q|};i{pypixY|yi=P%J6(<*`6sH!a6{ir#Oi))H&i|dN(iyMj?i<^p@ zi(86Yi!H@%#qGr%#ht}n#ofg{#l6LS#r?$t#e>B|#lyuT#iPYz#pA^j#goNT#nZ(z z#k0k8#q-4r#f!yD#mmJj#jC|@#p}fz#n$4@;;rKC;+^8%;=SVi;)CMD;-li@;*;Xj z;hrK-H^JW zb;Ih0*NvzfSvRU~bltc*c#q*#cpm^>^M%(L{PK;GdD$R8#?+g4G=};CzFP1#_*7mO zgICd@9$teOXnsM)yk`K_-p6gtYYZnB6xy}>h8HunBYpF(4!pwfRd^%#GyqU<-(rAQ zjRyMWwNbj*d}ZcPA6|r^uQU1@fbfD@(VtKk0}ak0HC^AaIrC~mupsf9ORJv8yod6 zH4ZL}8rQ_ebqkH_CdL&jjDu=xTqAske$^!G4?uO};9_H3H`ln_*gE&$*vNmWiEv@m z#BiK-3Qg=JCcaQ%B2-%wo0irvaU1~EO@xb$iJe>%i%oU4{*8_Jml_HeMh(T>HRG=# zG_*kseX+t&sJ4bSFRNkb8~~~t3Kttg8(c$Q7*qGz|Hg*@OU;D~qvm2Z<0SPvI1!c) z)4V#&G{yRVOZ6F?u&`Q>Y)qC)hKs~7G-UWgVz^mRuPHDLl@yopO|txukm7fV;s{Yl zy%l#M+WuSA9$wCeSyr0=9#tQW()>HBhwlaA=2{>+4h33b_;vzlIoS25_)WDO6k2i~ zUk>%lH%S`;x-)T{l}nusbUPEnzD)*D%K>gnREh@ZVDxvJq9QW@o_K*>QJ;9_Z_{}T@vRFi73oCa?e8sQa5?GCiNiuA_NrK_z=_fz}? zIAwtyQQ)ivrcn^&iklYbMHwQ%Z40bSV&nyQV1Z6FKoHOdU^uE6(AP0w1H}&k&4F>M z!qKM|hXDJnz)>kwj({fb7n11Y=pELtu^fHj`ywIl5zrjs5Y5r67KZ@qEcq!41_GK~ zJf+Ed)*=^Ry(M2vDS&_`7f)&O9<|5?XtCsD$h8P)a&e(1?_P^sfZdjSI3*SWnp|9{ z$-C7e7hs5YpqRFC`l2}Hi>gJN9T}(bJ0!=0z~o%0ZlH>)#RNV^6BsedoYR+ z&>UUEwwj}3SkF3Wroehlfo7Yy-3jA%$1A@`ar@s5?1=3Srp$BTMp2n3m@~A%m)$<50HhNm4dKw%(5)SR+h8ixG`pBXo zz&)FImub;JKnF$Qr#pRaehmSqbsbl!KWDj@&<+Oy&D|jsuDPGp;FkJJmist-+b-mz z05>dfl(I~KRtxm60JNI+MvV#g)iB&wz49v*?ypva8)|v?P%GSPYK9vwHsQV+4fhlH z^&PE8qNKg^#PN<@ArR0U5)IAqxE6;18?3`dQ)&pX(E>ZjlLGLXvY0j+0_?F@jLBlO z1lVhV0klvcpbd}!Y6BkCGC+Wc0RlvNML_rZG-TJkJ`C$Q>8L*K`osEM0!O)lS`O1`Tbht0z}9Kh?s)NfSifm)ssYw1~2 zQx9Bh^qh_Ax#{SU0P3jR45JbmWC&;rZv->E3DUQ^r( zn`(;dVLej}qB#=*%^_B6j%y(Yk2Xqktk)D`w5GV~P}mt70nH&+YmO@+$06kntk)D` zwWhcnQt<8YQ~1tm7*qoAvVNOZ5Ck;2xI>d)a>$R;y^sLBnb<-@839c$j?&~89rCNR zj3S^p#8H~#g2Qp2tQX*_@+{VCZgG_6KJRd^rQ0b4G>7;@bDVQHI@4}U0FDkNAl7Sc zbcdN?PKGng$#Bd6^@L!-#9Ot!ErtKc!u* z0Jm){_N6gJ1-PSvg!S49=yT(QqoEUy!dK^zNP1ay*}4!hNIj)(Q^L@P%CzI*IUDK9|iU79!rVDH*VMF1Wo zw`jT+fNw;}omj8+p9*zq{l~(3)^9_KfaZ`WYmTE1#|6?M0Ec7+jcx(hIddph2~K7w3=n{C!@JO3nE)KUE@Z0!57mT)^;(ViT&p?cs98l8 z3BZ0FKuf9s7gVQMuR9f&>rM|kowm@HNr2OqdpTJu0IQin<4FLHsy$2-fD^caNz0U| zYMBRW$rOOO?ce|)oO;eyOg;DX)PvV+4H8PNVZWnc5lxr~Xby>t=Gf$MsQfL62DQL~pa4gt*}j@BG|9F9F?kpTSUi*#eX<`&Ot?%fXe zDB>1?y)}q7hyw7yogKa6x=rVvv>lWyIDz((;}Ot0#pPP(E=T84;t+u4tff@|0ZlH! z(BwNE^3gOx1z_cN1{8o7Hu5~y>rN#|y3-wDr#$-YSQdaiZ|8IYINWo|R0MRd;&a{W zcBfYoF7-UK93pN6G`F~1b8oA`E%m%T=t+|+0-9SKtGQbo?mxg?EY@oZag(OlT7yFB z*>rLV)@y2XhS_-T4L6>9J^XGi*l7J;#l{nA?Z&ea{;H+f_0w52K1!w+qWS|+{cZp* zHXF~q(T(S3XX7cJ)uwKCOtnuJ1>n(Q2QLEJSaG=~-{g?@A%_XTiEEcF0ocPf)dk?0 z*KWuJ;DJ}sg_bGd(=s=PGFc5#V7;c;5;oNo8^U^~c!)2sgtpEVwBKjHocf4M!mRM@l>mAkGNHqeQ1O03gdS{r>JHv$jc11!%E#KMIO6WN? z6B;fy3B5C#&}*H9mY`{y);cy_q_stWNNNedXAtM<(VzhQLPBp^*aWy`V>g+0T?lBu zNjSC6HIB{)p9nLuL5wd z2dP&9cx!BX6@YWS2N{5XmLpEra#lKW)>AM9;2G>G-P{SlPIwZ%f^VIl(}o4F&~hYx zTFwebj$M-lxNnn&W*My4+!7_tz1-n$PYEdin|h3r2?0$mKG)>SYLNpF&O2~cU|!V4 zho!gl<{husU5TT0S4*9)#!%P=;MwvDO^O2W^nZx9`vUOx>Lgi?fR-Z>&~lc9ayW18 zqJx0uK!=)XuqB)ZTf%AZ4=biYsI}AJ&RWyp?wZpeTx_PnmgqFN$e9KuYTBknwQLfA zhujgG0RRZYT)DhN?$Pji?GuTC*09jgKpBtqnnFULDHhbAka`}%J?TsU0nNSGaF~#7 z3PZLjJOlibijYCA4Owfgklm^oGPu};Y*RF3^GfFT;q8$5o&Ok)o`3s2^Nrg4+5vvO zHlM)FH-fbd5;g6gd5(W3(3C3xJ7^<0NB~~DZqpD$K=&#E)VSJq2Ks zR#i}GIpSI^XPP6&o<1U=ImCsUW2(b(kA@2ZnnPTuIi@%qD~SUE%^`Mcj>!&(eZ2|+ z%^`Mcj!6!Oy-P(vbBNuVM)56wN&;T}pH0`R0Zi-LfFCKpF(@(~XCRmu_pnB49f5YXh}K}|l~A-_lD z0EGLp^A)!>*Ys@-Uaz|nf9kG=IbB_$t`N{1;y2APH002+!g@_1-q938YEVc$FYry2 z9|G{hi*+>85zw8AJ9Vdnolfl_KtOYdpESoHhhr%1Km_3P14=%r=l$JM5-tEI&I!6J zL_n(%k83pp9W~o%autA=g|U?K2xxL~ttKDfkk27g1&Ej=0GnhN5(IRw5)9pI|FBo> zN37Qr;z3Q(uLgzGvr{IMK?3kTX(SB_1azn3S>0)))2ZF{AfP$KPnx5z!(lhX2xt!R zlji6Xa`58`%0sNz6k@if=v{+C>e+u=XmBE+xy2otyO+Z~mN*d59O4em(KF=WEyZfO zJrsaf+pVDf z$wS)e2*5K7b&B=6QwfLew2RZ}cJd?wnnS#lmiH84)K%b=;UzhC*uVOvzO8u z0ZlIc(BvH*^4>&_faVZ~XpRmJ$6y+w0`T~!AV@t2=NzqM2otY= zNmDe`ppg2=awP!kK2Ew3(4C57b*InG-$=rv7uLOvvI>B3>sEOp4cj<8RI_ig;q|&J zai{L;nbVbBVGz(9;!e%+)ZwsGI0BkO+@m?3)Z!38zg~v*np@naxgR^+b_E0=bZ6z; zIpEq#cjEQBD{-6d>XFmcE{dE0yc@8;Cy0P17k_HwGn&Os2v64KBfaVaZHOEbdV>EFfpgF{B&C%*`%%`mm0-8h2 z)*Lq+j%DIuNlIY&@lBcRE}9h&^SLvDYA6amd4{?HufLJnT=$wsW#6ygp|akd79 z)N>RYDZB`1ZgGd^KI3rP69EC3oOENo<`y?;?$ZvpJ+DSUbBLQX$0>)ye#Zp@nnV1d zIZlQge2zn=V7;agt2M<5hhi*E_yVw->~<3YO)mb>@vm zpgF`Jn&W85!M2c}uwGM$*_z^rLt&pFA)q`n*LrPz) z*A!y5rdaP#Y@js`0nH(1YmRjxhmH!?YYMSiQ>=9;>^G<&pgF{9&9TPeSVfUVKy!%M znq#%Yag*jR1T=@3tvOazOm(L!p6OlGCmGNz`ht~sK{KwbU_4#H7@0hwS(H(Hq8V3I zFrKMkystGvvnZo@NHZ?4U<6&^D`pooBQ%RLq9dxj+_bC)<5kTF&7zFxh$@UrD;O)E zwnpAAgl17jbVL=#B^8X7R!0s2p;?p>9Z`jGv0a^&&$F#aF`o@j=Vz*#8D0C-zpmBD-FLZFIN73#fz2Nn{Y{#9UWZBev39T z^W3KBpDIn~8X@+a`kUVgE&GE%Gat*>XeR)TqFtk{Rruz7Cf^m}gGNz4w5|%@?9b%8 zN_@~L%7-qf!q@zneAkH&8b$fg1y%TFeJ0-x;)6y}K6F79zL}rN*GhcQD9VQ}sKPhH z@JWOv(f*(!(N@Q#enYGRI2HR&!L!+MKK5sHo^FE(%ux~Ow<;p0|LcpmMVS zUW;EBeJy@sF={4SZUn6g1+DVRze!&mvaV7PR9iu-Y6)6XQxIHi1g(k+n&1dR{~AHd zLP5*I-y7IiDF~|BL+fgJ2!QGyf{TryWl=%n9YGjHjP7dtzV$5nMYZv6ct15&M*N~s z{G#yNXtq>}hiWT+XD#u&Yl?@9jrc`T@naqF6Isx_P|!TD{3{gRZIyzc+6rn|U&EUn z0I2RwxY!7q7Zudx2%2C7DR0_u3uy`gc=N+-rtRZospqffa4z8AAF`)}as|KN--|}M z0DLrQf6rY2J{4J0afQAK!l;`NH-{c=_R7CT9^F~#QK;qtQ_G`kYI+ndHXdz`dUUko z(QzE&>7k(M;kQoisT2g&R#4-H8X@ivK=lyA#YWKdsGw1fps_4yawuqWI0*Jv3W91Y z=wvNHr)vs=i;bYkQ9&aeK@wgb!8V0PhB+6>Ed+Gj$AoPoBqKr~LLvY^n6kg*jeuq! z9g;-Yhlc>O+b{GMfPWu!2j#2){E&Sf9dHQ1>A#S+-2!m>Pp9)@0r;D~?fV%BXbVP# zLL(Lo3xS9Q0`Mbrs|NwiE|H3`53R*60JE>4gAD{UyF@C&KEz?4LkCU*@C$o4=wY=0 z?8~tgSLklVy^(GQhu!iFMjTSl2xxNgZiIYLNX{f9i5&q=g1$17-Gp!`PYAaw zhbo6MRP)@jvDVxIK=ru=E;d7XLUbq(aE7wDm#rE@2c-jyRa(4wE7G=q=mEH4$E9Wp zon{KK(E|7Bk-GqUEnvUS5CPq-_?NpqM=#3sb0rQmRmQ(fp?{k~{~oRMFH~FqHrMj+ zoSOcHi;aJqqW!{Zbu0pYVt)<;bLfVH4 zdGoW0wlWCl4$*S#kV)*zj}2~zVh6W9O)JCm0^hc93_8|UmTV%*%w=K$2yteHl(-r|<2=-zCMW@T`kPikrJaR7QF`n*Mm%(lLZ4J?`ye!Gn^j_egIEBbn(S{73lNDt z0$L)vw_3ON!)|%@wD%wgXcF-W``y}kFC^i&J?(xR0ZoEFsix#^NOH?MfW{xzYYsHf z%)mp#8F*;;y|Sk(XJDwdGjPME8r#1P094=p!Nq0<9vYp2Zx^D&MZCzKA?>$~O^8(LOM${idTG{c8j@hJqTs@>l5K;9{j9sAgww zuH{Sssyh=dHi8#Y43fzp9q_H8sV< z#YTLusQBxS_*;(n6SR@I?ux%@s*Lzd4A@lb8Wx7HGWtEPCk*of~I6@S$c zj}A71I)#Edg@SHY3W91YsA)@$R2T<9^;Ce1ji63ZL023>Hym#^(|+wr2%NQ(MJ1p| z9C}{OICk5AIV9ob!v4`H0eHEvdwchwUC|Dcn@u;E*U_#z3Q~RM4Q`- zZd);(c`woGcOfLvFap{!;zUh;$|3JXdl>=v*k(B$&j=9d6#?Dr z*^piLdNQnM@?}IWK!jWX-jv&4>_9;GioV56;$An=#_dGdEswT}vZVix^l4bCO70&IWBHgN_-4v~Ki=(f#+V z`|nxzqe|U>&$@Tk(!IN;?!QOfe;?6(#L+GOX5Guk-$z{C$HUlgTRRaQcH5%aD!?m< z_fSZ(!A=$S?==a)hhg^bH3`7ex&3=h0`MKI{d-LU@MLH2U=h&4I2!6^#S)WCho_P#NFd0E?SKyvTU0!dtBXUwQ75iY}{?y zBES*rMT;K-+W+VeWBkwA_@DEiXl(E*9;E-Ajc=`G{H>bC{~V3~dBpggj`4eqZlmRW z*7838`Hha1THa?Z&D&}OVh#Y+1Mxm;c|W3Mhoc4kjXs1|#PC+z?u>W1fe?R3+HMa6 zu|<;zz$Ms0`Q7v|85NeItzBO=YPbW{}IpK zT`E2QBli5sTAn{$)AK(<&;Kam`4-3XJ6X#QS<4UkyQjNXYWX2+Y1&>R3&sIZJqvz_ zT7DSOvenUo!LF7CTf@k5ob1sV0@}F}2u_bCnzptW65V!N$h+CbpN8~iw`q&hbR#`) z+~l?uxlG%Gl4PUX7FC&1_O_xX=)_Nw&%_u2oeYWaUnP5*x%{r~-l|JOVIN3+#T zW*5ozu5L72wY5jW>)f_zwrV?qX2`XsEdr5yDFk$^HiT_?nzbZrLLfpSz*QUF?$oUS ze45ms_E`w%Zqan5-QEYTuGK99ngo4EKGkuG^x8Cf)vq)9r8gwyAGry8R8OTWhU!yHzvY{sziKKb^4Ae9%N#A@?;SQ*eit7GEOT9mUd3Q?)62AnTI#l4 z5yqF#XTOECOWdZ*ou>9d&0@DLszM1&YuhdgN%#@}IlAFOK&JxARV4Q8c%j=N>czH^ zd-DZuThxnfc|3jtZJzHo6}jAWHo0=1+ZI(R+EFxU=bE+%@EEeE5(4lTvZoRVXlI}u zCKvyTbMdeEkKhif%*DUrTx_kCi??d#;$LAd{#7IwXFIuwjzWWYr@xh!n%S;b&{5Ue zHiuqe`^w z+m50akjJ}iQ5N!Y+mY0EoZA*^XIRhEZHd?;NTE54xv5u(J@;%nlu&aia4ggfw@;%h@y@-}Ejuv#Q(efSE@*Upi zkFM169oDj{mXW?%vMJ1v&c6C45VSf(BoRC@T03FWZ)R_BNii0BkR z>y-M490YW)=vZ z^9jHy@QA))0Ri3Z@Q|ImJ!@ZzbRB>`QO>d_3xnLYVmkBMwgbZs*_?xPKOq2{vxLk+ zK&uuDBkTieu?xWLi|wnAuJ*wpNrb(B2yi@S(pNGdpe3SDsyVBlL$Z)a1mK0UD+L+> z%`VPjiFP7sbj=adBW?SJN;nkuZwU&(_p8rHA_7{8_$0#KCuHZH1Iu>!D?OJo!R zT1mH%gxPn}yw@oNczCX&c^CmrBBnFREF$R`l3cM~r1eeec}1LDae*dBmndJ^?;7f0 z$aPz>oP`XaDYAVigr~Cm)S&=8@7uq(E5KbVeMAM7?pA!o-F7598(fKEI=9_O3+nT{ zV-8xVOrK7>k7sUM(aXHIY4`Eev_%s3wf)vU1hgTjS9SQ9&V!$XB@5S_W@(QmV+))a_n#6Ja+9zmzY`QTRf|Li)WP?m9xsXcve|eYgSoPb5{8l z&MM!E%qkD_(KJNQ8ZF;sE#Ks$^5#k{-()Q(YiT)MQ_D9|%Qquh9ynUW-JBQpki%RyLNdY)7=0>mJ$;qD3%N4wfvG0f=pyU52?3T6L9J=Rfe`uMbj0peTDjp+g9{4uWfrd z?2vt8cTEEDe*6IKnh?+yq9N6kTyog0#|7Y12m6!(0nLu4S7W~zvh$h7O4?2d!11)- zJ}&_8v&g$xuX{x&R_pbG)9Y~xy#Q>}J@S|U{2Qp$tJJfa^E4GBpyfz7A^|)f%HdwE zClSyj;`Ip0xmqLwXz5t5$;G!3^0Og1bJ#CKMnIE@OCls^9Fpa911kUz(P=cg1mJDI zJ;M=zGkXo~yAaTN#Jdqar$as5?L7)A0-8je7$G?olCTnc&o2OHhW(B@1T?$2m)R%N ztadVF=arrmO8o`9PwqhW3&4}wNg7WA+_P5r+F3Z_NbLk1UtvHd}ULc9b zL+Ly#*`!85cPPG!kQ@t1czU=;<_N%hBl{9J0RHQC>%F_;E6n!j@2JA-wFvPLi?~if zIOc)ea+xn94*>-EW6>-FiH>-8(R zUcVAquMaxwHJYt#oJTeu41>Z+`J6&701qg8=YW9rHabeN_oT=kaJ_<#Qf(L0{IK6` zdngpg-rh}Z_k{rOjqMvL0`QROPuY%uRxK7XyIpGcx=PRxrTQ|tX^-0$4N-0FpNQJ+ zwiUgs!T#0kU2a>HrFiWsy3=iovQ*m#w3O{IZ4uyjccmL`0eIKClx86T^xY=bYmcIV z)q36T^lA?e1mK;4JvImp=(O1>F-R5*_@3{rgHetOcN9R@}Zwbj+y8Qz} z0DJ0=jufCL}2xt;9oJkrf)i;JDtfUgqBw{&}oTbyP4TeOw6$`m- z3&nT6+g1#Tv|Sg9f;7isS^f zUF~)#hOju>c2(FR&op*3Apn~`jpk?sv=X#XDY-%BtPI(C1+WiL1mG25DXm2aXm;@l zYrjYNx5Bj(O;<`Tlal43l89pw&>f2D+@U2|7LxFsVV^`Gph?6+Ci#?qwg)If7P)OvmeO#Nco&+s2(Zy3Xbus8_u;4M@hSq^8)%4P z-$P0kxJu9v)%FM}nI9_Qdy{*#+904iL<<$kHX@l9lCbJIluZa|60wl|w}w>DbycGw zs>8$7;T*TE7{a_;hZHuy$hPBjomfIF(skSSq?M$~V%2I6`X|ppXjzlS*qC9J9R$4l&w#{IQo13>lIyoCOEDdLaejz7?BrF%WC zfx}(hXtrv5huRJcjpt3vT$o zf*G_~L_muZL%7546!q?Ihp1O|_?W!W&25W%Rog{mbXT{n=w%H(NkbR6Ey_~7M<}eF z-L@!8wS7!E*U4>*vQ*n8q_?Bn7GK7?l1>omY z59sNb06Zw|owWcwWiF?|BLE+?G}BCtfF2O&lWIaAIYJ+hPyr%B5zy@D#cJ#iLw24} z?5_bLph?iZiewkX^MUJkF`dWO85%G5-L_%~x7|v-_d;=O;XK+~2*9Ceq7^^@HfJmi z5dkx=k3dlKf@*UU7 z=y%n&g`9lbZ7V)uJ8j!rp`AQv*3j5MK#LPYm}DDuc+>3=^(t|;ZEM&eXTSo=00eZ0 zXow=YK;F3Fb|{9hxa}nFdaVuxh$IjKS_xXHlnkNZUJKdz3mT77w+LtwF`Xqg(zw4G zl5i00epCS77!tYE^K)(cyRicB7Tx~#qX40?o!GIRhphLO|hOAopID10y;#fR&lh@^YYUn2|tqFM*B_(Ihq6up-Ko^`aI!Cn3BYPD(>W;uS`E5Zso7@V8F1uGCm);) z0cNkfLbFRWIH2|w;}eENw-q09+nc2NxYJ=Jph?7ZCb4fM9CJGqLn3XDh8^+^>;k&= zLO^$jhA98p4v&N+I%-(2ImEz-ki#Jd&o`8ASg$$IzG^xSIUK_&zyk17y*u=qrwC|v z@dztiNE6?|kex@$F&b_NXcF-VliZ*H9|%b}qbdPSA}(Q)5p>tG-yxY!I~fEtiI~nL z7b&;*84}%AEabNQI)K|&4B@stDRK5V;wk~%p%}s>>&e%W%^cc2Z@PPeVNgxij%z2OeGt?1>p`{?#)d)Ofx zSP5uDP_Hti4{eaPg(U3P7Rm+znEf_|69LVR7FJ_#3E3kg2xt;CU6DMc&~J5pBBrxX z8ptPG+_qu}w_Qpx-0Zd$z1+4V@osY4qAaDi6SdvwwnbU0ZGURJ!EKAORNH5iN9*0T zC`+~NPQ2^fwkS)r?LlqVx@}RGYHQz#UE{VzS*mS6O0CtVEdm_;N@M6D(*ahI&*%wm<;MvpuC20h7w-S$t z8 zzH260Zk}kp$WOFh+s*eLrA4G3r@^Fk6hJAA`1}E7DzU544x4C z3}bGS4E;1MPiEER`fEj zoqQ*|ZBdruwQVPvwg_;9?FpO!oTC#c!U$+@pdr=RKMmPAO=b{@0L(r!3h;R;I7TsH|+X+s$XDO2r&?MsCNbtsoB)o;` zO&K5nOYB8?i-2Yq_eR*qIqWxx1OZJVK8cWw4M})jqbUdLHHWw)!qHTNL+aT|JFO$2 z$;BfP@-ZPfXBP>?dd(pYiExZ|IBfnSph?6b5t303$rMU+1T=|Q9w8a&knAB61T=|Q z9w8YKlJF+W{{D>s5r-q7*~KRj_Tdh@J-tLglZa0uB*Pq%78-*HXcF;Bgk)$f5&?K( zv9mh@nq7PnVILB*^CGs4_Ff2R67fldWN=8r>&$T)g9vC6@kxYaP)NdIqZF5VJ_uY+ z$%23;7nelH2Rh{AX(B^FlZZx^eyJhyNWE29LMBK|H_S<-Rha@}=H_`rA0A7aeYc~S$HkYzm>N(pl z(xQlfmLqOwIR|MP=@rUh$K0V^fdHIjEwq_KK(mW?nce<&dQZcy+lmvp?Nst^k5CDF zcLsSE0Zk&NGs%6LExLy!Y|j2Dz~-DF2gntCS7*Ni4FTP)_=LN)zsT7w?3TB5XDIIk zh@1rpz%#G?j&%Waw56Wi^PJ)#0Gn*jh!D^QU_8_=%^J|vF`$8RNq~p}0z?cDAfg8W ztw&>0a_M?jN^vmzv&9Fhw(yCa}U#KH(k$B=|gx8nc- zO@bDxP*~GDI3)J{R|GVPn9fSKJ`8HDbr^K3=3!7B9tPD#4uhU$qgupK5#7(SeEY^T=@wer1mOE{yCV~T z2i-v0AP5l2Ed;c0#ODz`PaQq0h+P2gmS%RW*W?%rl`b|&PeSsDO#-m=o0QQAX!hqJ zNksbN5MYHHY1R>d*_&y?KtQvL6C>=89QI*!mMH*l*sjqcfq-TgCo=mMQt~il=k<}i zi}jjAd=+xgqWr*c=(b`xx3%A*a^Gz$c5vI-WXCJiz+W z(I5g^wOGjP3rNY;key9$rBp{ilc0sloWV3XUI|HfMBk!WN&p^5cK45fW=EeW_L(H{ za>&lF*xPS5LO_#|JqYui0q2=Wl@S3I5iRI|uKu=1qpNhP~az4 z^SB>`f`pTSP~eYUU-M`a3jF!lXdWm~AS)Jil1%8v?1YAb1U;c3K~E@1(1QZ02lX!W z?BL_rhDeVe<#+QC8VdXn>b_tL1=2fODw+57k@o~AD3DG<_4(ULm~<6nn;|#{C$e&7buVcLn7YEVeUq&bwKD9wQ-IUS*2O%73=^WHdaRBe7{}A z4Tb_KE+pc`9b<7TbU^480o{j&mTMMz1v`^9EQ`eXYX@f%3;fieDW?;*@SXSo@3<(C z>4emMI_@(?OSME~C?tNtsV~tiG8AlA-mw=)>G-qDX;bBWH1!uccH67J!lz}MkwJ`+WObRH6y&Syr>{l#!M z6-9yc5%QKkW<);xVaPq0g97OTd5cP2#FLhfBPafD>9JUWbP^KxLgw;*Gd*(R`|M|Y zR6v1r5;FHr+H(J%W}QejWa!z?`9LvMvmp`BZe(_fW`kbOE@yVKW+6+|oI9hLq*=%k ztb5jGqGeIwyY4QoWfaJyp;tI*!`*9w4hX%1y&sP>f22{!73dajqw$e8-*_u{NJW9v z7V_{F=blX;rvpN+2!Ywy$MT;FET>fdZK{^a>|+ zc|P1Ha^kCh2dhSbbb>^LlYKm6=&e;F5y1}N4%RCQ=vBLW4=9iULx#SZc5qYmjGXxH z>mETyfpiiw^iH<2lOB;1KN54>V!iZ%1VuK7*++Msc}UR5o5q=U(`?AYv+m|?SIq{w z32c`r4?k-Ar!lk>=oN_^*jxP`yJ>YkbnTllAD-7i?7KsFm{5gB-)pYZ+Fjyl^=i=O4)&_Sc9E8&*W_EEj~ zi|kj{}_5`k?a=f1WVaBZVN`as@Ee6)#t z__N5_Jd&V5`Un*wlWZON@aF*ac+5wE^bzuwK3YXS{5Ud?2QCyyA0cn=V-QdBT3RQP z4f%O?7;9^x*^q~4-G@q=Yc}XjV4FpW_@2I!8wCYY9Q2AtnZYA))5wYM9j*BQhyv*Z ziAX1$VUx&-zx?IyL!&@CL59M~2Hv9^M^1bPcHdDG3Vi3A&v~LidJlEtE2}>@SR?CQ zvLSWP-j0cHLyd+6J<7k8*+8Q~tw(wC_Mt|hNmRvP7F%Dl5GB}w%+}K^R0*~?6056G zs1oQ%uG?=5sQQ8`!OrE>zR@gH3HC}zIImHt66h?>?3`wyO0YYa{aUk7CDR z?@+F}Ls25W=Eif)g#thO7{L>SP~cOa&(k#&$kdUckT{bMFrR6O$WX9T`8aY=v&c}e zbGS+mXg28esUK%{zh)sz_;o*Hu}`y*CD<11cduq4OR!_P&h}^)vIIMc{eG%h$P#Ql zo+9qnEMy7ReNJhYW+5vXyVJ5L@ZH5ddXECx_|Pky%;m=4p#ws%WV!J_(Jb@|b^uq< zcFhL8zVY2x&~H1^%HQ-chh#CWt(Q5a)brz|P%2munWX1UrN$fy*=tS%TfqewS(%vIOf6ol7(eS%P)X z;4IcGWC`{&*04ykkR@1mfpno}Axp4dv)=`pg)G4?<$g9_vydfN_k|zxGz(dRb$4gy zS{4QVC_R{u(xJd#h}+DUjY5H+v~}cDyHMaysM_;B9SZzxZf@ch{=9ootU!(~sH|ie z%!xAadOENZ6i6qiMd9QK56-i7m7x{|>wg$Qvmrx2?5yD?o~hZO*RvmT#m>+yWC;x| zcqsW;v$O0f!OmiKx@I9uu%B~1Ow%l633eF!ovK;L66`qcO;a=rS%U4(Q;*4-g)G7D z=b?I%W+6+kXZeEAM9o5$VBPO~PtYu633fB*_K{{GOR(-b{dmnnmS7jN-*K9SEWz&M zb{nf%$P#Q%ZnrU(MM1(8EEM=lSUY$sfdaWFArawyH4jvywGt#E*y*f#lxC5LU}v-H zk(v#9{hri?{f^KqWC_2E*za)7LY81RbI%;6S;!J>KW2wo76pDZ9KhWW1+p@sS2&r+ zl{rKQgkHh+^i0e!Q&^I#OnfJj6*ImQ78>3|^-U&}K%;6TknukgEu*#VjjdcEJl zTyy<33t7VNdDh!cvydg&fjkZEt69hr>{<5PN3)P6*!9fz)+}TRwm#QGFU>-hU^lVf zo|=U$!Or8{dT1811UrWJ=I)k7f$u0^uoD!>jsm^HNj)BTyXk$%%^&_Y6jKJ$7!XtmcY@(_McaE-RpY|!id&f!*Vt69hre&@2^HkL(!zl_tK z=P@XdNkgx2(uA9^wGIfqf*r;CLo3ZfuV6cIi?q~i(Cd@l$d%bbvydhHI<~oGQQ%wJ zU6@3HOd5KHlP#QdGaV3m1v{U+XH(5WuV5Q+4{D;>pw}lok6W*?W+6-X{faBIk!B%F zuzk6a8fq4@1iOH9YoJ-k5^P7-@S$Z<;7>=U^WcI4S@F;-oGjp^>+68fE7+dgi|c6? zdIh_X6RE4&px0OYB4)qMPl-sqkR|-KU=81B7P17pnEjr&EDHSYc#8LF6v%+kE1V4B zT0W-(La$)waKNuM3%!Er+k)qmCo~ILf}P76j%yaO1lxp1 zlw+EOEWr-v`u$w9kR{kXT)#&(3t564!+wuw7P17piFfeBnuRREF5`APq*=%U>z}LT zD*H^MP?ZEdXi=>9Qv~;!C=|%u5Neb7I1u^prw;RZJV1f;fh>d%cOPzlA zj}z__`=P*JrEwp?M}drr3P~1qpNi^ELP9}8*ihgv-?)zlgn|S;p}?P}F60wt6iA_= zIed${`)_-#P|1dB_v|e0DSI>rE|~1$2)%;s z%j{;&La$)QGP_B$L9ef%&CG7pEMy741DV~RS;!LXVy?{fnuRREy5G8Ar&-7n>=D+m zRp|*_q#JZ6i6S)Lijk$N3xY# zGtv<3Uf$DJXg1{GtHQC%BXNGHcY`hpq_~iXce0)5(aUr|=oRAJhwYbY7J3DHkP}&= z*`U{J=*QE7#hQgI;dcPvR9K{0$P%nOIa{b%$P#Q1zH6{Rvydg&jeI3#zGfjyu={xs zoM%}S_%VGI&#^**Kiaz^cqs7wwhc=}foxV}m@MqvC~Sgv6i6pXJ(-g^**OUXUSd0U z??u-PDoc3p&!fp~&4$!{4{_i1JS$S-k6rEnhys~TNIk*H%78L4uk^fJ*4iv z59dxiL#KlbMe1$2nm*QS$k6+BcVnkVs{LNRl_&q9!1pZobugj8w|$@37BZcXx)0ly z2d!yQSl`4;d9V%z{_5LS9!XFjy`xS<1-M(bQ*{NPP6X?2txnNw$k3}E&O6m)%?7=m zbx%W2(kx^N4RiRgGf}gUm5iNWSrqsNbHj2d@ZD@b?+GZ7HHSomclTq#A892>M6kWN zn~jf@_=_j5hlB#(xb?Y*pg<}?>cV?NuIq7HNyyOG^f(H==FZx zFP4thEMy74jvZxL6!`sR3MYaBnKbk!b22h=;=jA^7?1NPkWP?@aMFh}9HCVs5y8&q zVP$v}(0^cQA0J>*AOnUBeKob^?mJ8e42dMLL!*Fx^k~F8J_=+&BqCJzNR(TD>(J^!m!|#eREf7P5rj2JE+|W+6+k z4VmqsS;!J>BWAm67P17}p4o1ig)G4~X11$lAxp4LnC+rj$P(-}J|K41EMy6`KOb&8 zX%?~sJC56;qh=ustpENB_X(p8nuRRE@*vS(qfjMK_j&GinuRLCy0eD1nuRLCx(Qbs z%|e!7{jVEq7P18EF8j37EMy7ReWg=N%|e!7-KVHqSQZ8HflaKJyLamd^-c5+WBI%f1u_xn6#?Chv7rtKy@GWYv>Ip@dIj5!kANR)Ht6+{fSlnqH5PAhWh}o|+ z3%!Ev&+M0)4SJ(a#goianuRRkcQZHANzFo*VBIqaUuYJx1iOSaoX{*}3HBsU`;S`| z1%9|%z=zRL;9F!mt44ur5hNnKAL3>`rj;NO!5(M7pKBJ02zEXXrbjg!^!gTYmoATJ z7P3Ur?pSG0cquHR>cg#M_eyUl>5`Nn;yIZr6 zC0KV+Z~LV#5v{5PFlbpF{!ubpkhSLV*m3M1&JJ z9orr`@h7VDcwY|%zHfXUE0Eqp>b_le@u;&c^6qc`eZmuG6i6pXT}a%7R`o4JnO1yvu1-{&$`*=Ce1>Y(7TyW5;tlVvXZeIEQIx=oL;@ zaEq+h0ijp0+qi<(X%>0~+merQYc(76`U-OFnkW%}sdqW|eiX<=pjQNRU;eXN2ZUb1 zZsK0CO0&=_SU2rmso9{{C*m%$tsD5O)_~krs_S1nWLXwNSGm z4=>JrzvTkW2ECp=#YxXsiLB-B84CQU9os-Ah&+U7=X_q|T>8Lz=>z!*AMSp^T%8Q^ z6YLi5F>^E<((sAzWp=h^gIv$n=P_rNW+6*xaJK?xY8J8tyP0>@8JdMG!Maa+eXLo? z66`LXr%tyl3gl@#*2{W_T;ao=bWYRpAXl*N$TwB9kSo}pyn{^9Y>?~g{Tv@KCTkY5 zgx^U#)=$zbWC^w@k3thQ3t56~$^CtTWl`XVGIy&k6!;0kVfG#he6K&q>7YPXKQa_y z_i;KOX^F^CuuHicjn^zP6zmE<$rz{Epx0Obc|OdI)huKQzpZ$<7^7Lp5^Q5;M{5?c z1iPB+VU%VeORxvI9!6Rg1-_})aEpWj-y+<$ST7p{c?jpbdAt~*g&+^XZf7CGHH$n1 zdz=#-7K!%r1J}P$AjO3|d`r0xYY)`{p;rWS?2ss+pIN!l2n8}A5=rJ{aO6aeFjy~r zgao~4cf)s(7L7baybU~R4AgAM!}~qRm#PM6HpumC1McGeH49ln!)QJV^wTV433dpN zzrdxq<`vt}Vn zuwSu;PMU=*!Jg%Tx}#yWC^wfYiO@o$P%o(jo41JkR{lyJgB$TEMy6G z0kds13t56~!fb2JLRK=im1R-jPyBhD$9lQHK(6rdIgj%#bv(!wY)h`o7Mg`z!LH_m ze{;)0+Gfo{m0(-((RY(( zAuAcX(XuG;&nLF#fGCjN8hVpC*`S=XVke>Cv^yv*;{z@Vq<3T}yf5S0U$4_ahJtl> zR@P}YWaxL?d(l#BH5%l4bUsU1qgkjD68bW`TC-3k*i}4OtjF;lZauV**$fHFg~kR>!M z@Z|**NIj?%p~u~>8LukBfJ-dS0 zF`9)epYhGz(dR-M~%NSF?~M*mmr6|PLY82MFxy45kR{lm z%y!l+WC?Z{vz;^xS%RI)Y)8#PmSEj^dk4)zmSEkpvF$YrS%N*uvzB(6g{)+3Tg#%r zA1N1cBB8)vigPD8D3HAki3sn@c(-X2dG}5}<1P>i{IPXXVu3$|wvM@yVNoX{YzH1) zTI&ixod|XU=h-Tf==Z92JnW!A1`KuLdvqgK-BJe(8G5!Mvn@0Wy+T|=o;NhtY|!id zy02|%rdh}mevff(O*IQyf?dg#*+jFDC0O@i%*L99EWxhk25n?n6!@Muo96SQ}^-847k6SLuhE4SIc39b>=sH49n7 z?;iGBPqUCE*zVk2>S`9U1lxwMlzy9+s$bR_WC^wlx5zh|g)G4~!yz3IdIkFxpTmBpS?Cq4`-SO)nhkn=r#j7k z4`>#$gx{e&RP5I*WC_+?^4OhrL#77ODi>iLb1$(kxU7*8TY1O3gx+VBHaTg=Qg3umicumTMNW1iPOHz-5|+ zEWu9TGq|Oig)G5t=WD1-Gz(dRErJ_Pi!};W0{zn6xX>(A3D*7a+d|Dkm0<6IgasOf zDuKE)&H0*LWLF7xJGz(dRozE>XTeFZQ*rhy?ouygG z60G}$tC^aGEWx_D&44BH z*tJ|iQ=@?XxZ@t04h8;N`Y|3uQ6QBdL*acBAH1e$B_TsU*16kflQoM(gkLv%nxxsF z*ZXxhq9$q%69uv|p;tJW#^T25 zfY2-0uQ}jY%|fqW-Lz_qW`kZ|nXB3FXw5>F@Vk-qj?ye-33er)yN%Q=WC`{NYZ#$f z$P(-^)-YVNkR{kAe7-PDvydfN_fXSN%|e!7*RkFqnuRREx}OIctXaqs>@MET2Wb|v z1lx<*ftrOZ!H!^dfMy{}u=APiuUW_v>_TSyX%?~syO{TjzM6$B!7gLAk7glDuL$v66|?C*=eg;$P(;s*3d??kR{l<+-|Kk3t58Q$$ndD7P17}nonR_Y8J8t z+n4)U3(Z27VC%Eq=9Wc)KkyCX(-joR10VDzbJ9#X8O2UeAe|r);bafjMpLaCi3s*X zR^3FiArXH>Y`|<|%?7=m-NzalX%@1ChKB67p=KdVu#K2)pjpTgY-46W)GTBP_7Ll> zuUW_v>?l4*tEXAW60G}9v$~pvEU^BwbnX=R+YIVKvW+22u-qNK(I`|2bSBSC&ubQ{ z1nZ{4=QImdf}PF>jjuHeS%UqVPkPR37P168nEjs7EMy7xFdvLgYZkHu+mW02E6qZd zU^_7TrDh>Zu3%!CJ#_@EkQ_Vt_U}rPCTeFZQSa zGSg!5foaci{@_DEWv)x9cZa$Axp4R z`FvrCW+6+kLwV;|tXaqs>~z+!NVAY7*cr?&)GTBP_5iaBGz(dRt;=;bU$c-U*!s-Q z(=22Ob|<&-T+Kq3U>ov&F~_nf@H^534u}G|!$7Za(u3EPU z*v_2uRLw%JVEZvUMYBP!&!aup&t%O)mhd~5*-4s(EWyrTcA{n>OR(-6;wESovIIMi z_ppyN3t56~z@2!!Wl`XdxvjW@P#_x#dWDmjoX9vG5PAjMmIIE}Ec6Pt8#n$K%?7=` zk+!hk(VB%U;kPmS9i>^w66`td@FO(~S%P(60Wm_ekR{k{tarF(Axp4}Snn{+LY82= zv)-Yag)G4i;dUFMS;!LXJZ1-L7P7$l8&<8k0}j$GWC@m28>msJN`elsDAxOd*nKh# z1+uF`t?<#BuetQs@t{_)?qWkf%|dQ6wyzS`hUcy*km5ojzTRBGKFUcezB~~M{9f9b zs}BXzJ2FfrvA6O*h3f8T#LphD$6fBQ-YZz0cPwF$)jsoc<)UJ2(Q&dcM>qDZU zN<2E5wRO{M$iuTY!);fM2DKjLi^N?t3Q?kYr?7&~nuRFAwq>@HW}!;3?iy4_%c8(v zD{|iehyqzX&?}s{CoenbfY2-0DcmvIYZiJ1>z;UMr`e#_*JW=`x~*m*OZat9CbZEk zWC_-Na<{c+Axp6Cw;)?t76pDE2g-`<=^Q_Tjw-mkm8-bAyICHzj~TbYeD3t58g$iqP+ z%|cc(wxMNF;2UorCxQZ5nb0eoe8DZx7k{l;$P#QzK1!a|EMy6`A?rP(S;!Ks`|QVQ%c8)KyUlnyhXUD1&?}sL z%ZYrY146H0+i}nQQnS!2*srZ>7P168mi->kEMy7x1n)2VH49mSZOFUDKFvaw zUMsLg5AZfwp_DNCD?;pWy>@RS%NJLxl1()RRS#vbcseG zN}z6%w^*|fCD<{X*&@wCm0;Z$Pc76eWC?aBkFyIj3t577zkD)Zvydg&)~t7)W+6+k z?tzK9nuV-n>>SIYz#lMraUv*?V>0v#C+<=H**YNf3ib@|v9mM_y@FlId&^AC2EBfq z-Nft+%|e#&yOi0FH49mS-Nd`gbj?DRVBHruPSY%83HBqNXin8EWJ#7sf+?DXEWwWB zjxt%ZkR{l@%udoQWC?aKvlBH7S%TfgZ9GAL*v0I3jAkKAu>H92jMgk<33eU(9i>^w66{K@-;tVyEWx^mZANGo zvIM({=PJWB3t577pCTEiS;!LXXwGe@W+6+kjd;HpqFKlith>lQShJ8NSodJgAk9LS zU{~^?Y@lW#OR%e%9iUmr66`=e0qbvB6!;0SyNZYcIoLw4aN@3f_0s{NSFrA5xqUSY zy@Kt-;`(Sd==FoGdtRfrW+6-Xox&B=OS6zA*j3E-)GTBPwmF~j_0TM23APIlpxreK zS%P&Bv~<%fWC_+?DekIS$P%o((APz?kR@1m8^5z=Axp6Jc<}0^S;!LXRBq#rnuRRE zcIQ*n4w{85!H#0T?KKNof<4A(IqftHS%MwI16^CqLY831vxYXBg)G4?<5Tt4nuRRE z&gTKCm1ZGJuwz(5OU*)-VBP0)T4)xs1lx!8HrFg<33dy&TQkiINg)G6gVhxQo3t57l#Rr{6mPJ9rpojuFC_=ArGMoc8)B&Mau#31s8)z1K1-pj_ zjt?~(^!h<@5;uN*%|e#&`z2osuBTbZ66|T-Vd`oYvIKjG{eGKHl_~2CvIM(=bNfcK zkR{l5oZETLLY826b4NL+S;!J>V`jhBEMy6GAG2pQ3t58Qz@6%hW+6+klR3B3nuRRE zcIJNem1ZGJusvAAmzG6=@7R5~1EN4S67&iu(^=dp9T0j2`!Ta8H4D9hoz3hQnhkn= z$DYIN3C%*5@H>~;H`Z`avydg&$y{d#Gz(dR?Z>D4`!x$$f?dx2 zcb{evyMS zAxp3axZQSG76tw~3lHvCFUNx25$gBD1+4889S?Fvyaqgz-mY256>L-XyG^q}uGesc zbKa_1$P#|PVRnmVAxp5WxgIua7P17pm;2Nv%|e!7*K=+gH49mS?a3N8Xcn>rJDAz^ znuRRE4rg|qW+6+kBbZ&QS;!LXNM_e)7P168mf6*sg)G6g;l8>`vydg&bv#2~saePp z>|pl0LbH%1*oA!QWVvP`ORyt({9LA4$P(;f?hi{f3t564&3>0?7P168hI3o2S;!LX zSY{V#7P168j@gBpg)G7L;|bCN%|e!72QWKdvydg&fy~a+EM$T8_wc!!&DAJW3A72P zHb=8iC0KX+Xtrfh;D^r{JmE!w96upfI9b4bYL*TNy@K7z>`cugxl48m3q2np-QlKK*BVQLX|-81Uglt5GBx`1D&E#XcDNq=P+5b5GB|x zT(^@n3sr)h$8))fnuRREy5CxwpjpTgtovrek2DKef<3@`$7>d{1nWLuIZm^XC0O?? z-dN2-mSE>_XBeYd$P%o(e@EWsY<+P;t$P%o3u&*mKEMy6G4zry#3t57l#B3+cLY81V@xIwnvydfN_j&vdnuRREF6FjpuUW_v>`Z3c zX%?~syO(oot69hr>@Lo&jb z6U~M^e6#<=4nAg%|e!7CotPUvydg&p{(~q%|e!7zu{>? zea%9aV2^WGtEXAW66|5tTUWD?C0O?zmEWdO2a;_JS%PiHe!tNyWC^w-v*$GnS%P)n zeR)o^kR{l$T)$sy7P17}o;94+EMy6`1G8r|3t577k1CzkEMy7R-5mZ(vydg&?L72+ zsaePp>}{CcpVBB)33NQ~#wRrkRf2WDk@tmWp-QmsfteGUg)G4?=F#uCW+6+kr+BwG zW?2+`>3(nU91r6tkbM<;g_G?(7JRM)La$)m$90Zs7J3Cci2L6W%?7=GJaSLG9M&vk z3BRYg?GI@dvIP4T_tnod3t56)%$0divydg&<=jXIGz(dRb*Df3H49mSbst6Ar&-7n ztUIOIt69hr>K61nuRRE?&r$_TQmz(JW*Mc0XT^U#(fl671JpzpFG0S%P)<-dAcCvIN_RPr+7b7P168nETFh z%|e!7&$HfTnuRREc3=%lH49mSZN=;o%|e!7_pye>nuRREF5pYni!=*af*r>5!G)TI zEWwWEW7Y!ALY82^;r(L1W+6+k?u$m}X%?~syOY~(u4W-iu-~$VIhutm!FK2UZMJ41 zOR!(@{x(aqkR{m9dB2#cS;!Ks`%=;wnuRREo?#6iYZkHu`!%!EH49mSJvG?js#(YqY&&MBXcn>rTc6p_?h~EWvhUcD!aGOR( zs8O1QD#5xb?MTf+mSEjO+#@s#S%RI-O*~w)kR{j^%ns8mWC?Z|vqLotS%P&}-iK%w zvIOhCKyI*RAxp43x!ne77P17pmh}$QEMy6GGG8qlpjpUD#`d=?3gkB@VZA)1f?VOl zeUoB89S?E^>!!MWH4C|d{e*|EKAH`3{Vs5r>$10IAxrpmQ_^0Vg)G4i=0i?T%|e!7 zJMfkM9-4(L!Mb1V>8@GG60CbxwVP%kOR%4EJ#^J9WC^w(A9}lJ7P17pl!v*_nuRRE ze$5&>X%?~sJCWEXXq`hV#ORy`spS9C0WC?Z)kDqNd3t56~ z!gbb0vydg&1+2HVW+6+kyLmd;O0$q9ST}WSsaePp?5FIvg=Qg3u>H9$nrjxa1Urz~ zW}1a8!MeK*O*IQyf*s5IW)saqmS9ga+gP)ZCD{5rem2r9WC`{lw_8KaLY83na(`%` zS;!Ks`)0KdH49mSJ<5HuzGfjyu%}o6BrLfyTgK&wW8KLJ=xfmVzFuLbzJ0<9PUUIXwo1zIrzbkqB) zinMA3>Av-$iXyEXLAtBIl@)3A2y#3(PbEcKJ%V)8%Pd7&J%V%}G^0Sdas!hu zDAMW?q&q$Rog%FsLAvAJ^NO^31nI_^=M-u62+}>K^{gVT9*}3b&BVT3Fd2FTK4eEhB2>gll6y7 zl?hutAh!CA+Um)$)%@5PzE_UnkE`u|8@AgN+wQdA?g)4AvH`pM=}V2Pw+ElKf0jMY zyGvQ+{E4vD?wIpq-Z_t2kE!i`lYF~Va=Txv?H&)??TN)p*zQraT`W7Qu;>7INXC0a zZTDD+7vJur-0oqu-LJ!T`(p7<$nAcmwtF;ecR2g)9#Y#q61Mw1wp~KKJgByNIBZAS zqIx+abv~fB`&HO3*74=;a=+T@p|DlFHmQc<(~vT1y9dK|yG-bupBnC++{x}!n>|3A zMf09z=EoNi_Zq5OUj%3t;~HwdkW4?_9-Bf zv?4j(4101(91_ZY#XIuPXj{1>-<0T$uK(mf#EprYU7e%NZb;mWF8Smjnw2efWV)o3 zgUzl_+>9<+J=%KKBR(x`_H8yM7887%cHf#FE&zg?$Wavs&E4H3QN1!Xc zIY{rS#LaHV(Po7cH@h)Mn_1aHe3I)f6e>mKc-flH{K{;^IcQ>)fyGwHE&jZGpk(f@~>9QDpxV9 zTGcnI*3PO~u4avY#|!M+idEk>)yuza-pHz6$5g9YtF|duwR**@>b0{fma9;uPW9Kz z)v8^-Carz7T6ua?t!niwza|#KF;!~6QH{VF6<#lQzo%mB$|PBLUy^mDldQWp$+}WW z*4^XR*<8v|g}hFASFBN+ta@!qjjEw$oi{0;c=;PutJkS@LE_D7-vyMfNtIS5t5($u zJF+#E7^zl;npH_s?MgLjcwwYiO0+9pEnh3E5~blvs7c=_SApJBU{kJq?RUym$tqv5 zYW2#|$_h0qW|gayRlP!1w5no_3U#QhYsdRR`D#@wS1(tyYUL_PR#(faRNGXlS^kZz zT3Pf*)vRh2?H?7r3e=s{8ee5APL^U#`HB^5X4R@?DrVI#U$vSi-VBwPu#GIBj#x3P zR_&_Q%h#@2qk6gcX4Px3R#UsCDOa0vs!Z)qZC;}$b@iI2N=;*HIjg3rVEZ_|v$aEA z-PRQM+-P?({Qj*v#JiqkP|Ib+yZgku(&F8{;$126?jHXxwjbFBBvITsZC8=ifHgJV ztX;WgjXH0d3N@`eAud4^!(`Y~8Rqz{Y1M^1tsuZ|J%{MTT z*sh7uq#XXD1}guCsZ-5LHg-UW_r9pU%T>s#CQ9%gk*bw{HLIGbSk08HUL#?AqE^m& z$L`q}nyA=rfFPWqd9zNfD(=s66{^*!<=WYVVJ>QW-5aPIm9Ox+DOamTrP@6By;?J? zJguNz{SB&$3g!5HwW_9e6=U4hS{Hx_e^lxaQ*r2vZgNps7{+-D+9X z4X6v&u3DS^RIXFLGL_Y;T9Gu=s8h2-mZ4p>YVCJJ zKQ`5A468|%U!e-scm*8Vd>=RE{#}JSU9C6E(}uMx(%Tv}E1GIqm8o(~x!BR@o^p0g z?eewkQ7s%RsNYw~s_=SNk|JMFk;-g?7^=ML)w62eRl)EPRH65=z@=h6M9yZMqkG~2&1$06+_LuY z&#B%+BT(WYF)B*P?jAq(VshZOXOC##gjL}X6?tSBhkOsC5_FA;4CIIyc?-JCZ0i6?1IAyb~#K4U7-@;Q@5 z%jZpHTK>*drR59ebz1)3yg|!9n6vrPOor){CymCc$tIJ|EBX$ z2`gbotD$Q>Sn1ba05_74!%v^4+Fjtzw<|=cwxyD>;eqydOKQ-5zpP3uXjpim( zgic>>p)_uzr0%5TiqV$8q^QrBduaKbxtEsDoBL?_J99rRUoa2S^7k|(24y{+2b1f~4nd@j-+|-~(?rPqq<$IuDN>qwgqF9OhoXERi{)D-mhb&3$Ty#;=_jI?PsU=t9*g;S z3St(ZaT4PX<(XlgrDbvRJS|@`HE4ZT)05hJhB@nM`sMU=8iBgg2${Nij`yxVL^;10 z%Q;@pFQg!6*{}Xg*_Jeaq2<%&FS*{Y@*Ccx|BvEjn13Mlzi9cCDUR4*xSDPe_36K* ztWV3lL+qzbiHndokEy&bzkB&GRVwZfq3&9Vy7|W1MXFipakb z+WoGJ-0mUv)8?1CUiUulFnT{_xbNSS>v?CocCSwLT-;P~bE`FO4|^>dYu?9Prn{M7 zs%74yjvwEDW+lheHxa!iEnhNEqSxj2nCkO>8ba^sC%8E0U9x{wQ~m3ii_*V3QO_!Bx}bk`qh(3c zj`s2lx(1X+XCd~yueF;aPca=bZ&kERN66OfiMK?2q zuGRe0^oquq`WV~nd2f%*yg2LS^VdF^*ORLFm8_i2B{JRhp%E0bfEk6zmr~4bW>^%n zDPs08!^{xMJ$|Om*UWMjDSo{y9`!O8wGg6?i=sAjQHPr`2~qvEy5!gCQNy!iH9S4aHPrBwgj|1k4bMZ=<+QxatVC3<;U!VjP{RuoqWroR5x~rXWSS-CRn` z;^qoO&B$-ZRj&)Kt`PK65igqPVooYl_pd zgt?QV+H2qK#I^X_D54#IWz_FK{uUBf|M=L$N1eMUdNH#H$DZ!FeC)~Rjy-(TDTVAy zC(ZlrT+h1<^4^WSyX11-ndSkCmSG;IWpVQ;ElZlmXc#JxP%=%=5G? zZhlX_IQcQ?nUuAsJR1BFc_tnWTDd$^cQnXn{)9YVj^&xG7yTuLdHw?tE7G!ryGQUU z)vvu*nRu_DT*{&raM$zR#!;^-#V%(4L!-lcrX-CH6HR4Okb8G23!1kmVi8jdako<3 z#pZKb_ApB+`T%o1ofY*rZ;*xzI!4sV&v$NLHm~Q;pyLRi5!A?kV-9Cm_U^WyHTyf; za*X29>EEQLWHiGW;QH2Nle#fVju`JEUgC(+AeTps{~&7Oh%qqbBZl06|BI-J`|qHX z@4v~~^?#yW!^}06?HV7G>m{w>)Snl~8vZb84d=#unV-1eSxr-lTEOrbbz|C3`H6-` z)UNi4hGC|mo7wpDpdWs|ml*Xj!{>V~qo|Et)ZwOiLew96zL)(xvNh#h%q+otYUo9s zPw`mQ7Okz%#!}a(^O^P(Bg1s2WpUFL@ve4fVH4bbJKl6m*%2wz#p{lEJrVC7s@bk) z6rFQ+Hy;}xJ-1@^j>Y`wBE{?*i#a{lV&*j~si&4V1KkmI1BweyUZ=~&#K z_8EJACeOW+U$01ZMD@>#G$&+~nT`{;10%Szju-X1 zC|Bw=KE83^nINKmobtYtX(myG4D&H9GtDeo7B_Q6%+Oz_rl`Nlww#Y>Jg&~nwed#A zTo{WvE7xMm{NWA$u_oom<@gx8MbvKSv#AfE=hjhiu}%`Aiu=3lG|KA%~Qai z3r&m0fV2Xs7y}Asdu{vTT-DA#{w|yi^~~k7q3q*doIA?-Ox^-^3@${=;^s11mW<9! z!f_^lfmB`B&OYCxsPB=vT;Bzf^tUT0>teZn?>@gNjGDd{(XOLqag#wgcQrTA^}(8^ z9WA??r`$Q|)dg~TkL<#qc1aes45B`Hk)!?`QH$kb)C_Y&0lvn%%AFD3R3MxYUQace ziZj9tbE}K8I!Tn9Qx@gt3Aq%Ddh4NC>iZNM0+QSb|@C@?G!}IYu={_bbXzc z^qGh#&i>fQ_rH{ke142=jz~?DMrxdrNU{c7=5h@to9pGnT(4v<9w#YBT!U?m*6Tvc zTTREPPUF3#T?%?hvK((mj(6r_j>-1;?v!aU^Z(=gzb7J($>mW!1xv7QXn3EgxrVKNf;$@hV?#%Qq%B(n^ z0vwvMYh4*;xQJ0YiZLpMF~%asnS?x^HsezkBafLtG43*3GW=EGO&Pb-J&YgMtHSrO za6VR=mi^5}SF7t|=VF_G)VWx) zTzgU6-2C)ghS`BU3Y$C$tyt9TqJENid<#$e1JzCu`C)C>E+Rbqs+w7Q~y7R&e^9AbbI+K~u-nW}m7p$+Ge;O^re3cNx zx5@41Y|3JsM~u9tU_u^um~SbDd~PiC+ISWhzROdLz7tJ}3RHS)SVGnMoD3{w>Ka~F+%_T1_&H~QU?vf4>Ev-0{8VMgB6YR~N%sXUv$ zooeXEw5PlidJ}5Ah$)8g`Fe9NYJQ2ilSbtqa!%zxXXLL0vG^dK58WYR*Bmq@A=I?&{LnJ7{v_l6?|v%4(uEL&SdKDN9Hll#PZ1-<@00a z^LIq~q?_lW<8tW&sXRWUo8L#Ce^)S-M@`>5|A>0=BlnXR=s)om{;qSJbn{ZA`%fv= zo!>l)zVlb~9lnafJt%)JjjI{%JDy*rh<`M%W6U_8@4NQ4?^9bD?z7M3Xe6>vK>stU zyT8*|bm7@XVw_~(&0LH|h*wE{hAEvezspM5{4Otj&$T_?@pDgYsV(A9NTzCzoo=c` zb@OfhRPJdR=99ei?1^~|^;DhoH!{_z=U#YBO|1P3eV2K<&I^3 z7Wrn1j%74|%;kA%ep3zm_1kK{{z)$F*XiaxS6lBCw6o&W-NmrS1K!)L-;(tGBiSF7 zzfTX!G{;d#@00f;?h5i%xmG*Rp?31pn8ABNYL1n546@_I2dJTk+<9ibl#N5_W` z*O|TFIU|bK{ddh-(oG|(5BrJI1}QzBS-rNe>`nQ(Q#04Qn_v|Cp}jlteA3r%-l%?4 z(=XCZi)eJ*o4@Ffyk{A%ndC98sHe$0qCf6>%&lVH7vIlQb>2&Nym5Y#TW2PTeJ*vg zzIIU?wxO$|Kkj@zv|-|SnyT}_409L8q7Gs#>XgEmsSUx=z0f;KqzD zDIGI1-L;VTaqIG2I&S&%njYBezx%8>ehm2TvzlJ$Df{uXPU?>9iTh)!`ggkNgZ3~i?0<;!U(Wd-lp_C`W*D_&F|$4PB({9MJ#}|R zGtCIZ;xpA7aXl`#&Q#OQC|6_uz??00$NVwydmVjWU@W*_jDBZ-LmP%a>twIg)cR+qjN9XXIRO(vS z*APdMPfVvSioC=AV{6FYKMvc9d>5MQ7+6=sedy$MF4s-yr@JL;OB)QM$JGL!MjE&w{6$EvVmGstvI&qu%Z zTZw+_w+j8z?`!n?yg7-pjrPim5$BVO6lZ4&;_QZxJ+!>X?5AZZ^BFDgHAiS!+8m?h zedY^VmibZblWD#rABD|nT3%z$(vp6!mF6r9O+#8PGK*>1&zz$Xp6_O5ns30T6|#H= zEj_;sd{H{?_A{A<0w4EZAjY%*`S4#T8~#$oPXd2=#OEwbTK=i8%<#Rf(p*gC( zM|~|yucPHPj=8>2UdLF1exD7*O(1TjWf5~LEsL4@b z-Rjsr7i33)UG3O$7i33+UE|o(7i7DFo$1(l7i7nQT?>|eMLAz4?dPRv|1M4Ybr~8B zleNpIME5hH_vg#Du1itBDQ38q^BMlWsnS?i20iKlTE2weW$$Kg!zX_8ne5v5A#BI3 zn=jkm_b@2#cRnJ=dK4v(A!02$T68miK*V(OIIXi!kmjZSOyA;9{k<4{9)i{KwohQ& zC$Vj9Z2PiP8>gNI^(-j+oATZ0JNWGPgey z!Zd!_y3tDOCQ-`Y7sTHG2JgjV-bXm^|4`m}yMNIZ#mp7)|FAGCEz2U`!*sj zuL!mhs7j!UI4a&gRY1K4st%}sIx4&Uv>LYHJ=90!e5%7s=~zC#PFX$cdK25%1X~3W z?{g8e+uLhn3*OK3ne6u6w?Xk<>veI&cTn;kB34Dj2VKOtuJ=KG0O~bRk2)&5BS2kj z!DD~2K3E^+4QTnISx+td3VCgm(6VW!i8$V5m=?spY~p>gSt0(Njy$F{l@&2<;Wr#7 z@|gBm-x2G>7(~}OamTYWtuJA^(()xUm97tWrM)o2bf@=)O;1{0W0sJ*Mdl8QV~@P_ zi+I@_$9N3MV|vq8MND5>7Nfg;enyeU^vC*v3G3-wr1Qqd5vgl1(Jz_H>GQHy7d=^) z&DhMNoX>SAVhoRtmD1-(_?+Q#rO&d`dq0YdroW50@#7bm6Q!H6wC*l?PSieI%kvXo z*W;2%ddZpJq4?e>u2EFD53F|Z5IJ1b>moQ6c`I5QcwVy0o zhO%EJEn9)IN0OGULRs0QWouCOc+#?UDEm#)vJEJEB5Bzsl>OF~ry}m}msFuv{E>WMM zC;F}Rns5Y3=Fj&_t&@A5Z@CMr;-2Q&6Vd*WEF$;1WF>qIN>;+hcVA~=98bpbsO$U8 z$w*5c_vFl%=u4+*S&V+Akmo38>D^1_Qp`y5xa0actb53<`-a{XH|7${m8aucSK5}J zvSokiR1*AUfZIwl63cOnd|0>5t>bO;)4C$2pxc(mr%S+nkKWn|#aUAbDaDQ_g$7Ak#!QBAvBy^>jn<5QqrkSc^ z)^jtqPj~gX1Y6SQE@^FX^B(Fj-PO%$SH{ZEh<9hjyRXH&bK>23_l}SGMP2lG4W%2- z><(APpDSavUgv7(g=c+tiX7IsvWoOwRV7U&T0TW1VR-j6z2jP4?#R1AmU3ixXRjXa zb2Er^Q;edNG$m+h{e^ed-zJ)|{n^|_Wp~lLyXm#KX+)zyS9%xkMd>E|`nUDMI#xOz zoMS24+CIZjnqEtqGPJZ2*vkX-&VEnEBBRQkBE~D1;TGBl3 z%D7k6a>`hBUFU_%K5%6rcUw!`JG?U1@}Ai4L$_UgPfRx_u;2OZI6@0Y{uyol7uV+D zovrI9NN-`&#MLgxd)LK#AKQIEOH1-h^hx52QKn6zzQR4AvFiaG`CS*8-&Mss@jw&D5n&n^=x}e46=#+wMij{n@?#gvTQd zt$)+K)^xA6-Rs-#mFFI5<{kI`J$kkIzVF_D;9h6Yd70Hy*S)XrINo>COau46k$Y|8 zUYpUYt(O+=eJgske%jEh)!WX!@(d--ba3xGxz{fAYVY)QbFV$<)wX#rdbNK0(5v;+ z&%GZ&uU7vc_d3MA4s)*~=+*M0==BW%X=GoL-E#@)BpXO} zfb5n_jj2f1n`{%=RhN-2vc_a{$i5~kbGb2f$R?5&{D~rJ+i?8V7nXEV28nV1Uqq-z} zooo=Z)|6}(*(tJO zHycx)tOwazvdeBEJ!DPDrjZ>a`^l}w{GO~H*?6+OWY^qA^+EPuve9I_$gV0%^+@(E z*?6+g$$oJ=wJ}*!vT0-o$!@uW>VT{v*(9>#WVij?n3u>}lg%dkn(V$ijd_c#KiN96 zv|kugimV)28?ukdj*{J2%$VoN-X$AG_9@v7#f^D}>>aYbY8 zWWOkB%wNb_lg%PKPIl{Eq=&3N*?6*@WLN&un8(TfLpFqL9a+JXT<_~1=lZ_zTM3!-{F?W+yBI`gli|jDjb)}7Yn(Q62QDl3_ zuDFl(GqNgVoyg{q9V5G;46$S%kPRc-LRRp8(nj_wSu3*HWQWO$JV5<`?7w7V$UY^z z>_O^xWYx$zlg%MJMRxN;)E~&+CL2n&j_ezWW&j3kgX=$LUx4AJVx=z?jU=R?0K?s zWHreel64^)PBw#VA=yf@?PLeZPLbs)OMQdv7P69L50aH7`#o7Xve(JpC2LC7o~$?7 zXtHT!3(3}!?I1fuc809L|wIslD$Cocd{yEZ;^dK)|jjVSzoeIWRu9|ku4|NNVbdY5ZNiR zZ^;V$mfC~tMzWui-9vUiSy{5@$X+5VM^=@r7TLRG|08Qk)`qMzSue7IWFyGNlT9U? zO}2<^CD{hD?PPn%4w0Q8J40rkr2NP(BfEy|da_%{?j-vqSsAiN$etj3j_gIUm&sls zt4vm%tPa`xWFL|>BWp|6g{(K(AhMBUACXNXn?tskY!%r?vQNnNk{u@dg6u3=+EX;X zlU+`BE!mA^Mahbj-A#5s*`s8?C3}YKcVvGgdx`92vVW45C(9zMO7;fXTV!>}-X;4l zSv|6bWKGFhlC>r4NY<6CCs|*zfn-C;Mv{#sn?N>&>|?UoWb?@ulPxD(O}3tFGud{s zU1WR74w4-qJ5F|r>{7BT$*v*$DcOx=w~*aV_6xF-WcQHWNA@7u zBV=XCeoH3*rQxmpo@oCso&L__-pAMba{o$O&}^L+{2x$S@c(x8#rWTg5y4o4m6gr^ z->EFY!*}uwL|Y|>G>J|WU9TIMBh|perfI&cjNCj510qdL*`fJ zVe^Q2)co2!X3Em@Prorw(0@;2`3!v~`#JNx`JH*e{NDV5&P-l3f1>jrx(jUnYW|Pz zw*8HMKk*;*9hv_!<;*Llym{4Bpl7DyPfNW{&sl~4YMNT~Umf$d`L}t;yi31l{~z!X&?R91^2zW)Axh+ zG`&o3)5r9sCkgt~v&jSLsjb2EI~YUh7p{lXuRD%3qs(YC#*8)N%y{#WnP4WGNoKN{ zLU;A1nd$V~#WT!IGt102bIe>b&&;R$*$eHHLS~6sYL=PhW(9SQN_6I4iL$Fqua)Vw z3cXgL*Q)edm0n+?*VpK^JH2+N?fTPef7)^&y$+=H_O99tS|9$|PIojj9m_mFVwr($ zf7miFw&i6${C(kNJc=f{G`?Q*aVB0GUw>f_*_CP!@ip@RFJMl1c7AIqAdk{+8X} z>?dA&Vb5{$!hdVe-tUa|cjbIdyw_IEWjt0+{*G^}|BmPWegD?p;yK1k*F5crSZSx z%=c9r=d+8e{cAGrNAtQgyEWN)&Tjn=i^bmbGVU#2n%(+1^JD(L9OJWH(mg{gugqp? zOOIw2j*+~Ew+f|4Gp~wB&s(2zy=Btq`FFdXJ%qnK$$kRKyk7b!K?yVI=4wQ_CSj@9 z=K<;CUQ2vltYMN%c@1AJ*UO`sb}LpA-->%5-49nJh%p-T&uG7Zl(3yvObp((YGcm;r>9l^W*i~9Q__`M_O)2_LgjEYmc=t$^Aqw zZAPqwS*WPu&-%%9**}M1NFLXpedI?Akp$Hm5 zN+7gQ1w=Ze011!~0)!5s_uh*UX(A$^0)IpVq@y%xB3*iuE+Q)YzjL#5cRP2#U-HYl zUx4@C?dNuPzB4;JH?y}_-rYTpFYYHUXr^0wvWi~5W5 zVyEEwJg+C&IPY_jFCy7?>DGyi=#hFp7d<^`%S)?o1Z%i|)M?@_o@o+kz-M?I^N2|15&X=HC9h4mdCi$Nja^<| zbmkGM7|J0k-SX47m>;LmA6gm`CRk}l5Zt?^1TR>_os04GI?$JUZkGa zIW>3E*T$Xnb@9yCgOl1O4TovTHq{~w6^{|8~^zWDw>%Up!t6fQ{a|1)wjeqT6+ zCgS&l!|{6p|CaE}Uf%zIm>!|O(4+JiJx)*1lk``5ivC7V(=+rep0;|PUZ5AD@g;hh zUP1mEy-si7UjJM4Hs}XHzX$pu(6`XX@Wg%eFZ{OfAU&MwWF3zrTAqN8C(*v2dmFvX z;AV6mLwXiBtNVtV&3)7Lb93NTvDKdv!z%7V%Vb>q!qV6eL%>9j)a7((S+|q6U z@PTewx13wvt>C`nR&*=5r}5s&AoqRu4D`ycApKqbSCBnvQMZ_z1Raw|w2Y+ziIzJl zujg5>XD+uR(l9sNjY!sA+^%kv8|`*;ySqKy7&jJaoZAcaz1==QCtw`|dT)1-I~bfp z-C<~PxH|&1qukN%7o%=PY-@5DF4emzwe+hnr`@Opb_^pt* z-u)5yo$fAox4Q@HeaQdh{_Gxb54wlk!|oCH7x$=p%sme5N#suff51KCo^{W;=iLkL zMfZ2ezwBN??KSthdjt8C@WN9-{_fs$@4FA&hlz$q-R|@+_bGI-zH5WznV6;75>v&Jxo?bw? zc(eqNrJ`rtfM^-$SvFcOT0UAK;;neueK%Shy(|uVEd3xFORHgBBibFU-UG5WIFGu; zqYvEB=tJn*2;5B~_Q7V+X3-YWkHG(Nv{ke<@NJ_{A<0^I0yT+7L{YSJv`bV&K02BN zjk^FJi~6`|!DxImB6=T^TR_8Vpoc~WMF&TRM2AL)MH8dLqa>qobmuqhpYci;j;@ zh)#@7icUuDRHV~UI~~|vbWU_`bY66RbU}0>to}(fDf(%2arCq3lIYUtvgmT;KL_W$ z=qlisL_bL+uY%OMkhwm(A-Xa8zv!mucj)8y(Jj#*VCQYo?a?2jJEA+`wY#HxqI;wJ zqWfXXpJB~|(L>S0(Ie4qXmLmMc=SZ{Wc1hQDU=TaeKLA3dOmsq`Cp;qWoWzyyvL(A zqBo^#{Y>Hi~k!f94`ra>39Hm1LI}m7oz3k72@4##rV@`AfyL@`XX8-{z1Gd>Z_xC z3){X{JS5(p7Kzu5*F$Y+ynOswv=(}J8`iCwXj=uc%f^evH$*Ffzh1muygky6@lNrx zG&~*=k3@YJprhi^@mShD-Xk6pkB#??$HjXgA06+5G?w;@_m2;V4@7=&ydP?Zp&S`c z!rKE!LDMnuvGH;7@$m`qiSbGC$?+-3_li%C&xp?iIwn2`+!NySkxoWkj^X%e|7(}T z@>b4$aBv>e)l?E6i4Oh^j(@qtb3@73SiXs8p}&Q*A--pYXNT~v!@@WZ;&Y&^#%Ds= zIU}C|t%S3m@8j$zTUN!H5I-EaJB&VF4NuscyMv;$b7^z87_@yC8rHxo7L{#rwAwM@4?{j8Id2?|v$!z{ zmSubT>RGv{%!_!>i>EcRG%sR45RqLO=do-JKf5t6&P!!n5Xy&w#P|f9BeI@H(v*GW zD|_Z;PxLEMITJIG>dWlwO>H+fZ|-TDX%&^dS}-bSf9<2PcMC@4w6A?s_HDta9Q3u1 z%6=^vm3_YUQ8}OmqcY}eAC-ezFe-a}?W1x?3r1!9*FGwTwO~~C{n|(6@D_~9{$Km3 z;BB*(yS5+MiON^-j?2lNsC)&Za&RXqU%{yC(22@dFe*EDqVg4t%Hf@;dV@Vh;~ zTgA&Z>hd%x`F=9)&o+h^_Hc?rKIAzaL+y$x_ zm4y(Mg}mz)GErfCLZW4%UPVRqd9}SPc5@3&^QfE%8>dZF<|9t|sLWRqmH7}Aeix6I z`Ak$8pO9#ouUAn~eTmXuXS=!irg>COhLxSD^c1QuBPyp(v#89CCjjRT_LaFy_LaG@ zukhQtyv%L(6~-qdTITN6zM}dPrM=E}b8}DgedTmm+1Xcm3f1KIl{4v_rjqn7I?e}i z5x!xCukPV(m7n98#9!fiRlmh|s(ASw-Ws_L-=?}7?~P=&KV$U%CE-7g{I7Tyjf%`*A3}9! z22=l*mEF%^DqZCaHm#%5pE%{OBKwz|8TZGTaeu$L{mq#%;}a4s{d;w0toqc8y(@Qf z{ipev@y9T+v#<0Nr|$QaPkgf~_mwZ>Xn*jReq33Q4*nV(SEk*4u=(*;-~7Q0Hh;+s zHa})C{tgu{^P3rr@d=5R`Fk~kslG&Mue06U{L_2}`xI944E8B{?%Ox;@LwW{M_$4g z4d-@sc6Q4u+o58uFJd;(BTy}6UG`jYNV))P-ecXRlshV*;Z`8V*`cf9o74E8Bi z^|mV8S$<}`B(vz%G%;=8R$Y>I!q-7$Su%K6^@9W-pL{)JNqjC{m&w1as`|Xzpy-Jw zg5AHZ`kB}CU;3h{Bx*zl{$z{rr%+h>;cFV7(aiX|#_aA}Zk`02-!0@8!51)=#Md*j z+Oqh5#!3l)5b{;rYHqL_;?{LT6U-ZlVk1w7SCR6ir7S8;+o&v#sE8bY=PCaN@dpV$ z9yuqCiF1>Wit5Yk%W6IGM5_C!sFo_DvMD@QM>OznL2gTwTf6P>U6J8#=ftC5U4w58U5{@M-R%C5tKE*T z4&9UR???UszO(bFdjj7LdM3fL8RB_QhsTwry}&ypN$Bv~6ief{qH;@{;|k;B!R6GO zz3!{jCy}m-{>_ zFXOvZeH|6~jwAkS%O|13ePsal6_p!c_7%p*gUhKm`-ZXAV5+D3 zsPNp#QQ;Y^ucNY@-wch4&|y@TLsV35ITIDeCxFYTH&Ic2>cz0^nL8y>sq$$5D&lcI zj`8@XTK9Hxw0{?0=KI+F3kd$nFL#X3j{E<^xBTMd?p_1=bkU5_EYWPy9MN3nj^DhV z4zE{q`&%U{3;3;EC|abcB%?%hEC!uRCVB@%%SH<%X@zK|Xi$Q!f-esaj)p|*;_HK1 zZKG(@X!C@>CGxGK?V=r{;rNE(s07Q#;O?Fd&%u^Y1>Pl2O6R{1wmgn2iZ5@DD~yi^ zms4+!E2=N;-c)+#cYh8x)^BB%Ggx>IHa6l^#aG*SlgrP+vaN58sAzDjjEWrX$06Q) zTp5S2bdHPq@~dt5Phi~)JgLM<>HJsQR>W~d@fFQ+h4BgCa_Y@-MfIiKn@Z38FXL#> zdm%?f_LaVk%1Ws>Bq~bhqp}jBqWDTCDvVD6ms4+|qWaSAO{Hi4u_TTr z_#fS9e}7y#2;Z(f90<`-xuZ#TTsaost35g4pN9O*=p1~D_M+sgwBd$cp2*-|ljA&| zZGCG*h36Sws+_^X^WSLYZ7x6B=fC9oKE0p&vTOE@)>RM{m0QI`h4Jy=a_UV~RG*DT zM)xGqeN?XUQMo$0wy7j`h>qVt=M9P8oA90Lt9-f@U%qCkyYSWPKPBI^ek@meGWuKe zY{Gv5`AgBO__Foe_@ea(36^c;$DR()j0a@`h{^l-&jDSss0>0>6dz=w!uWV_IrSzg zs!w|>mOV*y9~IS8WmNtJ&)tAn;EFN&Um6vPQf zcmaF?e9;6`EAcC$r^DmQ_cHpYJOJ z@J|qu=1q3*ssBzsH!nX2TO-k|*@u*;*8|Mo$zOwZO7S(!(VpeUgUhKmNBev~P*J+9 zQHnA+`J;XH`^rGSuMEV0b4co|@8oB(5(m7M#*S&;bNM^@tK+z$F_DzD0RjqcRv#QGBq83gZ*N<k7GuR68N=+p(MRW{;&Q%h{5d|rHEJSqNJd|85JPXD>5!|RoBJbwN66`uWhsd5Gj z&y4k`GSL59Nnaf8SCx(M`G@%ToNUpf{i?K6imz&p_AEahTu!|?+UNVpdJJpb%XELV zzpBrpa#frqwzn^$lK;;j?!(g4;=hX4<&Jx`bRQK}b9H=eQ%UrQj^7~CH^es~zd8N` z$}F+e?eU%QJ@NhV1M$PT+N1Fk@l(kNc?S9O@!#WD;@9K1;(sKVT1oW2r^6ZS&P*Vw zS(?wm?!?hv@jK1Yp79Cba_Y^|UiEROLz13*lXZWzS1nb}U~(1tQ7q?RAH`YX-m3ox zDN4PqTa)(|W78rk>uBdwTz(l*;kl8|jI;SmX3JKMN|pNx&ojJKIfMNZI&Vnyej5KL zH-AZApXcVSY%X)PTCJ!5EiiK~INg#`ZX3>95;yh36Sw zs+_^XbFggwlD=B?Xup;?vKM*xNBjPM>@8aB-&7J~R0omhRa#4y>R($BtE8+ZwD4Ib zOQKnGwO8rBEwdeU!#?Y%O!H#C<0Wc~HI>9@)j=eBmDZA_w7ytv@wBcr;pb)4KGCmw zm6{f1Bkg31v{rS+YF^4mWk7AIrjppHI`EcmqE~4xS&Hie)4sAAhfO+>Gkm znxEX!$p(GaQMu;jeN?_vTfV6zW~&Y&(W|tUEVX=XrL?E4CKAGDnJkHB&DCC|`?mDX zK__MNOf`L%X&&S|Ub!}?sU#Mv4kFR3w3aL-^&h0YWi@T1G9npnyV@I~eMd#9>W$J` z)s^lwuU;G6R1!;72NBme-yo@FxNwvX%N-LX(k|nKK49+DE+Z$?aB}>~H1C!@^?KK; zt=&`-lT-(OYl1e(6%$0Tg%ds_SuNv-?ZS;^tyr=}Ggtwe_1yVyo)Fy%uPbTroiuTO^TSEUR$~ z=Btdf480&SA=Z}^UlGf)wN@%GyVqR(^$K5UWY;EA;|$ZK?b*On0 zzm-3%ZPHW{b5#evu$1UkT1%GF`X;r_(;l@Z{Jf0XC;C;dQq!XRGOq2n^ikQWwq;XE z{G&REM6c3XvXs`hN&Cxc!p~$_Ry1pFX~p}ptf$IhFW`9V@;6r6qg#>vbRO%e`Ky{v3?Yp0e{Tsc}>?OwYWs zvoYc4Wxo7#urKp^rTTL)K3mfBEiKvWTF=hBvYPNSnXkq<*tEZ1VL$M)eQn34l4wyK zL|o&1SfG`qRHj&tL<{DJGK^&lbEkIMZZZug$Dd5|ZrQ!pyQ|;I(Y0NhN@9}gAY#3{ z_gW)s;g+h0%Wz{Y9dcWZsy-^a*2Z+AGL^$*Pd^s-s_ofSlI~Rpk?7sCHVbN5VyTR> zwPg||+%nG*&6=ydO82H5HW}f`@h8)~TXygD>Ti41{}#zc4n|b6UlpaVJRR00Dt-4` zr2N*|t@CJd{O()!%;~*eJ$hDuv}a3qtnJ;4qy2l&VB*T|1SJh{)PC+CR#;vq%Y3>b zt^FdcDX+Y|8gk+#;a>}iYm(>NuvDJ5b>W9HM8C;JMSGI|h#qa}s9ayW(Wjfz+V9hv z^2*CwAtzoE{vSbcP4avjmdew%F8oji-+9YLMSGI|h#qU{sN7S#&!<17wFlFh^2*Cc zASYfD{$rrHCV9RMOXX=>7k(&1^kgn7+LQE0^mt1~P{~Q5{6AcY%UNmQtCrQuv{aXx3cqRk}6Hg?v={ z*B5FkiFZ^75$j#Fppm6irmPfxC?lFRS9_Ii&2ljxl_lznHI>9Wssmq_fZnAF8d*wZ z%1YsfGNM^?wO8rZEC=|g46F}mDv5Vg2NCOCuAq^nRHm#HekdcFHCKC;ZtZe~`ie~@ z@s8@if6zkPcMBR>N@dDQ;fFG!S@RYBC{nt$%R%*(n@Zvx)j`Cy9~3mQl**Kq!VhIc zv*s(eRa921uhCQzuc;0quB}3otQ3AIBbqheu&tu9X??S%l6XyZ z5OHmbf<~57nX*#&p^RwOe6zNQ%9egBx2kX1R1&YL4t&P|dbcTPWGR&?D}^7*h-S^z zUZq>R+^)VuQ%Ss|I*7QoQ$Zt3sZ3cZ{7^?^vp!|NlPN}@+~5OHmnf<~57 znX*#&p^RwOd}LchMUS4(R6Nqdn2M&e2}oJeB46`aVr1=~i_Racx3DBTK1F zStDKOFKd`AJdQ=Ay*A6adWGR&?D}^7*h-S?XY^$goTA$cd60fNa zBCZ`#(8y9MQ&tK;lo8FEPi(8G992K2sU%)g9YkC^uAq^nRHm#HekdcFH9w}UqH;q0 zq^6R1O?41)?UaH>mQtCrQuv{aXx99swu;JW^)s4E;x*NQr|O~Ytb#_CQkk++_@Rtw z*8GgNipn|l^O{QHHPu1HwF?RwSxRNfO5ukxqFM9v+9E3Y&C}|?21;~MJ^!s${obm@ z@-+Lt^Nm&=fkcz)7dMqeljYr#GP`|44MC(*WLz`9qG{u(n zt2<9qsC%cfyrzC#Q%Sm29YkFFO+h0|sZ3cZ{7^kwbj3YH&6YR&Kr2@-d>is`muL= z{nn&8nm^K3QK|m7 z$X#oXcYcf1!PLuAf7`46w@7^7zVlmTFQcL2dIzKWJ1#hr>fCWrH}|sCZ#`81Z3?2N zJKv`0Wi(V=?~GS}Cm;VQ(YceaZl20=+I)ACjkDjH%;@Kn^Q$lKyOYOce-Js9UaYFw zi~daWrTU9aC9y|!5V77@3L05TWy(t7hccpBbG29L)-3g^too}+qSrcCkyANn^s21- zt4N|ZI#-cX85Mm?r~0>wh~DbFRWy}R(eG_l|9vo`cRJq(o64w6^hf*Z-}NGTxAU&o zR7PbDAC>CA+D7z#=c{c~85MmWr26+)h(7GRzcQ6k(QngL|1CYDk2~Meo64x@`|H)e zZ%_0|=Y9LBjEcUKTm3u4_>ZB^JH%5N6@4eS`ge$lKI^U4+-{Q*?&nWCj3w) z^t##`qOC?v?`N|W^O8p zcT@-d#V*mSw3aNT^|DgIPjs1E!qVWL-QEm=zIWu@># z8PTk{+N*SHmUH^3%+;8)sU+S}9Yms6X)Rew>t&_zLmAPmx!S9AYnJo)sLa=xr>P{~ zQ62bi!9=gpTC$YZ%Sz#gGNM^?wO8rZEEn)mS*Wo2cBsE z!h1?(B}-*gu_pXbMl@^AdfR7Np|N69Nj#-GaL)wZQz|Q2Dx-=u;fFG!S##Fg`b%Fv zr5FAiC;w|F`?KudI~5atC=4*>B|!8Y?%I#8avRqdn29w3aNT^|Dgs;+o`&2~xz0{7@$JJ8NzIWvJiE4I4w7O5!QiK_q&W){>>PURDY}lo8FE ztG!CMcDZq5)25PmM|BW!O>)HqDPl!_C=>dfwYL7UncvDS8k;qh#8avR|GJ&%Ra#4y z(t24b{7^#8PTk{+N*SHmOJ>U?9|wysU+S}9r!QwM6c3X zvXs`#O5ukxqFHmbSLxO+hc`wxmBc%$1OJ5%ZIUY{ND(XYLz&R;thIfXT^ggBO5!Qi zfqzklHpvweq=*&yp-dP9*4jSHZjC*fO5!Qif&c4)HqDPl!_C=5>KfPBCbiUm>@;0$PZ=07_ipXU;6SpE<|gXFS)QkEtaQ+AIgL=PRti6q5zU&b zy-K%cxtWj3ruEI5O5z>WfqU_ZUZu5UDXo{4!VhIcv*v2A(k)v0qh}v{1%Vw_{*?oZ zX0)^k+l7vAi=(}6@)3=rno6Qabr5k)a>WEGVnu!^qZVopA>EqP^nM2GSD(JAB;HXS zcndhutF)FZrS-B>_@Rtw)?DpXy0yz=8pk!2#5<~kh-;E7CP)!0@@2s`;7oKN$ zseT4yt@+uhGlQ{5dbga=IH{>5y{Qf&u1T(#AVsXm4`ssbV6Cmc?BI_p+tqhyDv765 z2Y%yC^eU|-OKH8V6n-cpnl)E@m2U0wl*VaICGn2xAmWzxwMH_E-K|q;tJ8CSOw%6-=Nl6wAM{q+jlp1&69T(9&#n&&jmYbr^Ps)Oh}9F>*IN|wr~Voms=jA+)J z^|t<^=l|7Tudwg(*CL(kl_?)h>-n)){Tp~}L;mf$&Kr1B9u+&8nybA^w`Tcj^1W@M*K4mfmBc%$gGlr$ttCroy{r^|C?lFRS9_Ii?XvoRCgiUZ zt!)0AkbP*eJT3fCCiHuIeDi3MpTRC}Olm5L?^Fko=v7)vmeP7zDg01IG;6N*D&5*; z_1`>N8S%E?HDVuHEKdtRlnMRLTH9x-{yQ%G#TEOl7WSvb^0e?nnJ@fwZ3^Ay>c36FUy-oi zpWxmsmZyau%BY1>w#T`#m3Y2k-5VGLMn`z+P}XE(o+Da9K_Z{YE}a`r7dw!~t2TKJ(% zXnlLUfv4MC{Tq1vuAF_#j(f9Mo)&&6qZUfp9&h03mz}Er5*2$f|COoEm#D(=H6>K_ z?|SiDS@wM{w!~t2TKJ(%XnlLU>ov+B?W_Mj7<)7S?XS-F!NT!1B~KO>M6c3XvXs`#O5ukxqFHmbSLxO+tA8e+ zpPskR-LnrZmZyau%7lJzk7x4ri-FaD;gG$V|9WBP3y0zO>WR99Ks)I=MDy=0;X}zozekdcFHCKC;Zq4!DG*EE&HJF0_7 z^eU|-OKH8V6n-cpnl)E@m2S=QH$E!YH-6Jp67Q%EBGId~mMo?9vQqe=jA+(e?Nz!p z%Nu=EZfe}vR1)u~4kFR3w3aNT^|Dg_@Rtw)?DpXy0y#d?}PCjD|@eteQ2>fE&Na>^m}{U2fNkpE4MdpZ7PZHR0omh zRa#4y(t24b{7^u~MfRt~ z^0e?nnJ@VufU+>u~MefaFd0P0Pj9MsVdt628HdlWY$=7@KN|AfBSe_Ps zD5Dli*&bJsy3N&JMe_BYy;9`fES9H*AIhkOQnuGsWcAl8d}U#;N!YU%%hSRSWh5G! zx7YQ`J&pUCO5!!uLBuu56%(Y075Slz#zT7u>DHzWsvXc&67Q%EBCbiUm>@;0$PZ;g zzq8iXUwB0GQvJTdTJ!rsXJ28D^lq6{zqqL+y{Qf&u1T(#AVsXm4`nD=SZn(%v({&B zDv7652NBmKS4@y1R^*2=VGLMn>n}P_SASe#-{p@Po#RUHN7J13*_%qzqv{~yn&gTJ zQpAe(xd7i;+o`&2~xz0{7@#` z7S`JOi~a|8_5az;zT3zA4|o}tml1y0j8no8mu)j`BH$rTf%h!y#v zOz3si+CIyl8V@v;#8aw+h-;E7CP)!0@`Y#8aw+h-;E7CP)!0@o5EJu{WWye^W_3r8DDekY<%2Q67Q%EBCbiUm>@;0$PZ;gzq8i%Sw3le+EfxxsSYBp zNv@b6MXbmVWx^P+*7jMhuie;G5>KfPJXH^Ek}D=i5i9aTnJ@;dwe^>S{n$IOad1;f zJf%8_M6c3XvXs`#O5ukxqFHmbSLxO+tN)G*e{scrtA%}Nu{WEGVnu!^qZVopA>G>*aZPf?1Sw)gekc?Aowc^la!>8P zrjmF{b>L}yXp>wqL5f(BAIgL=V6Cmc=+o5IKmEd+>+VuBR0B0rP~w}rK~&oXOc=BAQ(N_Ai_fi}q%6Qqb0`Jqf01J>F;%j}KWno8m+ z)j`BH$rTf%h!y#vOc(>!+CIx%jX9f2;wjZZ#5Ktk6Qqb0`Jqf01J>F;%Y2P_no8m+ z)j`BH$rTf%h!y#vOc(>!+CIxdjRl%Y;wjZZ#5Ktk6Qqb0`Jqf01J>F;%bT^gn@Zv- z)j`BH$rTf%h!y#vOc(>!+WL#W(N_H%c%)ovQv_ zFZNdcJ+98XUQ<4PZuM{J++M%6sU$tB4t$9v(W|tUET#3bQuv{aXx3cqRk}4xeRrz* zcfHtO`S-Xw?|Suq1XaJU@Tj)?1KVY>JT3fCCft_x*jMz8w(8%&V{hi)uIs#k7mlx< zD1D=?`Zw^n*ZkXcoj34$ibKfLH`=Oy1CRU3zg^dP126Ralu)nNUTrE#AF6|hYmzG_ zND(XYLz&Qe*4p}uzGqnddy(wB{5y}G_adi!G(F|-D^>pn9@~(AyRP#F-jqj0-)O7; z4Lr6X|8`yH4ZJCj%A$T>>EBqisU$tB4kFR3w3aNT^|DgVLQgMMeWUyR;*3*XU2y%CN`DCH>!h3^eU|-OKH8V6n-cpnl)E@m2S;a-!rWK zy-4;~{+-9pdy!KfeiG8}jcwcHWDe@~G&0hSk3p$u{KQdF;FwIptBg%O6*& z{|`L2A^-n&o&Ug_@~DK8&Xch(od+CklOW0`8kSU*PxP6gcQW+c5i7{#=M;UZ!e1Ks zKO6cjLqBflcMScYp&u~x^vNb>_Pl58x!ch982VmA-*4!hO@G4;J*Vma9K%1)(B~Wa z4ntpP=&KF=D?=}0?ES*H(wrG~!A(7!YEtj51H8UM{}=x-Q$RzuHb=x-W&c07v^b`DEUK=c2@Wc^vakDUNYkieo+k zuKU00FY3iOE^UwTT#92{m*N=Tr8vfUDUR`8ieucD;u!y>IQD~59P?c%j(M*X$NX1{ zV;(HUF&~!V*bhr_?2n~5_RCWI;%?-zpO(sFe=Wta-jpo+8~l!LaGY#qB(#xSfX-xATzVb{^!Boou?GH^OWLt zK2qGyM~d6|NO3zKDQ@Q@#qE5gxSfv_xAT$Wc0N+v&PR&d`ABg)A1Q9Ulh0V zi{f^EQQXcWiraZaaXXJF{+^jf6u0w;;&vWU+|DD4+j&HBJC7)C=Mly2JfgUrM-;d7 zh~jo0QQXcWiraZaaXXJFZs!rjWgf}q3B~O^p}3tV6u0w);&whz+|CDz+xb9oJ0B=+ z=L5yaXTLWptzkE6u0w&;&xt8+|CP%+j&88J1;11=LN;>{;#;*{}s3U zzv6cPSKRLZirf8Pal8L3Zufu1?f$R0-TxK0`@iCL|5x1Z|GOpodv<+sd9r^O_h-fJ z{;ar-pW?DVyY#wL@W{|aHqP4K##wP2PsQzitoYsLxUaa~j}^E3vEp_=R@{yw#qBs! z+>Rr~?Ko20jw8kGI8xk>BgO4FQrwOs#qBs!+>9gV$5A1!cAnSvc0O0!&gY8Td0TNi ze=0uQ+;3Cd&Yz0g`BQN_e=2V0PsQ)(re8aMs=S>)6}R)J;xd1x`;$8%*LA zjf3;Qd-}cFDXskI_OQLDv;9-}M0Y3qPug$vN7GK|tewzAvOi|+?lA3y&e{o0B>SZ| z6Wg_~wDUkpCn!vF#~Z$NWn@9E5L;Srk&7PJE4hW zzDc(?GQY!lI%_9<{9b3qo8mIRq;|OTLpzk#4r#3LQ+ixLyU6=LLE&#t7wxh8ui7K~ zU20G5NSYqHv^(eNRG+7bex8UF@hiK*@9GABE;+<$|9GM_p^JK7G;;qFTDQCI?e>N{ zPp5i3U9daFjom43>`w8Qj6I)+_B4z=o=)xYbitk!H}<5su_wh}G4_x=m!kgZGWK{n zwa3#1ds5uklj6pn6o1v&6NUElGxm5owa3#1ds5uklj6pn6o1XwQxki1ytoSzPk9pa zmZue$=Q^DiakjV|I|YNSej^{X}_$Y zmoxP8hF-zYpM?EK;r`|fP4tDxCtA2VRP-e%kN0qYva~)asq+tx4}m={GRI4$)Bfs# zJkk1w-azQop2L%x_UEFICh9Ws;r`^JkS3Z=MJ)QDTGki~{{4)&S(<%RS!#8xi2l!oCd@P-ucjkY8 z9s&6r9gi{X4eiYMv2@JWz4`tP?T!NqR%pHFIkN~e4?UTU6JKCY{R`B7=**Ua%o>6Gt}?`i+$cwh79$0*}jISQ@$TR8ZTVW zhvUiU?-DxF_73-zljXWZL+S^QD~;=_q)A4fCGtj3dcPpmD{{sDJg#=9@}C5D;CCt0 z?;nGHi9GG~5KS`ek2Lf`M&BWZKab%bYWU*}{ehtm7ux%U$kPeR;ERF0bK3{^VR8Kt z@Qd_PhJSu?ewz9{`i1Wqc0sa;j}qKVxm4cciFWL_DGqyqIfE-rw0pwwY-sg#DsQ(B zzh7d>+V7B9lF@q_8g{XKf^z0*)gOEPslbdPpVBAD`Fq;_Bg41!=ZR!y&$+?*Kfcpy z_{SOkw~YSt4F6!^r}jK*<{FY!p1*eE(dl~mqE>d}HXQ{m4>VK6_>DX_N*T*B3tI99Z#~S+=G4h8enzHss zz^eRs+ePRkXO5+xHT?SxJ=pYjyzp67vUobxKbx`F&W9(M_MT4L|2EN@#o;R759V*k z6mj&=xR-K~$_uW3QM}k5_62%>*-d|n!``635YPG(JnK*8q(7+eOT+lr(nlG8E^6e> z`G?y-(VO|>Cg0Eh%yv!=@1OWP+a8h6wny6^mb{;pZBIXApQT5b?ODvo%kQq4{lD|} zV?PPvG&vmmO{qNgAK=+`iJq*#o3pqC?I(vLPNnjQSJ0p6$@-J~8@P{Swr_Gc{2277 z^6+QSpWs>Ce(tx=?oW!Jk&KhfKZ0lWhH^!{etg3QS?)<{st@mX8hRHQ4=P^Z-!1Yf zALAHHrq9wCw*gSG_J0d8LWuzCz|# z)ra$^(tf(baX!fMVkC|xxB8lW&ML&!4wcWgOXb4nu85W~emw}iX4{A7G=%oO6J2Zg zmfpu~|r_=uDHsk16 zBk$={{$AmyG+*b#z9i>t_F(&q^oE9iRMJx8h~EJW|7yeM`y6?HgwOd9JtX!eQk;3( z8|d6+2`97HUq7Y50MmbX-;3ycX`jlUXlTA~lDF?C37sfqJokX4jF4-pn5o_vCsP z?TR?;V%%HfB9#|h^(tO$Z|8UZ-9Pgq_8UXrnYbj$5{{=m;M{>mUg)e}+0T>fXtrD1 zMT$cok6F$mR{EI z&ouse#B7JBy*?M&ar+k|FLdVDKY33=4sctY5vqgLZ6>Z>Mredtll>(T9nI($3laczxr~H$*NWAN9-G%``7-nd7$N3!C|Ba`-N0eh_)xexf(c_IWzpzE^|o!M8aA`(db|r!#at zpj~YJ@ww6enz6^zsXe;}`tX02M&8q@{K!N;8y8n5-o@{@#*d1>ZpL+ptKVECxY9&u zhb8S+=@j_L*c}RG;9Pf94YV6RSD1^f!c`8er);P5b{E`z*b=(YLc{9~*v1yLgAh zQZ+~Eq!BpTe2aFa+b_6ozl)4MrF{pk=I4o|SI)YOeL@%e2VUC0;A%h7^hTf1MSZ|a z^$DJCkMq}w-d^YFbewyd=sR*hM7Psjm9$X1oTsyP!q;|1`!Fs_?Grq+&!6w7_6eQ! zqvv_vK9!5Y?ZWdw#&4GXj_`eZ*lpy88+uhkuV~tPS{+2&8Tq{fd3(M=w6T%jG>|9q zbgF+FBR|a0t0Z*lx8(TE8@opm2lY46>Onuce<t3O#cpnJdvlh0qtn`o=*9r zjlFvY@^b!*?MdyC`=2T8!uvL^K>s9ZsrEWINodvQx9;D-!+L{wXxkpUruW;%{|>Ki&=gPB-|2-QW)-xY|c_wDIQ_ zF&mum&k6W0`kC3DdlMS}H=v~!mvdNMw?Gmyvj1@z^Hf2W~e2xu2wg4WaL`u}+{aBTvgU!kLb3o9Z=u{)Wpy8ij_}d%)ONPIq z;h$mnD;fSPhX1Oe&ocD)P5a{we>KB@&F~*IbR*bbTrKpc><=@h{&Ueii9fUXQ)nW@ z4LxW3!DXhOcMbh>L%(O}pP9J%<7l#V%=(j&UoO}VqAd*nQ$r6m^cRMn#rUx+(C_Nj zzVNtAbg|L*fuZYxKBBh_|7}D6E1@&L>wF47z|zd`|26HlGX1=1_@5d2KZd@<#Nm6! zFRK`ORYMOp^qPiV%g{p%y|$rmH}?G7*t@Nf|D%z2fj?aR(tz(`-rkvV{=6gL6aB;V zcbnm_X86|!v}>6D8b34bCj|0DD+Tt#HnV*z8+uJ6Kg`gZ82TC0|FecZ(AYEBwEvfp z-!S02cqhZ(G~g4>Z0uj$@V7DiZyWx{fqoa+{#G{fYa4pFp^rB7DTe-zv1dEOKf=&= z8UO8P_&+xKb~XI{P5YY-eTb3Y&CnAJy@#QXGx~Ni{2v-Re14YbfWZGm=NfujqyId^ zKRBRWyj?&Ot!nyTD$th@Xpzl-C!6+T19>8Oo-LDK+we~@?ZfBK@ozDPf106BH}rNU zzV>?bLLswIklHTZ6{X`n9pNn%k4kb>rwY%-}bDtC-bX2@Kt=!mxHmPk% zzf9&~-QT$V+Bk0?MxWJ3(BWM>v?}_2A=;a^sO?LKHmv+e`el7*+8H(; z*EN=&>l#OM)W>5R_oh8d^iP8ROR&vrHb&6?HEH`(Z1Y-;UE!0_^!>VQ^K1Az5O44Q zpz9duIwIPKwjb56I~ z)01x2H-bh*yp5x1pT=lfr@jZ`vKO7)HJ)~L%oqRPOg!H9z&e&@z}uq|KjB}U#C86`L!)*Li{7@*VvNIi-*xYjq!9wP1_8n zEgNgnKim+y4%@j0#=(8?-*p&wbNO+%5yt<&ld&|8jzimxy7s11q62CBLn zPB$U;Phg+_F5-F$wtd^$D0&v#ew*Lt$3%P5+qLnuF7~@UG4`K>{l9bj(VckuZDewO znCZI!zT6u7(+G^48}U{CZ83jL#CUto?SXqdV`vaz|3zFE#hKv!(6=A-Jq>-oPxR%t zZy4=X8;*I2W4JTT7I7SRr{}Oux3KT=W)7Z_@ylSkzcGaVj=AOsm``?1{56u^!1x}A z{b=plU^+YSul23WgG*vQxYX~z&nNT1DEe7_H2n>4Ime0r_ohQ(>+`iCbUpF@yE)cv zk{CZ1vk$S~`|)-m<_njMH#TtyzH>2*egfT(!OuKL%>?~R;cMczW32xKbHpibEIovN z*f%d?zc>f}SpwU+Km79o=GDdO`_pgI?Hz*eatx!<(D!m(;vOgd5qs~zJUuBIOLN0l z5%kT3efLDzeYv^|_PpJ-9=$<7Lb)ZqNyF(n=y|(q6tFRLT8;O+eduV6?YFxofY0M> zJ&bvtkGa2H>E*7`v?KQ6>98+M4?DL=$EIxWbxvZ(DeiIS`AXn759HZw`Qs-$vYbjE2&iwM~*( z@czc*Pv%qBF@{#ce6noUp7dfeukJ%EU=oIw9T+`{$ssF8F={w+XEWy+4JGYx}YJ3dZK& z<1zU4kNvtAZIAKB`_rlTTA)0)p#EKtM!L=DzqC1R0Ua-5E`AjI=33Y<2gal6DfG|B zwnO1l`CVu|_;C~38M2q7zrXk~`-LB4E7nHSc*Nye%!jj}-LL(eR!fh^++J+ud4>lFC#WInIJ z*qWtlFB*rr@^IL3IQ;l+>~|a0HitdK=psKpehr)7#Q3~D+Liu{#vvGb7=G>Q|>oy$sc&yI}TUN$ceaVmY)k((s3EnT? z#v9nzz_&bB&qdo^;DcAXxb0{<3uEfd+88<(ei(^)c0@Fe4r}a1hhck;MEi>{kI0xG zMyI3weK@bWKJlT%fA(bG*q4@ruP+0Cl8@b|KJLS?o$JEJeX$?Sjkx~-@j4lL{{|mV zhc7c+gE{%dWS<&F7v_!w`TdFa<9GbF%>W;5jWNeD=l${9=`nE_-QE~Z2O!pudK>0R z&W%RViWsjmW84iutT@IE97D!N`_Z6uU*_>Lj7H#G=K!1&J&EzKG{(c}I94BxxBvf$ zW7ZMikH_}ykK@W!$-acSEZK+HmVIew_;+9QeP5UCBRt+`!aRKdwtHV32M2gY}8{tkDY}*e|2ofc<6hx#xEr{){pIu|H#M%P+9~TVeir2KL0T z=Q#L(9$Z6gh4F-Ez5FtQE`o2rTi=y_hq-(xj%`n1J7%K2Xm6Z9Zrs?1HdfK3EWcjw z>H2tb*Q54&t9RFbi_r79-dY9MTNB+{v>DFZmUe5?aJLQ(kJhDSaQ*dfTAv2Hp>$ug z0sYWzNCRjiy2@=#TVvi|%WX$nKMKMdz}_eQ(pS?N7!W*mP8A_f~^ zY>dIV?L8P9zr(TaGmNb-u+N->vEkeV`W)8==SKU}?%2mK$2g&2{0GjTJmGq`$S-2j zhVT957tkI_x&?RkN)UMilM zpVg#UP8=JFCE3(;Yo|y&l@8s64$8Hs22YvxOnut0V7s^`wB1POjZty7u=_dVYck?F$)wtYc!}w}sa8V?oyznep2~#nyMaTYfYLob085D} z(>-NUX$Lv+Ts39eippqd`9<1GcHf|sQwoamQdTKgFea5sD+Q4h$cc2nd!*96t>m@$ zw(S(jO9umSC3^>|7x*h$eroY%)StxKA1j_B#Th%kCJUW&;4JuiI8(hE<-bvm?;`pG zS4K>Xsg4b}y**08<_T@$cA z0NG`*^3~g!kpDKYqoIFRtj7Rb2YsA|a!r&oL+)3ozmIYTlz4oWE@d9pXVDgF>0Xqx zqMQ$U?#H$q2R-|uemU@Oq1_Ike*yVfAj7{dqKb1%qGhrE1pFP*W@gk;;4GZz_IrsI zni1tiMB_e==rlYQ@xXyG-F`WkSD-Jf6R(8KRj_O8n8u;ZFSCw`X`$OiA6$W7gr!fI zK@)*TY(RHQ-O3UV;@}CB9*sd3gry{)Y8#@b18RE7og4 z-vg&_gmqTXcVnZj#5xBW565=#U8GIX_O~c6$9gW-gzDAh#9PebMGj_wwhW>xR zp8jYz57vKhe~>>5D?i0>Gw6Zn^E4h8*V$b1hakG0FO-M3(y z*TOmn*3Y2l1*{Vw^Jq6~gSQPlUI$@)A9^-KIWO>wP(Fx$=0JHHbUz85+hgsIwF~Pz z(ES&z^P>$u?M^3RJr?rQVch`GdC+kH*2l1I55pICqkb9IW3ave>}g>9H1GLnZ`Y5& z+XQ+>Lg&dS7eqf>V_gmF9iWF`ofCcBfVRAkJdASv>G1R~>~hl+U4#0;sGo-VFHkOr z^$={skFYL-wT|^U@CIO=9qT~IZ-eq|$lZwYRPcAjx;oZxVSNwSy;#@7I+j=l8Jw5!=nn45 +import { onMount } from 'svelte'; +import { MarkdownStreamParser, type StreamingChunk } from '../../../../src/tree-sitter-markdown-stream-parser.js'; + +type ExampleFile = { base: string; json: string; txt: string }; + +MarkdownStreamParser.configureWasmPath('/tree-sitter-markdown.wasm'); + +let examples: ExampleFile[] = []; +let selectedExample: ExampleFile | null = null; +let delay = 80; +let streaming = false; +let tokens: string[] = []; +let txtContent = ''; +let jsonContent = ''; +let parsedSegments: any[] = []; +let parsedBlocks: any[][] = []; +let currentToken = ''; +let currentParsedChunk: any = null; +let error = ''; +let jsonItems: string[] = []; // Add state for parsed JSON items +let currentTokenIndex: number | null = null; // Track the index of the current token +let parserInitialized = false; + +async function loadExamples() { + try { + const res = await fetch('/llm-examples-manifest.json'); + examples = await res.json(); + selectedExample = examples[0] ?? null; + } catch (e) { + error = 'Failed to load examples manifest.'; + } +} + +async function loadSelectedFiles() { + if (!selectedExample) return; + try { + const [jsonRes, txtRes] = await Promise.all([ + fetch(selectedExample.json), + fetch(selectedExample.txt), + ]); + tokens = await jsonRes.json(); + txtContent = await txtRes.text(); + // Fetch raw JSON as text for display + const rawJsonRes = await fetch(selectedExample.json); + jsonContent = await rawJsonRes.text(); + } catch (e) { + error = 'Failed to load example files.'; + } +} + +function handleExampleChange() { + parsedSegments = []; + currentToken = ''; + currentParsedChunk = null; + currentTokenIndex = null; // Reset index on example change +} + +async function simulateStream() { + if (!selectedExample) return; + streaming = true; + parsedSegments = []; + currentToken = ''; + currentParsedChunk = null; + currentTokenIndex = null; // Reset index before starting + error = ''; + + const parserId = 'demo-' + Date.now(); + try { + // getInstance returns a Promise, so we need to await it + const parser = await MarkdownStreamParser.getInstance(parserId); + parser.startParsing(); + + // Subscribe to token parsing with proper typing + const unsub = parser.subscribeToTokenParse((parsed: StreamingChunk, unsubscribe: () => void) => { + if (parsed.status === 'END_STREAM') { + currentParsedChunk = parsed; + parsedSegments = [...parsedSegments, parsed]; + unsubscribe(); + MarkdownStreamParser.removeInstance(parserId); + streaming = false; + currentTokenIndex = null; + currentToken = ''; + } else if (parsed.status === 'START_STREAM') { + // Handle stream start if needed + currentParsedChunk = parsed; + parsedSegments = [...parsedSegments, parsed]; + } else if (parsed.status === 'STREAMING') { + parsedSegments = [...parsedSegments, parsed]; + currentParsedChunk = parsed; + } + }); + + for (let i = 0; i < tokens.length; i++) { + if (!streaming) { + currentTokenIndex = null; + currentToken = ''; + break; + } + currentTokenIndex = i; + currentToken = tokens[i]; + + // parseToken might return an error + const parseError = parser.parseToken(tokens[i]); + if (parseError) { + console.error('Parse error:', parseError); + error = `Parse error: ${parseError.message}`; + break; + } + + await new Promise((r) => setTimeout(r, delay)); + } + + parser.stopParsing(); + streaming = false; + + if (currentTokenIndex !== null) { + currentTokenIndex = null; + currentToken = ''; + } + } catch (e) { + console.error('Failed to initialize parser:', e); + error = `Failed to initialize parser: ${e}`; + streaming = false; + currentTokenIndex = null; + currentToken = ''; + } +} + +$: parsedBlocks = (() => { + const blocks: StreamingChunk[][] = []; + let currentBlock: StreamingChunk[] = []; + + for (const seg of parsedSegments) { + // Skip status messages in block grouping + if (seg.status === 'START_STREAM' || seg.status === 'END_STREAM') { + continue; + } + + if (seg.segment?.isBlockDefining && currentBlock.length) { + blocks.push(currentBlock); + currentBlock = []; + } + currentBlock.push(seg); + } + + if (currentBlock.length) blocks.push(currentBlock); + return blocks; +})(); + +onMount(async () => { + // Initialize the parser early to load WASM + try { + const tempParser = await MarkdownStreamParser.getInstance('init'); + MarkdownStreamParser.removeInstance('init'); + parserInitialized = true; + } catch (e) { + console.error('Failed to initialize parser:', e); + error = 'Failed to load parser. Please check WASM file path.'; + } + + await loadExamples(); + await loadSelectedFiles(); +}); + +$: if (selectedExample) { + loadSelectedFiles(); +} + +$: { // Reactive block to parse jsonContent when it changes + if (jsonContent) { + try { + const parsed = JSON.parse(jsonContent); + if (Array.isArray(parsed)) { + jsonItems = parsed; + } else { + console.error("Parsed jsonContent is not an array:", parsed); + jsonItems = []; // Reset or handle as appropriate + } + } catch (e) { + console.error("Failed to parse jsonContent:", e); + jsonItems = []; // Reset on error + } + } else { + jsonItems = []; + } +} + + +
+
+

@lixpi/markdown-stream-parser demo

+

This is just a quick and dirty showcase of the @lixpi/markdown-stream-parser, the parser itself has nothing to do with rendering !!! Please keep that in mind...

+

This demo is entirely `vibe-coded`, while the parser is painstakingly created by a human being 👩‍💻 :)

+
+ +
+ +
+ + +
+ + {#if error} + {error} + {/if} +
+ +
+ +
+

Parsed Stream

+
+ {#each parsedBlocks as block} +
+ {#each block as seg} + {#if seg.segment?.type === 'header'} + {#if seg.segment?.level === 1} +

{seg.segment?.segment}

+ {:else if seg.segment?.level === 2} +

{seg.segment?.segment}

+ {:else if seg.segment?.level === 3} +

{seg.segment?.segment}

+ {:else} + {seg.segment?.segment} + {/if} + {:else if seg.segment?.type === 'codeBlock'} + 
{seg.segment?.segment}
+ {:else if seg.segment?.type === 'blockQuote'} + {seg.segment?.segment} + {:else} + + {#if seg.segment?.styles?.length} + + {#if seg.segment.styles.includes('strikethrough')} + {seg.segment?.segment} + {:else if seg.segment.styles.includes('code')} + {seg.segment?.segment} + {:else} + {seg.segment?.segment} + {/if} + + {:else} + {seg.segment?.segment} + {/if} + + {/if} + {/each} +
+ {/each} +
+
+ + +
+
+

Current Token

+ 
{JSON.stringify(currentToken, null, 2)}
+
+
+

Parsed Chunk

+ {#if currentParsedChunk} + 
{JSON.stringify(currentParsedChunk, null, 2)}
+ {/if} +
+
+ + +
+

Raw array of streamed tokens

+
+ {#each jsonItems as item, index} + +
+ {JSON.stringify(item)} +
+ {/each} +
+
+ + +
+

Concatenated raw LLM output

+ 
{txtContent}
+
+
+
+ + diff --git a/demo/svelte-demo/src/routes/+page.svelte b/demo/svelte-demo/src/routes/+page.svelte index 98a345a..4e5c772 100644 --- a/demo/svelte-demo/src/routes/+page.svelte +++ b/demo/svelte-demo/src/routes/+page.svelte @@ -1,9 +1,11 @@ - -
-
-

@lixpi/markdown-stream-parser demo

-

This is just a quick and dirty showcase of the @lixpi/markdown-stream-parser, the parser itself has nothing to do with rendering !!! Please keep that in mind...

-

This demo is entirely `vibe-coded`, while the parser is painstakingly created by a human being 👩‍💻 :)

-
- -
- -
- - -
- - {#if error} - {error} - {/if} -
- -
- -
-

Parsed Stream

-
- {#each parsedBlocks as block} -
- {#each block as seg} - {#if seg.segment?.type === 'header'} - {#if seg.segment?.level === 1} -

{seg.segment?.segment}

- {:else if seg.segment?.level === 2} -

{seg.segment?.segment}

- {:else if seg.segment?.level === 3} -

{seg.segment?.segment}

- {:else} - {seg.segment?.segment} - {/if} - {:else if seg.segment?.type === 'codeBlock'} - 
{seg.segment?.segment}
- {:else if seg.segment?.type === 'blockQuote'} - {seg.segment?.segment} - {:else} - - {#if seg.segment?.styles?.length} - - {#if seg.segment.styles.includes('strikethrough')} - {seg.segment?.segment} - {:else if seg.segment.styles.includes('code')} - {seg.segment?.segment} - {:else} - {seg.segment?.segment} - {/if} - - {:else} - {seg.segment?.segment} - {/if} - - {/if} - {/each} -
- {/each} -
-
- - -
-
-

Current Token

- 
{JSON.stringify(currentToken, null, 2)}
-
-
-

Parsed Chunk

- {#if currentParsedChunk} - 
{JSON.stringify(currentParsedChunk, null, 2)}
- {/if} -
-
- - -
-

Raw array of streamed tokens

-
- {#each jsonItems as item, index} - -
- {JSON.stringify(item)} -
- {/each} -
-
- - -
-

Concatenated raw LLM output

- 
{txtContent}
-
-
-
- - From 72c0159deb1c19a2e8872babe60faa1f678ddcf1 Mon Sep 17 00:00:00 2001 From: Dmitry Bondarenko Date: Tue, 30 Dec 2025 22:05:44 +0600 Subject: [PATCH 04/32] Almost matched the old functionality, strikethrough remains --- .gitignore | 2 + COMPARISON_AND_GAPS.md | 473 +++++ ROADMAP_SUMMARY.md | 331 ++++ STATE_MACHINE_FEATURES.md | 440 +++++ TREE_SITTER_INLINE_CODE_SUMMARY.md | 119 ++ TREE_SITTER_MIGRATION_ROADMAP.md | 351 ++++ debug-italic.cjs | 41 + demo/debug-header-stripping.ts | 68 + demo/reproducing_bold.ts | 18 + demo/svelte-demo/src/routes/+page.svelte | 940 ++++++---- demo/svelte-demo/static/grammar.js | 474 +++++ .../static/tree-sitter-markdown-inline.wasm | Bin 0 -> 379177 bytes demo/svelte-demo/static/tree-sitter.json | 3 + demo/svelte-demo/static/web-tree-sitter.wasm | Bin 0 -> 195953 bytes demo/svelte-demo/vite.config.ts | 4 + demo/test-bold-tree-sitter.ts | 32 + demo/test-header-bold.ts | 34 + demo/test-incremental.ts | 59 + demo/test-list-tree.ts | 31 + demo/test-paren-backtick.ts | 39 + demo/test-real-stream.ts | 57 + demo/test-table.ts | 33 + demo/test-trace-backtick.ts | 59 + reproduce-italic-bug.test.ts | 66 + ...tree-sitter-markdown-stream-parser.test.ts | 364 ++++ src/tree-sitter-markdown-stream-parser.ts | 1571 ++++++++++++++--- test-cat-coding-regression.test.ts | 58 + tree-sitter-inline-build.json | 8 + verify-italic-buffer.test.ts | 62 + verify-italic.test.ts | 79 + 30 files changed, 5271 insertions(+), 545 deletions(-) create mode 100644 COMPARISON_AND_GAPS.md create mode 100644 ROADMAP_SUMMARY.md create mode 100644 STATE_MACHINE_FEATURES.md create mode 100644 TREE_SITTER_INLINE_CODE_SUMMARY.md create mode 100644 TREE_SITTER_MIGRATION_ROADMAP.md create mode 100644 debug-italic.cjs create mode 100644 demo/debug-header-stripping.ts create mode 100644 demo/reproducing_bold.ts create mode 100644 demo/svelte-demo/static/grammar.js create mode 100755 demo/svelte-demo/static/tree-sitter-markdown-inline.wasm create mode 100644 demo/svelte-demo/static/tree-sitter.json create mode 100755 demo/svelte-demo/static/web-tree-sitter.wasm create mode 100644 demo/test-bold-tree-sitter.ts create mode 100644 demo/test-header-bold.ts create mode 100644 demo/test-incremental.ts create mode 100644 demo/test-list-tree.ts create mode 100644 demo/test-paren-backtick.ts create mode 100644 demo/test-real-stream.ts create mode 100644 demo/test-table.ts create mode 100644 demo/test-trace-backtick.ts create mode 100644 reproduce-italic-bug.test.ts create mode 100644 src/tree-sitter-markdown-stream-parser.test.ts create mode 100644 test-cat-coding-regression.test.ts create mode 100644 tree-sitter-inline-build.json create mode 100644 verify-italic-buffer.test.ts create mode 100644 verify-italic.test.ts diff --git a/.gitignore b/.gitignore index 6232ac9..5f363d3 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ build/ # Node modules node_modules/ .pnpm-store/ +.npm-cache/ +package-lock.json # Logs npm-debug.log* diff --git a/COMPARISON_AND_GAPS.md b/COMPARISON_AND_GAPS.md new file mode 100644 index 0000000..c7badaa --- /dev/null +++ b/COMPARISON_AND_GAPS.md @@ -0,0 +1,473 @@ +# Tree-Sitter vs State-Machine Parser: Feature Comparison + +## Quick Reference + +| Feature | State Machine | Tree-Sitter | Status | Priority | +|---------|---------------|-------------|---------|----------| +| **Block Types** | +| Headers (1-6) | ✅ Full | ⚠️ Partial | Levels work, content includes markers | 🔴 HIGH | +| Paragraphs | ✅ Full | ✅ Full | Working | ✅ DONE | +| Code Blocks | ✅ Full | ⚠️ Partial | Missing language extraction | 🔴 HIGH | +| **Inline Styles** | +| Bold (`**`) | ✅ Full | ⚠️ Detected | No segment splitting | 🔴 HIGH | +| Italic (`*`) | ✅ Full | ⚠️ Detected | No segment splitting | 🔴 HIGH | +| Bold-Italic (`***`) | ✅ Full | ⚠️ Detected | No segment splitting | 🔴 HIGH | +| Strikethrough (`~~`) | ✅ Full | ⚠️ Detected | No segment splitting | 🟡 MED | +| Inline Code (`` ` ``) | ✅ Full | ⚠️ Detected | Wrong style name | 🔴 HIGH | +| **Output Features** | +| Segment Splitting | ✅ Full | ❌ None | Major gap | 🔴 HIGH | +| Prefixed Content | ✅ Full | ❌ None | Part of splitting | 🔴 HIGH | +| Postfixed Content | ✅ Full | ❌ None | Part of splitting | 🔴 HIGH | +| isBlockDefining | ✅ Smart | ⚠️ Basic | Too simplistic | 🟡 MED | +| isProcessingNewLine | ✅ Full | ✅ Full | Working | ✅ DONE | +| **Naming Consistency** | +| Block types | camelCase | snake_case | Inconsistent | 🟡 MED | +| Style names | Match spec | Wrong names | `'code'` vs `'inline_code'` | 🔴 HIGH | + +## Detailed Comparison + +### 1. Headers + +#### State Machine +```typescript +Input: "## Header Text\n" +Output: { + segment: "Header Text", // ✅ Markers stripped + type: "header", + level: 2, + styles: [], + isBlockDefining: true, + isProcessingNewLine: true +} +``` + +#### Tree-Sitter (Current) +```typescript +Input: "## Header Text\n" +Output: { + segment: "## Header Text\n", // ❌ Markers included + type: "header", + level: 2, + styles: [], + isBlockDefining: true, + isProcessingNewLine: true +} +``` + +**Issues**: +- ❌ Content includes `##` markers +- ❌ May include trailing newline + +**Fix Required**: Extract only text content from heading nodes + +--- + +### 2. Code Blocks + +#### State Machine +```typescript +Input: "```javascript\ncode\n```\n" +Output: { + segment: "code", + type: "codeBlock", // ✅ camelCase + language: "javascript", // ✅ Language extracted + styles: [], + isBlockDefining: true, + isProcessingNewLine: true +} +``` + +#### Tree-Sitter (Current) +```typescript +Input: "```javascript\ncode\n```\n" +Output: { + segment: "code", + type: "code_block", // ❌ snake_case + // ❌ Missing language field + styles: [], + isBlockDefining: true, + isProcessingNewLine: true +} +``` + +**Issues**: +- ❌ Wrong naming convention (`code_block` vs `codeBlock`) +- ❌ No language field +- ❌ Language not extracted from info_string + +**Fix Required**: +1. Use camelCase naming +2. Parse info_string node for language + +--- + +### 3. Inline Styles: Bold + +#### State Machine +```typescript +Input: "before**bold**after" +Output: [ + { + segment: "before", // ✅ Prefixed content + type: "paragraph", + styles: [], + isBlockDefining: true, + isProcessingNewLine: false + }, + { + segment: "bold", // ✅ Styled content only + type: "paragraph", + styles: ["bold"], // ✅ Style applied + isBlockDefining: false, + isProcessingNewLine: false + }, + { + segment: "after", // ✅ Postfixed content + type: "paragraph", + styles: [], + isBlockDefining: false, + isProcessingNewLine: false + } +] +``` + +#### Tree-Sitter (Current) +```typescript +Input: "before**bold**after" +Output: [ + { + segment: "before**bold**after", // ❌ No splitting + type: "paragraph", + styles: ["bold"], // ⚠️ Style detected but... + isBlockDefining: true, + isProcessingNewLine: false + } +] +``` + +**Issues**: +- ❌ No segment splitting at style boundaries +- ❌ Entire content emitted as one chunk +- ❌ Style markers (`**`) included in output +- ❌ Prefixed/postfixed content not separated + +**Fix Required**: Implement segment boundary detection and splitting + +--- + +### 4. Inline Code Style + +#### State Machine +```typescript +Input: "Run `npm install` now" +Output: [ + { segment: "Run ", styles: [] }, + { segment: "npm install", styles: ["code"] }, // ✅ 'code' + { segment: " now", styles: [] } +] +``` + +#### Tree-Sitter (Current) +```typescript +Input: "Run `npm install` now" +Output: [ + { + segment: "Run `npm install` now", + styles: ["inline_code"] // ❌ Wrong name + } +] +``` + +**Issues**: +- ❌ Style name is `'inline_code'` should be `'code'` +- ❌ No segment splitting (same as bold) + +**Fix Required**: +1. Map `code_span` node to `'code'` style +2. Implement splitting + +--- + +### 5. isBlockDefining Flag + +#### State Machine Logic +```typescript +isBlockDefining = true when: + - Block type changes (paragraph → header) + - New line in routing state + - First segment of new paragraph after code block + - Transitioning from any block to different block type + +isBlockDefining = false when: + - Continuing within same block + - Not at block boundary + - Subsequent segments of styled content +``` + +#### Tree-Sitter (Current) +```typescript +isBlockDefining = this.isNewBlock(blockInfo, nodeAtPosition) + +isNewBlock(): + - Returns true if currentBlock is null + - Returns true if block type changed + - Returns true if level changed (headers) + - Returns true if node starts after last segment + - Otherwise false +``` + +**Issues**: +- ⚠️ Doesn't account for content position within blocks +- ⚠️ May incorrectly mark continued content as new block +- ⚠️ Doesn't properly handle inline style boundaries + +**Fix Required**: More sophisticated logic considering: +- Previous segment position +- Style boundaries +- Newline positions +- Block node boundaries + +--- + +## Critical Gaps Ranked + +### 🔴 HIGH Priority (Blocks Basic Functionality) + +1. **Segment Splitting for Inline Styles** (Biggest Gap) + - State machine: 3 segments for `before**bold**after` + - Tree-sitter: 1 segment + - Impact: Cannot properly render styled content + - Complexity: High - need boundary detection + +2. **Header Content Extraction** + - State machine: `"Header Text"` + - Tree-sitter: `"## Header Text"` + - Impact: Headers display with markers + - Complexity: Low - strip leading `#` + +3. **Code Block Language** + - State machine: `language: "javascript"` + - Tree-sitter: No language field + - Impact: Cannot syntax highlight code + - Complexity: Medium - parse info_string + +4. **Style Name Mapping** + - State machine: `'code'` + - Tree-sitter: `'inline_code'` + - Impact: UI won't recognize styles + - Complexity: Low - fix mapping + +### 🟡 MEDIUM Priority (Quality of Life) + +5. **Naming Consistency** + - State machine: `'codeBlock'` + - Tree-sitter: `'code_block'` + - Impact: API inconsistency + - Complexity: Low - rename + +6. **isBlockDefining Logic** + - Impact: May affect UI block rendering + - Complexity: Medium - refine conditions + +7. **Strikethrough Support** + - Less commonly used + - Same splitting issue as other styles + +### 🟢 LOW Priority (Nice to Have) + +8. **Better Error Handling** +9. **Performance Optimization** +10. **Extended Markdown Features** + +--- + +## Implementation Order + +### Phase 1: Critical Fixes (Must Have) +These are required for basic functionality parity: + +1. ✅ Fix style names (`'code'` not `'inline_code'`) +2. ✅ Fix block type names (`'codeBlock'` not `'code_block'`) +3. ✅ Strip header markers from content +4. ✅ Extract code block language + +**Estimated Effort**: 2-3 hours +**Test With**: Basic examples, visual check in svelte-demo + +### Phase 2: Segment Splitting (Core Feature) +This is the most complex but essential feature: + +1. 🔧 Detect inline style boundaries in parsed content +2. 🔧 Split content at style marker positions +3. 🔧 Emit prefixed content as separate segment +4. 🔧 Emit styled content with styles array +5. 🔧 Emit postfixed content as separate segment +6. 🔧 Handle multiple styles in same content +7. 🔧 Handle nested/adjacent styles + +**Estimated Effort**: 6-8 hours +**Test With**: Inline style tests, complex nested examples + +### Phase 3: Refinements +Polish and edge cases: + +1. 🔧 Improve isBlockDefining logic +2. 🔧 Handle edge cases (empty segments, etc.) +3. 🔧 Performance optimization +4. 🔧 Comprehensive test coverage + +**Estimated Effort**: 3-4 hours +**Test With**: All state-machine tests, LLM examples + +--- + +## Testing Strategy + +### Test Migration Plan + +#### Step 1: Port Basic Tests +From `markdown-state-machine.test.ts`: +- Simple paragraph +- Headers (all levels) +- Code blocks + +**Expected**: These should mostly pass after Phase 1 + +#### Step 2: Port Style Tests +- Bold, italic, bold-italic +- Strikethrough, inline code +- Mixed styles + +**Expected**: Will fail until Phase 2 complete + +#### Step 3: Port Complex Tests +- Prefixed/postfixed content +- Nested styles +- State transitions +- Edge cases + +**Expected**: Should pass after Phase 3 + +### Visual Testing with svelte-demo + +After each phase: +1. Run svelte-demo +2. Load test examples +3. Compare visual output +4. Check that styled segments render correctly + +Key examples to test: +- `claude-3.7-history-of-cats.json` +- `gpt-4.o-happy-number-5-programs.json` +- `claude-3.7-markdown-with-nested-code-block.json` + +--- + +## Performance Comparison + +### State Machine +- ✅ Incremental processing +- ✅ Minimal memory (no AST) +- ✅ Regex-based (fast) +- ❌ Complex state management +- ❌ Hard to extend + +### Tree-Sitter +- ✅ Proper AST parsing +- ✅ Robust structure +- ✅ Easy to extend +- ⚠️ Re-parses on each chunk +- ⚠️ Full content buffer +- ⚠️ More memory usage + +**Expected**: Tree-sitter may be slightly slower but should be acceptable for streaming use case. + +--- + +## Success Metrics + +### Minimum Viable (Phase 1) +- [ ] All block types correctly identified +- [ ] Header content without markers +- [ ] Code blocks include language +- [ ] Style names match specification +- [ ] Naming consistency (camelCase) + +### Feature Complete (Phase 2) +- [ ] Segment splitting works +- [ ] Prefixed/postfixed content separate +- [ ] All inline styles properly applied +- [ ] Multiple styles in same content +- [ ] Matches state-machine output structure + +### Production Ready (Phase 3) +- [ ] All state-machine tests pass +- [ ] Visual parity in svelte-demo +- [ ] Performance acceptable +- [ ] Edge cases handled +- [ ] Documentation complete + +--- + +## Migration Risks + +### High Risk +1. **Segment Splitting Complexity** + - Most complex feature to implement + - Risk of bugs with edge cases + - May need multiple iterations + +2. **Performance Degradation** + - Re-parsing on each chunk could be slow + - May need optimization + +### Medium Risk +3. **Style Detection Accuracy** + - Tree-sitter node types may not map perfectly + - Nested styles could be tricky + +4. **Block Boundary Detection** + - Different parsing model may cause issues + - Need careful testing + +### Low Risk +5. **Naming/Mapping Issues** + - Easy to fix + - Quick testing cycle + +--- + +## Rollback Plan + +If tree-sitter implementation fails to reach parity: + +1. **Keep both implementations** + - Export both parsers + - Let users choose + - Document trade-offs + +2. **Feature flag approach** + - Use tree-sitter where it works + - Fall back to state-machine for complex cases + +3. **Hybrid approach** + - Use tree-sitter for structure detection + - Use state-machine for streaming/splitting + +--- + +## Conclusion + +**Current State**: Tree-sitter can detect structures but cannot match state-machine output format + +**Biggest Gap**: Segment splitting for inline styles (critical for proper rendering) + +**Estimated Total Effort**: 11-15 hours to reach full parity + +**Recommendation**: +1. Start with Phase 1 quick wins (2-3 hours) +2. Validate approach with svelte-demo +3. Proceed to Phase 2 if Phase 1 successful +4. Keep state-machine as fallback during migration + +The roadmap is achievable, but segment splitting will require careful implementation and thorough testing. diff --git a/ROADMAP_SUMMARY.md b/ROADMAP_SUMMARY.md new file mode 100644 index 0000000..1d6644f --- /dev/null +++ b/ROADMAP_SUMMARY.md @@ -0,0 +1,331 @@ +# Roadmap Summary: Tree-Sitter Parser Migration + +## Executive Summary + +I've completed a comprehensive analysis of both the state-machine and tree-sitter markdown parser implementations. The tree-sitter version is structurally sound but missing several critical features needed to match the state-machine parser's output format. + +## What I've Created + +### 1. **STATE_MACHINE_FEATURES.md** +Complete documentation of the old parser's capabilities: +- All supported block types (headers 1-6, paragraphs, code blocks) +- All inline styles (bold, italic, bold-italic, strikethrough, inline code) +- Segment splitting behavior +- Output structure and flags +- API usage patterns +- 70+ test cases documented + +### 2. **TREE_SITTER_MIGRATION_ROADMAP.md** +Detailed implementation plan: +- Phase-by-phase breakdown +- Expected output formats with examples +- Success criteria +- Testing strategy +- Risk areas identified + +### 3. **COMPARISON_AND_GAPS.md** +Side-by-side comparison highlighting: +- Feature-by-feature status +- Specific issues with code examples +- Priority rankings (HIGH/MED/LOW) +- Implementation order +- Estimated effort (11-15 hours total) + +## Key Findings + +### What the State-Machine Parser Supports ✅ + +**Block Elements:** +- Headers (1-6 levels) with level detection +- Paragraphs with inline styles +- Code blocks with language extraction + +**Inline Styles:** +- Bold (`**text**`) +- Italic (`*text*`) +- Bold-Italic (`***text***`) +- Strikethrough (`~~text~~`) +- Inline code (`` `text` ``) + +**Critical Features:** +- **Segment Splitting**: `before**bold**after` → 3 separate segments +- **Prefixed/Postfixed Content**: Emitted as separate segments without styles +- **Clean Output**: Header markers stripped, styled content isolated + +### What Tree-Sitter Currently Lacks ❌ + +1. **🔴 CRITICAL: No Segment Splitting** + - Everything emitted as one chunk + - Cannot properly render styled content + - Biggest implementation challenge + +2. **🔴 HIGH: Header Content Includes Markers** + - Outputs `"## Header"` instead of `"Header"` + - Easy fix but critical for display + +3. **🔴 HIGH: Missing Code Block Language** + - No `language` field in output + - Cannot syntax highlight + +4. **🔴 HIGH: Wrong Style Names** + - Uses `'inline_code'` instead of `'code'` + - UI won't recognize styles + +5. **🟡 MEDIUM: Naming Inconsistency** + - Uses `'code_block'` instead of `'codeBlock'` + - API inconsistency + +## Implementation Phases + +### Phase 1: Quick Wins (2-3 hours) 🎯 +**Goal**: Basic functionality parity + +- Fix style name mapping (`'code'` not `'inline_code'`) +- Fix block type naming (`'codeBlock'` not `'code_block'`) +- Strip header markers from output +- Extract code block language + +**Test**: Basic examples should render correctly in svelte-demo + +### Phase 2: Segment Splitting (6-8 hours) 🔥 +**Goal**: Match state-machine output structure + +- Detect inline style boundaries +- Split content at style markers +- Emit prefixed/styled/postfixed segments separately +- Handle multiple and nested styles + +**Test**: All inline style examples should work + +### Phase 3: Refinements (3-4 hours) ✨ +**Goal**: Production ready + +- Improve `isBlockDefining` logic +- Edge case handling +- Performance optimization +- Full test coverage + +**Test**: All state-machine tests pass + +## Critical Gap: Segment Splitting Explained + +### Current Tree-Sitter Behavior ❌ +```typescript +Input: "Hello **world** test" + +Output: [ + { + segment: "Hello **world** test", // ❌ One chunk + styles: ["bold"], // ⚠️ Style detected but applied to all + type: "paragraph" + } +] +``` + +### Required State-Machine Behavior ✅ +```typescript +Input: "Hello **world** test" + +Output: [ + { + segment: "Hello ", // ✅ Prefixed content + styles: [], + type: "paragraph" + }, + { + segment: "world", // ✅ Styled content only + styles: ["bold"], + type: "paragraph" + }, + { + segment: " test", // ✅ Postfixed content + styles: [], + type: "paragraph" + } +] +``` + +### Why It Matters +- UI needs separate segments to apply styles correctly +- Cannot render `` around only "world" +- Current output would bold everything or nothing + +## Implementation Strategy + +### Approach for Segment Splitting + +1. **Detect Style Nodes** + - Walk tree to find `strong_emphasis`, `emphasis`, `code_span` nodes + - Record their start/end positions + +2. **Split Content** + - For each new chunk, check if it overlaps style boundaries + - Split content at style start/end positions + - Track which segment is inside vs outside styles + +3. **Emit Segments** + - Before style: emit with `styles: []` + - Inside style: emit with appropriate styles + - After style: emit with `styles: []` + +4. **Handle Nesting** + - Track style stack (entering/exiting styles) + - Combine styles for nested segments + - Example: bold inside italic → `styles: ['italic', 'bold']` + +### Example Implementation Sketch + +```typescript +private splitSegmentByStyles( + content: string, + startIndex: number, + endIndex: number, + node: Parser.SyntaxNode +): StreamingChunk[] { + const segments: StreamingChunk[] = []; + + // Find all style boundaries in range + const styleBoundaries = this.findStyleBoundaries(node, startIndex, endIndex); + + // Split content at boundaries + let currentPos = startIndex; + for (const boundary of styleBoundaries) { + // Emit content before boundary + if (currentPos < boundary.start) { + segments.push(this.createSegment( + content.substring(currentPos - startIndex, boundary.start - startIndex), + boundary.stylesBeforeStart + )); + } + + // Emit styled content + segments.push(this.createSegment( + boundary.content, + boundary.styles + )); + + currentPos = boundary.end; + } + + // Emit remaining content + if (currentPos < endIndex) { + segments.push(this.createSegment( + content.substring(currentPos - startIndex), + [] + )); + } + + return segments; +} +``` + +## Testing Plan + +### Incremental Testing + +**After Phase 1:** +```bash +# Test basic blocks +npm test -- tree-sitter-markdown-stream-parser.test.ts +# Visual check +cd demo/svelte-demo && npm run dev +``` + +**After Phase 2:** +```bash +# Test inline styles +npm test -- --grep "inline style" +# Load complex examples in svelte-demo +``` + +**After Phase 3:** +```bash +# Full test suite +npm test +# All LLM examples in svelte-demo +``` + +### Key Test Files to Use +- `claude-3.7-history-of-cats.json` - Mixed content +- `gpt-4.o-happy-number-5-programs.json` - Code blocks +- `claude-3.7-markdown-with-nested-code-block.json` - Complex nesting + +## Estimated Timeline + +| Phase | Tasks | Hours | Priority | +|-------|-------|-------|----------| +| Phase 1 | Style names, block types, header content, code language | 2-3 | 🔴 HIGH | +| Phase 2 | Segment splitting, boundary detection, emit logic | 6-8 | 🔴 HIGH | +| Phase 3 | Flag refinement, edge cases, tests, polish | 3-4 | 🟡 MED | +| **Total** | | **11-15** | | + +## Recommended Next Steps + +1. **Start with Phase 1** (quick wins) + - Get immediate visual improvements + - Validate approach + - Build confidence + +2. **Test Early and Often** + - After each fix, check svelte-demo + - Visual feedback is crucial + - Easier to debug incrementally + +3. **Phase 2 in Iterations** + - Start with simple case (single bold) + - Then prefixed/postfixed + - Then multiple styles + - Then nested styles + +4. **Keep State-Machine as Reference** + - Don't remove it yet + - Use for comparison testing + - Fallback option if needed + +## Success Criteria + +### Minimum Viable ✅ +- [ ] Headers render without `#` markers +- [ ] Code blocks show language +- [ ] Style names match API expectations +- [ ] Basic blocks work in svelte-demo + +### Feature Complete ✅ +- [ ] Segment splitting works +- [ ] All inline styles render correctly +- [ ] Prefixed/postfixed content separate +- [ ] Output structure matches state-machine + +### Production Ready ✅ +- [ ] All tests pass +- [ ] Visual parity in svelte-demo +- [ ] Performance acceptable +- [ ] Documentation complete + +## Risk Mitigation + +### High Risk: Segment Splitting Complexity +- **Mitigation**: Implement incrementally, test each step +- **Fallback**: Keep state-machine for inline styles only + +### Medium Risk: Performance +- **Mitigation**: Profile after Phase 2, optimize if needed +- **Fallback**: Cache parsed trees, batch updates + +### Low Risk: Edge Cases +- **Mitigation**: Port all state-machine tests +- **Fallback**: Document known limitations + +## Conclusion + +The tree-sitter implementation has a solid foundation but needs significant work on the output formatting side. The biggest challenge is segment splitting for inline styles, which is critical for proper rendering. + +**The roadmap is achievable with focused effort on the three phases.** + +All documentation is in place. You can now: +1. Review the roadmap documents +2. Start implementing Phase 1 (quickest wins) +3. Test with svelte-demo after each phase +4. Iterate based on visual feedback + +**Would you like me to start implementing Phase 1 now?** diff --git a/STATE_MACHINE_FEATURES.md b/STATE_MACHINE_FEATURES.md new file mode 100644 index 0000000..0c1f92e --- /dev/null +++ b/STATE_MACHINE_FEATURES.md @@ -0,0 +1,440 @@ +# State Machine Markdown Parser - Supported Features + +## Overview +This document comprehensively details all features supported by the original state-machine-based markdown parser implementation. + +## Block-Level Elements + +### 1. Headers (✅ Fully Supported) +**Syntax**: `#{1,6} content` + +**Supported Levels**: 1-6 +- `# Heading 1` → level 1 +- `## Heading 2` → level 2 +- `### Heading 3` → level 3 +- `#### Heading 4` → level 4 +- `##### Heading 5` → level 5 +- `###### Heading 6` → level 6 + +**Detection Pattern**: `/^(#{1,6}\s?)(.*)/` + +**Output Format**: +```typescript +{ + segment: "Header Content", // WITHOUT # markers + type: "header", + level: 2, // 1-6 + styles: [], + isBlockDefining: true, + isProcessingNewLine: false +} +``` + +**Key Behaviors**: +- Header markers (`#`) are stripped from output +- Only content after markers is emitted +- Space after `#` is optional but recommended +- Newline ends header processing +- Resets parser to routing state after newline + +### 2. Paragraphs (✅ Fully Supported) +**Default Block Type**: Used when no other block pattern matches + +**Output Format**: +```typescript +{ + segment: "Paragraph text", + type: "paragraph", + styles: [], // Can include inline styles + isBlockDefining: true, // true for first segment + isProcessingNewLine: false +} +``` + +**Key Behaviors**: +- Default fallback for regular text +- Can contain inline styles (bold, italic, etc.) +- Multiple segments within same paragraph have `isBlockDefining: false` +- Newline can end paragraph but doesn't always + +### 3. Code Blocks (✅ Fully Supported) +**Syntax**: ` ```language\n...content...```\n` + +**Detection Patterns**: +- Start: `/^(\s*```)([a-zA-Z_+-]*)(?:\n|\\n)([a-zA-Z_+\-\s]*)$/` +- End: `/(? { + console.log(chunk); +}); + +// Start parsing +parser.startParsing(); + +// Send chunks +parser.parseToken('## '); +parser.parseToken('Header\n'); +parser.parseToken('Some '); +parser.parseToken('**bold** '); +parser.parseToken('text.\n'); + +// Stop parsing +parser.stopParsing(); + +// Cleanup +unsubscribe(); +MarkdownStreamParser.removeInstance('unique-id'); +``` + +### Event Stream +```typescript +{ status: 'START_STREAM' } +{ status: 'STREAMING', segment: { segment: 'Header', type: 'header', level: 2, ... } } +{ status: 'STREAMING', segment: { segment: 'Some ', type: 'paragraph', ... } } +{ status: 'STREAMING', segment: { segment: 'bold', type: 'paragraph', styles: ['bold'], ... } } +{ status: 'STREAMING', segment: { segment: ' text.', type: 'paragraph', ... } } +{ status: 'END_STREAM' } +``` + +## Summary +The state-machine parser provides solid support for: +- ✅ Basic block elements (headers, paragraphs, code blocks) +- ✅ Five inline styles (bold, italic, bold-italic, strikethrough, inline code) +- ✅ Proper segment splitting with prefixed/postfixed content +- ✅ Streaming chunk processing +- ✅ State tracking and transitions + +It serves as an excellent baseline for the tree-sitter implementation to match and eventually exceed. diff --git a/TREE_SITTER_INLINE_CODE_SUMMARY.md b/TREE_SITTER_INLINE_CODE_SUMMARY.md new file mode 100644 index 0000000..4b5a7bf --- /dev/null +++ b/TREE_SITTER_INLINE_CODE_SUMMARY.md @@ -0,0 +1,119 @@ +# Tree-Sitter Inline Code Implementation Summary + +## Primary Objective +Implement tree-sitter based markdown parsing to detect and strip backticks (`) from inline code and triple backticks (```) from code blocks, while preserving the `code` style annotation. + +--- + +## Key Technical Discoveries + +### 1. tree-sitter-markdown uses two grammars +- **`tree-sitter-markdown`** (block-level): Creates `inline` placeholder nodes +- **`tree-sitter-markdown-inline`**: Parses inline content for `code_span`, `emphasis`, etc. +- Both WASM files are at `/demo/svelte-demo/static/` + +### 2. Incremental parsing requires `tree.edit()` +- Without calling `tree.edit()` before re-parsing, tree-sitter doesn't properly update the tree +- Fixed in `processRawChunk()` at lines 306-330 + +### 3. Buffering incomplete structures +- TokensStreamBuffer splits by word boundaries, breaking `` `inline code` `` into `` `inline `` and `` code` `` +- **Solution:** Buffer content with unmatched backticks until closing backtick arrives +- Uses `pendingInlineContent` and `pendingInlineStartIndex` fields + +### 4. code_span nodes have only delimiter children +- No text nodes inside code_span - must extract content using byte ranges between delimiters +- `contentStart = delimiters[0].endIndex`, `contentEnd = delimiters[last].startIndex` + +### 5. Style detection must use overlap, not position +- Changed from `findActiveNodeAtPosition(startPos)` to checking all `code_spans` for overlap with range +- Fixed in `detectActiveStyles()` around lines 672-710 + +### 6. Inline tree nodes have 0-based positions +- When `findActiveNodeAtPosition` returns a node from the inline tree, its positions are relative to the inline content, not the document +- Must track whether we're in an inline tree node to avoid incorrect position calculations +- Bug was causing `relative range: 537-543` when inline content was only 76 chars + +--- + +## Current State + +### Working ✅ +- Code blocks strip ``` fences correctly +- Single-word inline code (`` `word` ``) strips backticks +- Multi-word inline code when complete (`` `inline code` ``) strips backticks +- Buffering waits for closing backtick before emitting +- Surrounding text preserved: `` "(`re` " `` → `"(re "` with code style + +### Remaining Issues (4 segments with backticks) +All 4 are inside **markdown tables** (`pipe_table`): +- `` `\bword\b` `` +- `` `.*` `` +- `` `\d` `` +- `` `[A-Za-z]` `` + +### Root Cause +`findInlineNodeAtPosition()` only finds `inline` nodes, but table cells use `pipe_table_cell` type. Need to also parse `pipe_table_cell` content with the inline parser. + +--- + +## Key Code Locations + +| Function | Location | Purpose | +|----------|----------|---------| +| `processRawChunk()` | ~line 291 | Adds chunks, calls tree.edit(), parses | +| `generateSegments()` | ~line 343 | Main segment generation with buffering logic | +| `detectActiveStyles()` | ~line 653 | Detects code/bold/italic styles via inline parser | +| `getInlineCodeContent()` | ~line 823 | Strips backticks, preserves surrounding text | +| `findInlineNodeAtPosition()` | ~line 567 | Finds inline node at position (needs table support) | +| `hasCompleteCodeSpanAt()` | ~line 556 | Checks if code_span covers a range | + +--- + +## Immediate Next Step + +Update `findInlineNodeAtPosition()` to also find `pipe_table_cell` nodes: + +```typescript +private findInlineNodeAtPosition(node: Parser.SyntaxNode, position: number): Parser.SyntaxNode | null { + // If this node is an inline or pipe_table_cell node that contains the position, return it + if ((node.type === 'inline' || node.type === 'pipe_table_cell') && + position >= node.startIndex && position < node.endIndex) { + return node; + } + // ... rest unchanged +} +``` + +Also update `detectActiveStyles()` to handle `pipe_table_cell` the same as `inline`. + +--- + +## Debug Files Available + +| File | Purpose | +|------|---------| +| `/demo/test-real-stream.ts` | Tests with actual gpt-4.5-cat-coding.json | +| `/demo/test-trace-backtick.ts` | Traces specific backtick sequences | +| `/demo/test-paren-backtick.ts` | Tests `` (`re`) `` pattern | +| `/demo/debug-header-stripping.ts` | Main debug script | + +--- + +## Test Commands + +```bash +# Run debug script +docker exec -it lixpi-markdown-stream-parser-demo bash -c "cd /usr/src/service && npx tsx demo/test-real-stream.ts 2>&1" + +# Run test suite +docker exec -it lixpi-markdown-stream-parser-demo bash -c "cd /usr/src/service && npm test -- tree-sitter-markdown-stream-parser.test.ts --run" +``` + +--- + +## Notes + +- Many debug `console.log` statements with prefixes `[STYLE]`, `[GETCODE]`, `[DEBUG]` are still in the code - should be removed once working +- The code has extensive comments explaining the logic +- All 13 Phase 1 tests are passing diff --git a/TREE_SITTER_MIGRATION_ROADMAP.md b/TREE_SITTER_MIGRATION_ROADMAP.md new file mode 100644 index 0000000..e058e6e --- /dev/null +++ b/TREE_SITTER_MIGRATION_ROADMAP.md @@ -0,0 +1,351 @@ +# Tree-Sitter Migration Roadmap + +## Overview +This document outlines the plan to bring the tree-sitter-based markdown parser implementation up to feature parity with the existing state-machine parser. + +## State-Machine Parser Capabilities (Current Baseline) + +### Block Elements +1. **Headers** (Levels 1-6) + - Detected by: `#{1,6}\s` pattern + - Output includes: `level` field (1-6) + - Content emitted WITHOUT the `#` markers + - Example: `## Header` → `{type: 'header', level: 2, segment: 'Header'}` + +2. **Paragraphs** + - Default block type for regular text + - Can contain inline styles + - Output: `{type: 'paragraph', segment: '...', styles: [...]}` + +3. **Code Blocks** + - Detected by: ` ```language\n` pattern + - Includes language detection (e.g., `javascript`, `python`) + - Output includes: `language` field + - Example: ` ```javascript\n` → `{type: 'codeBlock', language: 'javascript'}` + - Content between markers is emitted as code segments + +### Inline Styles +1. **Bold** - `**text**` + - Style: `['bold']` + +2. **Italic** - `*text*` + - Style: `['italic']` + +3. **Bold + Italic** - `***text***` + - Style: `['bold', 'italic']` + +4. **Strikethrough** - `~~text~~` + - Style: `['strikethrough']` + +5. **Inline Code** - `` `code` `` + - Style: `['code']` + +### Key Features +1. **Segment Splitting** + - Prefixed content: text before style markers emitted separately + - Styled content: emitted with appropriate `styles` array + - Postfixed content: text after style markers emitted separately + - Example: `before**bold**after` → 3 segments: + - `{segment: 'before', styles: []}` + - `{segment: 'bold', styles: ['bold']}` + - `{segment: 'after', styles: []}` + +2. **isBlockDefining Flag** + - `true` when: + - Starting a new block type + - Transitioning between blocks + - Processing header with newline + - First segment of paragraph after code block + - `false` for subsequent segments within same block + +3. **isProcessingNewLine Flag** + - `true` when segment contains `\n` + - Used to track paragraph boundaries + +4. **Whitespace Handling** + - Trailing newlines can be truncated + - Spaces preserved in styled segments + +## Tree-Sitter Current Issues + +### 1. ❌ Style Name Mismatches +**Problem**: Tree-sitter uses different style names than expected +- Currently: `'inline_code'` +- Expected: `'code'` +- Tree-sitter node names don't map 1:1 to output style names + +**Solution**: +```typescript +// In detectActiveStyles() +if (current.type === 'code_span') { + styles.add('code'); // NOT 'inline_code' +} +``` + +### 2. ❌ Block Type Naming Inconsistency +**Problem**: Inconsistent naming convention +- State-machine uses: `'codeBlock'` (camelCase) +- Tree-sitter uses: `'code_block'` (snake_case) + +**Solution**: Ensure consistent camelCase throughout +```typescript +case 'fenced_code_block': + return { type: 'codeBlock' }; // NOT 'code_block' +``` + +### 3. ❌ Missing Language Detection for Code Blocks +**Problem**: Code blocks don't include language field +- State-machine: `{type: 'codeBlock', language: 'javascript'}` +- Tree-sitter: `{type: 'code_block'}` (no language) + +**Solution**: Parse the `info_string` child node in fenced_code_block +```typescript +private getCodeBlockLanguage(node: Parser.SyntaxNode): string { + // Find info_string child node + const infoString = node.children.find(c => c.type === 'info_string'); + return infoString ? infoString.text.trim() : ''; +} +``` + +### 4. ❌ Header Content Includes Markers +**Problem**: Headers emitted with `#` symbols +- Expected: `{segment: 'Header Text'}` +- Actual: `{segment: '## Header Text'}` + +**Solution**: Extract only the text content from heading nodes +```typescript +private getHeaderContent(node: Parser.SyntaxNode, newContent: string): string { + // Remove leading # and whitespace + return newContent.replace(/^#{1,6}\s*/, ''); +} +``` + +### 5. ❌ No Segment Splitting for Inline Styles +**Problem**: Everything emitted as single chunk +- State-machine: `'before**bold**after'` → 3 segments +- Tree-sitter: `'before**bold**after'` → 1 segment + +**Solution**: Implement segment boundary detection +- Detect style marker positions +- Split content at style boundaries +- Emit prefixed, styled, and postfixed content separately + +### 6. ❌ Incomplete Style Detection +**Problem**: Styles not properly tracked through nested nodes +- May miss styles when content spans multiple nodes +- Style inheritance not working correctly + +**Solution**: Walk up the tree more carefully and test with nested examples + +### 7. ❌ isBlockDefining Logic Too Simple +**Problem**: Flag not accurately reflecting block transitions +- Currently only checks if block type changed +- Doesn't handle newline boundaries properly +- Doesn't detect paragraph boundaries correctly + +**Solution**: Enhanced logic considering: +- Previous block type +- Newline presence +- Node boundaries +- Content position + +## Implementation Plan + +### Phase 1: Fix Basic Mappings (Quick Wins) +- [ ] Fix style names (`'code'` instead of `'inline_code'`) +- [ ] Fix block type names (camelCase consistency) +- [ ] Add code block language detection +- [ ] Fix header content extraction (remove markers) + +### Phase 2: Segment Splitting (Core Feature) +- [ ] Detect inline style boundaries in content +- [ ] Implement segment splitting logic +- [ ] Emit prefixed content separately +- [ ] Emit styled content with proper styles array +- [ ] Emit postfixed content separately +- [ ] Handle multiple styles in same segment + +### Phase 3: Flag Logic Improvements +- [ ] Fix `isBlockDefining` logic +- [ ] Improve `isProcessingNewLine` detection +- [ ] Handle block transitions correctly + +### Phase 4: Testing & Validation +- [ ] Port state-machine tests to tree-sitter +- [ ] Add new tests for edge cases +- [ ] Verify with svelte-demo visual testing +- [ ] Test with all LLM stream examples + +## Testing Strategy + +### Unit Tests to Port +From `markdown-state-machine.test.ts`: +1. Simple paragraph processing +2. Headers (all 6 levels) +3. Code blocks (with/without language) +4. Inline styles: bold, italic, bold-italic, strikethrough, inline code +5. Mixed inline styles +6. Prefixed/postfixed content +7. Nested styles +8. State transitions (paragraph→header, header→code, code→paragraph) +9. Edge cases (empty segments, multiple newlines, malformed markdown) + +### Integration Testing +- Test with svelte-demo for visual verification +- Use existing LLM stream examples +- Compare output between state-machine and tree-sitter implementations + +## Expected Output Format + +### Example 1: Simple Paragraph +**Input**: `'Hello '`, `'world.\n'` + +**Output**: +```json +[ + { + "status": "STREAMING", + "segment": { + "segment": "Hello ", + "type": "paragraph", + "styles": [], + "isBlockDefining": true, + "isProcessingNewLine": false + } + }, + { + "status": "STREAMING", + "segment": { + "segment": "world.", + "type": "paragraph", + "styles": [], + "isBlockDefining": false, + "isProcessingNewLine": true + } + } +] +``` + +### Example 2: Header +**Input**: `'## '`, `'A header\n'` + +**Output**: +```json +[ + { + "status": "STREAMING", + "segment": { + "segment": "A header", + "type": "header", + "level": 2, + "styles": [], + "isBlockDefining": true, + "isProcessingNewLine": true + } + } +] +``` + +### Example 3: Bold Style with Prefixed Content +**Input**: `'before**bold**after'` + +**Output**: +```json +[ + { + "status": "STREAMING", + "segment": { + "segment": "before", + "type": "paragraph", + "styles": [], + "isBlockDefining": true, + "isProcessingNewLine": false + } + }, + { + "status": "STREAMING", + "segment": { + "segment": "bold", + "type": "paragraph", + "styles": ["bold"], + "isBlockDefining": false, + "isProcessingNewLine": false + } + }, + { + "status": "STREAMING", + "segment": { + "segment": "after", + "type": "paragraph", + "styles": [], + "isBlockDefining": false, + "isProcessingNewLine": false + } + } +] +``` + +### Example 4: Code Block with Language +**Input**: ` ```javascript\n`, `'const a = 1;\n'`, ` ```\n` + +**Output**: +```json +[ + { + "status": "STREAMING", + "segment": { + "segment": "const a = 1;", + "type": "codeBlock", + "language": "javascript", + "styles": [], + "isBlockDefining": true, + "isProcessingNewLine": true + } + } +] +``` + +## Success Criteria + +### Minimum Requirements +- ✅ All block types correctly identified (header, paragraph, codeBlock) +- ✅ Header levels correctly detected (1-6) +- ✅ Code block language correctly extracted +- ✅ All inline styles correctly detected (bold, italic, bold-italic, strikethrough, code) +- ✅ Segment splitting works for inline styles +- ✅ Prefixed/postfixed content emitted separately +- ✅ isBlockDefining flag accurate for block transitions +- ✅ All state-machine tests pass with tree-sitter implementation + +### Nice to Have +- 🎯 Better performance than state-machine +- 🎯 Support for nested styles (e.g., bold within italic) +- 🎯 Support for more markdown features (lists, blockquotes, etc.) +- 🎯 Better handling of malformed markdown + +## Notes + +### Tree-Sitter Advantages +- More robust parsing (proper AST) +- Better handling of complex/nested structures +- Easier to extend with new features +- Industry-standard approach + +### Challenges +- Different parsing model (AST vs streaming state machine) +- Need to "chunk" AST into streaming segments +- Style boundary detection more complex +- Must maintain streaming performance characteristics + +### Migration Risk Areas +1. **Inline style splitting**: Most complex feature to replicate +2. **Block boundaries**: Need careful testing for edge cases +3. **Performance**: AST parsing on each chunk may be slower +4. **Memory**: Full content buffering vs incremental processing + +## Next Steps +1. Start with Phase 1 quick wins +2. Add comprehensive tests as we go +3. Test each feature with svelte-demo +4. Compare output with state-machine for each test case +5. Document any behavioral differences diff --git a/debug-italic.cjs b/debug-italic.cjs new file mode 100644 index 0000000..5a9bbba --- /dev/null +++ b/debug-italic.cjs @@ -0,0 +1,41 @@ + +const mod = require('web-tree-sitter'); +// Handle both CommonJS and potentially different export structures +const Parser = mod.Parser || mod; + +async function test() { + if (typeof Parser.init !== 'function') { + console.error('Parser.init is not a function. Exports:', mod); + return; + } + await Parser.init(); + const parser = new Parser(); + const lang = await Parser.Language.load('/home/dima/Desktop/md-str-parser/markdown-stream-parser/public/tree-sitter-markdown-inline.wasm'); + parser.setLanguage(lang); + + const texts = [ + '*italic star*', + '_italic underscore_', + 'normal *italic* normal', + '*partial' + ]; + + for (const text of texts) { + const tree = parser.parse(text); + console.log(`\nText: "${text}"`); + console.log(tree.rootNode.toString()); + + const emphasis = tree.rootNode.descendantsOfType('emphasis'); + if (emphasis.length > 0) { + const node = emphasis[0]; + console.log('Emphasis delimiters:'); + node.children.forEach(c => { + if (c.type === 'emphasis_delimiter') { + console.log(`- "${c.text}" (${c.type})`); + } + }); + } + } +} + +test().catch(console.error); diff --git a/demo/debug-header-stripping.ts b/demo/debug-header-stripping.ts new file mode 100644 index 0000000..b2ed0b9 --- /dev/null +++ b/demo/debug-header-stripping.ts @@ -0,0 +1,68 @@ +import { MarkdownStreamParser } from '../src/tree-sitter-markdown-stream-parser.ts'; +import fs from 'fs'; + +async function test() { + // Configure WASM + MarkdownStreamParser.configureWasmPath('./demo/svelte-demo/static/tree-sitter-markdown.wasm'); + + const parser = await MarkdownStreamParser.getInstance('debug-test'); + const segments: any[] = []; + + parser.subscribeToTokenParse((chunk) => { + if (chunk.status === 'STREAMING' && chunk.segment) { + segments.push(chunk.segment); + } + }); + + parser.startParsing(); + + // Test code blocks and inline code + console.log('\n=== Testing Code Blocks and Inline Code ===\n'); + + // Test 1: Word-by-word like TokensStreamBuffer does + console.log('Test 1: Word-by-word inline code (simulating TokensStreamBuffer)'); + parser.parseToken('Here '); + parser.parseToken('is '); + parser.parseToken('`inline '); // Opening backtick + partial content + parser.parseToken('code` '); // Closing - NOW tree-sitter should detect code_span + parser.parseToken('test\n\n'); + + // Test 2: Inline code where each word is already separate (works correctly) + console.log('\nTest 2: Single-word inline code (works correctly)'); + parser.parseToken('This '); + parser.parseToken('is '); + parser.parseToken('`another` '); // Single word between backticks - arrives complete + parser.parseToken('test\n\n'); + + // Test fenced code block + parser.parseToken('```'); + parser.parseToken('python\n'); + parser.parseToken('def hello():\n'); + parser.parseToken(' print("world")\n'); + parser.parseToken('```'); + parser.parseToken('\n'); + + parser.stopParsing(); + + console.log('\n=== All Segments ==='); + segments.forEach((seg, i) => { + console.log(`${i + 1}. [${seg.type}]${seg.styles?.length ? ' styles: ' + seg.styles.join(',') : ''}: "${seg.segment}"`); + }); + + const codeBlocks = segments.filter(s => s.type === 'codeBlock'); + const inlineCode = segments.filter(s => s.styles?.includes('code')); + + console.log(`\n✓ Code blocks found: ${codeBlocks.length}`); + console.log(`✓ Inline code segments found: ${inlineCode.length}`); + + // Check for backticks in content + // NOTE: Some backticks may remain due to TokensStreamBuffer splitting words mid-code-span + const segmentsWithBackticks = segments.filter(s => s.segment.includes('`')); + console.log(`\n${segmentsWithBackticks.length > 0 ? '⚠️ ' : '✅'} Segments with backticks: ${segmentsWithBackticks.length}`); + if (segmentsWithBackticks.length > 0) { + console.log(' (Note: Due to word-boundary splitting by TokensStreamBuffer)'); + segmentsWithBackticks.forEach(s => console.log(` - [${s.type}]: "${s.segment}"`)); + } +} + +test().catch(console.error); diff --git a/demo/reproducing_bold.ts b/demo/reproducing_bold.ts new file mode 100644 index 0000000..3ecc100 --- /dev/null +++ b/demo/reproducing_bold.ts @@ -0,0 +1,18 @@ +import { MarkdownStreamParser } from '../src/markdown-stream-parser.ts' + +const parser = MarkdownStreamParser.getInstance('test-bold') + +parser.subscribeToTokenParse((token) => { + // We want to see if we get a token with style 'bold' + console.log(JSON.stringify(token, null, 2)) +}) + +parser.startParsing() + +const chunks = ['**', 'bold**'] + +for (const chunk of chunks) { + parser.parseToken(chunk) +} + +parser.stopParsing() diff --git a/demo/svelte-demo/src/routes/+page.svelte b/demo/svelte-demo/src/routes/+page.svelte index ddb17a4..a9ca9c3 100644 --- a/demo/svelte-demo/src/routes/+page.svelte +++ b/demo/svelte-demo/src/routes/+page.svelte @@ -1,268 +1,323 @@
-
-

@lixpi/markdown-stream-parser demo

-

This is just a quick and dirty showcase of the @lixpi/markdown-stream-parser, the parser itself has nothing to do with rendering !!! Please keep that in mind...

-

This demo is entirely `vibe-coded`, while the parser is painstakingly created by a human being 👩‍💻 :)

-
+
+

+ @lixpi/markdown-stream-parser demo +

+

+ This is just a quick and dirty showcase of the + @lixpi/markdown-stream-parser, the + parser itself has nothing to do with rendering + !!! Please keep that in mind... +

+

+ This demo is entirely `vibe-coded`, while + the parser is painstakingly created by a human being 👩‍💻 :) +

+
+
- {#if paused} - {:else} - {/if} - -
@@ -314,26 +388,42 @@ $: {
-
+

Parsed Stream

{#each parsedBlocks as block} -
+ {@const hasTableCells = block.some( + (seg) => + seg.segment?.type === "table_cell" || + seg.segment?.type === "table_header_cell", + )} +
{#each block as seg} - {#if seg.segment?.type === 'header'} + {#if seg.segment?.type === "header"} {#if seg.segment?.level === 1}

{#if seg.segment?.styles?.length} - - {#if seg.segment.styles.includes('strikethrough')} - {seg.segment?.segment} - {:else if seg.segment.styles.includes('code')} - {seg.segment?.segment} + + {#if seg.segment.styles.includes("strikethrough")} + {seg.segment?.segment} + {:else if seg.segment.styles.includes("code")} + {seg.segment?.segment} {:else} {seg.segment?.segment} {/if} @@ -345,16 +435,25 @@ $: { {:else if seg.segment?.level === 2}

{#if seg.segment?.styles?.length} - - {#if seg.segment.styles.includes('strikethrough')} - {seg.segment?.segment} - {:else if seg.segment.styles.includes('code')} - {seg.segment?.segment} + + {#if seg.segment.styles.includes("strikethrough")} + {seg.segment?.segment} + {:else if seg.segment.styles.includes("code")} + {seg.segment?.segment} {:else} {seg.segment?.segment} {/if} @@ -366,16 +465,25 @@ $: { {:else if seg.segment?.level === 3}

{#if seg.segment?.styles?.length} - - {#if seg.segment.styles.includes('strikethrough')} - {seg.segment?.segment} - {:else if seg.segment.styles.includes('code')} - {seg.segment?.segment} + + {#if seg.segment.styles.includes("strikethrough")} + {seg.segment?.segment} + {:else if seg.segment.styles.includes("code")} + {seg.segment?.segment} {:else} {seg.segment?.segment} {/if} @@ -387,16 +495,25 @@ $: { {:else if seg.segment?.level === 4}

{#if seg.segment?.styles?.length} - - {#if seg.segment.styles.includes('strikethrough')} - {seg.segment?.segment} - {:else if seg.segment.styles.includes('code')} - {seg.segment?.segment} + + {#if seg.segment.styles.includes("strikethrough")} + {seg.segment?.segment} + {:else if seg.segment.styles.includes("code")} + {seg.segment?.segment} {:else} {seg.segment?.segment} {/if} @@ -408,16 +525,25 @@ $: { {:else if seg.segment?.level === 5}
{#if seg.segment?.styles?.length} - - {#if seg.segment.styles.includes('strikethrough')} - {seg.segment?.segment} - {:else if seg.segment.styles.includes('code')} - {seg.segment?.segment} + + {#if seg.segment.styles.includes("strikethrough")} + {seg.segment?.segment} + {:else if seg.segment.styles.includes("code")} + {seg.segment?.segment} {:else} {seg.segment?.segment} {/if} @@ -429,16 +555,25 @@ $: { {:else if seg.segment?.level === 6}
{#if seg.segment?.styles?.length} - - {#if seg.segment.styles.includes('strikethrough')} - {seg.segment?.segment} - {:else if seg.segment.styles.includes('code')} - {seg.segment?.segment} + + {#if seg.segment.styles.includes("strikethrough")} + {seg.segment?.segment} + {:else if seg.segment.styles.includes("code")} + {seg.segment?.segment} {:else} {seg.segment?.segment} {/if} @@ -450,16 +585,25 @@ $: { {:else} {#if seg.segment?.styles?.length} - - {#if seg.segment.styles.includes('strikethrough')} - {seg.segment?.segment} - {:else if seg.segment.styles.includes('code')} - {seg.segment?.segment} + + {#if seg.segment.styles.includes("strikethrough")} + {seg.segment?.segment} + {:else if seg.segment.styles.includes("code")} + {seg.segment?.segment} {:else} {seg.segment?.segment} {/if} @@ -469,23 +613,123 @@ $: { {/if} {/if} - {:else if seg.segment?.type === 'codeBlock'} - 
{seg.segment?.segment}
- {:else if seg.segment?.type === 'blockQuote'} - {seg.segment?.segment} + {:else if seg.segment?.type === "codeBlock"} + 
{seg.segment?.segment}
+ {:else if seg.segment?.type === "blockQuote"} + {seg.segment?.segment} + {:else if seg.segment?.type === "list_item"} + + {#if seg.segment?.isBlockDefining} + • + {/if} + {#if seg.segment?.styles?.length} + + {#if seg.segment.styles.includes("strikethrough")} + {seg.segment?.segment} + {:else if seg.segment.styles.includes("code")} + {seg.segment?.segment} + {:else} + {seg.segment?.segment} + {/if} + + {:else} + {seg.segment?.segment} + {/if} + + {:else if seg.segment?.type === "table_header_cell"} + + {#if seg.segment?.styles?.length} + + {#if seg.segment.styles.includes("strikethrough")} + {seg.segment?.segment} + {:else if seg.segment.styles.includes("code")} + {seg.segment?.segment} + {:else} + {seg.segment?.segment} + {/if} + + {:else} + {seg.segment?.segment} + {/if} + + {:else if seg.segment?.type === "table_cell"} + + {#if seg.segment?.styles?.length} + + {#if seg.segment.styles.includes("strikethrough")} + {seg.segment?.segment} + {:else if seg.segment.styles.includes("code")} + {seg.segment?.segment} + {:else} + {seg.segment?.segment} + {/if} + + {:else} + {seg.segment?.segment} + {/if} + {:else} {#if seg.segment?.styles?.length} - - {#if seg.segment.styles.includes('strikethrough')} + + {#if seg.segment.styles.includes("strikethrough")} {seg.segment?.segment} - {:else if seg.segment.styles.includes('code')} - {seg.segment?.segment} + {:else if seg.segment.styles.includes("code")} + {seg.segment?.segment} {:else} {seg.segment?.segment} {/if} @@ -505,19 +749,33 @@ $: {

Current Token

- 
{JSON.stringify(currentToken, null, 2)}
+ 
{JSON.stringify(
+            currentToken,
+            null,
+            2,
+          )}

Parsed Chunks

{#if currentParsedChunks.length > 0} {#each currentParsedChunks as chunk, index}
-
{index + 1} of {currentParsedChunks.length}
- 
{JSON.stringify(chunk, null, 2)}
+
+ {index + 1} of {currentParsedChunks.length} +
+ 
{JSON.stringify(
+                  chunk,
+                  null,
+                  2,
+                )}
{/each} {:else} -
No parsed chunks for this token
+
+ No parsed chunks for this token +
{/if}
@@ -525,11 +783,16 @@ $: {

Raw array of streamed tokens

-
+
{#each jsonItems as item, index}
{JSON.stringify(item)}
@@ -540,7 +803,8 @@ $: {

Concatenated raw LLM output

- 
{txtContent}
+ 
{txtContent}
diff --git a/demo/svelte-demo/static/grammar.js b/demo/svelte-demo/static/grammar.js new file mode 100644 index 0000000..7db82c2 --- /dev/null +++ b/demo/svelte-demo/static/grammar.js @@ -0,0 +1,474 @@ +// This grammar only concerns the inline structure according to the CommonMark Spec +// (https://spec.commonmark.org/0.30/#inlines) +// For more information see README.md + +/// + +const common = require('../common/common'); + +// Levels used for dynmic precedence. Ideally +// n * PRECEDENCE_LEVEL_EMPHASIS > PRECEDENCE_LEVEL_LINK for any n, so maybe the +// maginuted of these values should be increased in the future +const PRECEDENCE_LEVEL_EMPHASIS = 1; +const PRECEDENCE_LEVEL_LINK = 10; +const PRECEDENCE_LEVEL_HTML = 100; + +// Punctuation characters as specified in +// https://github.github.com/gfm/#ascii-punctuation-character +const PUNCTUATION_CHARACTERS_REGEX = '!-/:-@\\[-`\\{-~'; + + +// !!! +// Notice the call to `add_inline_rules` which generates some additional rules related to parsing +// inline contents in different contexts. +// !!! +module.exports = grammar(add_inline_rules({ + name: 'markdown_inline', + + externals: $ => [ + // An `$._error` token is never valid and gets emmited to kill invalid parse branches. Concretely + // this is used to decide wether a newline closes a paragraph and together and it gets emitted + // when trying to parse the `$._trigger_error` token in `$.link_title`. + $._error, + $._trigger_error, + + // Opening and closing delimiters for code spans. These are sequences of one or more backticks. + // An opening token does not mean the text after has to be a code span if there is no closing token + $._code_span_start, + $._code_span_close, + + // Opening and closing delimiters for emphasis. + $._emphasis_open_star, + $._emphasis_open_underscore, + $._emphasis_close_star, + $._emphasis_close_underscore, + + // For emphasis we need to tell the parser if the last character was a whitespace (or the + // beginning of a line) or a punctuation. These tokens never actually get emitted. + $._last_token_whitespace, + $._last_token_punctuation, + + $._strikethrough_open, + $._strikethrough_close, + + // Opening and closing delimiters for latex. These are sequences of one or more dollar signs. + // An opening token does not mean the text after has to be latex if there is no closing token + $._latex_span_start, + $._latex_span_close, + + // Token emmited when encountering opening delimiters for a leaf span + // e.g. a code span, that does not have a matching closing span + $._unclosed_span + ], + precedences: $ => [ + // [$._strong_emphasis_star, $._inline_element_no_star], + [$._strong_emphasis_star_no_link, $._inline_element_no_star_no_link], + // [$._strong_emphasis_underscore, $._inline_element_no_underscore], + [$._strong_emphasis_underscore_no_link, $._inline_element_no_underscore_no_link], + [$.hard_line_break, $._whitespace], + [$.hard_line_break, $._text_base], + ], + // More conflicts are defined in `add_inline_rules` + conflicts: $ => [ + + [$._closing_tag, $._text_base], + [$._open_tag, $._text_base], + [$._html_comment, $._text_base], + [$._processing_instruction, $._text_base], + [$._declaration, $._text_base], + [$._cdata_section, $._text_base], + + [$._link_text_non_empty, $._inline_element], + [$._link_text_non_empty, $._inline_element_no_star], + [$._link_text_non_empty, $._inline_element_no_underscore], + [$._link_text_non_empty, $._inline_element_no_tilde], + [$._link_text, $._inline_element], + [$._link_text, $._inline_element_no_star], + [$._link_text, $._inline_element_no_underscore], + [$._link_text, $._inline_element_no_tilde], + + [$._image_description, $._image_description_non_empty, $._text_base], + // [$._image_description, $._image_description_non_empty, $._text_inline], + // [$._image_description, $._image_description_non_empty, $._text_inline_no_star], + // [$._image_description, $._image_description_non_empty, $._text_inline_no_underscore], + + [$._image_shortcut_link, $._image_description], + [$.shortcut_link, $._link_text], + [$.link_destination, $.link_title], + [$._link_destination_parenthesis, $.link_title], + + [$.wiki_link, $._inline_element], + [$.wiki_link, $._inline_element_no_star], + [$.wiki_link, $._inline_element_no_underscore], + [$.wiki_link, $._inline_element_no_tilde], + ], + extras: $ => [], + + rules: { + inline: $ => seq(optional($._last_token_whitespace), $._inline), + + ...common.rules, + + + // A lot of inlines are defined in `add_inline_rules`, including: + // + // * collections of inlines + // * emphasis + // * textual content + // + // This is done to reduce code duplication, as some inlines need to be parsed differently + // depending on the context. For example inlines in ATX headings may not contain newlines. + + code_span: $ => seq( + alias($._code_span_start, $.code_span_delimiter), + repeat(choice($._text_base, '[', ']', $._soft_line_break, $._html_tag)), + alias($._code_span_close, $.code_span_delimiter) + ), + + latex_block: $ => seq( + alias($._latex_span_start, $.latex_span_delimiter), + repeat(choice($._text_base, '[', ']', $._soft_line_break, $._html_tag)), + alias($._latex_span_close, $.latex_span_delimiter), + ), + + // Different kinds of links: + // * inline links (https://github.github.com/gfm/#inline-link) + // * full reference links (https://github.github.com/gfm/#full-reference-link) + // * collapsed reference links (https://github.github.com/gfm/#collapsed-reference-link) + // * shortcut links (https://github.github.com/gfm/#shortcut-reference-link) + // + // Dynamic precedence is distributed as granular as possible to help the parser decide + // while parsing which branch is the most important. + // + // https://github.github.com/gfm/#links + _link_text: $ => prec.dynamic(PRECEDENCE_LEVEL_LINK, choice( + $._link_text_non_empty, + seq('[', ']') + )), + _link_text_non_empty: $ => seq('[', alias($._inline_no_link, $.link_text), ']'), + shortcut_link: $ => prec.dynamic(PRECEDENCE_LEVEL_LINK, $._link_text_non_empty), + full_reference_link: $ => prec.dynamic(2 * PRECEDENCE_LEVEL_LINK, seq( + $._link_text, + $.link_label + )), + collapsed_reference_link: $ => prec.dynamic(PRECEDENCE_LEVEL_LINK, seq( + $._link_text, + '[', + ']' + )), + inline_link: $ => prec.dynamic(PRECEDENCE_LEVEL_LINK, seq( + $._link_text, + '(', + repeat(choice($._whitespace, $._soft_line_break)), + optional(seq( + choice( + seq( + $.link_destination, + optional(seq( + repeat1(choice($._whitespace, $._soft_line_break)), + $.link_title + )) + ), + $.link_title, + ), + repeat(choice($._whitespace, $._soft_line_break)), + )), + ')' + )), + + wiki_link: $ => prec.dynamic(2 * PRECEDENCE_LEVEL_LINK, seq( + '[', '[', + alias($._wiki_link_destination, $.link_destination), + optional(seq( + '|', + alias($._wiki_link_text, $.link_text) + )), + ']', ']' + ) + ), + + _wiki_link_destination: $ => repeat1(choice( + $._word, + common.punctuation_without($, ['[',']', '|']), + $._whitespace, + )), + + _wiki_link_text: $ => repeat1(choice( + $._word, + common.punctuation_without($, ['[',']']), + $._whitespace, + )), + + // Images work exactly like links with a '!' added in front. + // + // https://github.github.com/gfm/#images + image: $ => choice( + $._image_inline_link, + $._image_shortcut_link, + $._image_full_reference_link, + $._image_collapsed_reference_link + ), + _image_inline_link: $ => prec.dynamic(PRECEDENCE_LEVEL_LINK, seq( + $._image_description, + '(', + repeat(choice($._whitespace, $._soft_line_break)), + optional(seq( + choice( + seq( + $.link_destination, + optional(seq( + repeat1(choice($._whitespace, $._soft_line_break)), + $.link_title + )) + ), + $.link_title, + ), + repeat(choice($._whitespace, $._soft_line_break)), + )), + ')' + )), + _image_shortcut_link: $ => prec.dynamic(3 * PRECEDENCE_LEVEL_LINK, $._image_description_non_empty), + _image_full_reference_link: $ => prec.dynamic(PRECEDENCE_LEVEL_LINK, seq($._image_description, $.link_label)), + _image_collapsed_reference_link: $ => prec.dynamic(PRECEDENCE_LEVEL_LINK, seq($._image_description, '[', ']')), + _image_description: $ => prec.dynamic(3 * PRECEDENCE_LEVEL_LINK, choice($._image_description_non_empty, seq('!', '[', prec(1, ']')))), + _image_description_non_empty: $ => seq('!', '[', alias($._inline, $.image_description), prec(1, ']')), + + // Autolinks. Uri autolinks actually accept protocolls of arbitrary length which does not + // align with the spec. This is because the binary for the grammar gets to large if done + // otherwise as tree-sitters code generation is not very concise for this type of regex. + // + // Email autolinks do not match every valid email (emails normally should not be parsed + // using regexes), but this is how they are defined in the spec. + // + // https://github.github.com/gfm/#autolinks + uri_autolink: $ => /<[a-zA-Z][a-zA-Z0-9+\.\-][a-zA-Z0-9+\.\-]*:[^ \t\r\n<>]*>/, + email_autolink: $ => + /<[a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*>/, + + // Raw html. As with html blocks we do not emit additional information as this is best done + // by a proper html tree-sitter grammar. + // + // https://github.github.com/gfm/#raw-html + _html_tag: $ => choice($._open_tag, $._closing_tag, $._html_comment, $._processing_instruction, $._declaration, $._cdata_section), + _open_tag: $ => prec.dynamic(PRECEDENCE_LEVEL_HTML, seq('<', $._tag_name, repeat($._attribute), repeat(choice($._whitespace, $._soft_line_break)), optional('/'), '>')), + _closing_tag: $ => prec.dynamic(PRECEDENCE_LEVEL_HTML, seq('<', '/', $._tag_name, repeat(choice($._whitespace, $._soft_line_break)), '>')), + _tag_name: $ => seq($._word_no_digit, repeat(choice($._word_no_digit, $._digits, '-'))), + _attribute: $ => seq(repeat1(choice($._whitespace, $._soft_line_break)), $._attribute_name, repeat(choice($._whitespace, $._soft_line_break)), '=', repeat(choice($._whitespace, $._soft_line_break)), $._attribute_value), + _attribute_name: $ => /[a-zA-Z_:][a-zA-Z0-9_\.:\-]*/, + _attribute_value: $ => choice( + /[^ \t\r\n"'=<>`]+/, + seq("'", repeat(choice($._word, $._whitespace, $._soft_line_break, common.punctuation_without($, ["'"]))), "'"), + seq('"', repeat(choice($._word, $._whitespace, $._soft_line_break, common.punctuation_without($, ['"']))), '"'), + ), + _html_comment: $ => prec.dynamic(PRECEDENCE_LEVEL_HTML, seq( + '' + )), + _processing_instruction: $ => prec.dynamic(PRECEDENCE_LEVEL_HTML, seq( + '' + )), + _declaration: $ => prec.dynamic(PRECEDENCE_LEVEL_HTML, seq( + /']), + ))), + '>' + )), + _cdata_section: $ => prec.dynamic(PRECEDENCE_LEVEL_HTML, seq( + '' + )), + + // A hard line break. + // + // https://github.github.com/gfm/#hard-line-breaks + hard_line_break: $ => seq(choice('\\', $._whitespace_ge_2), $._soft_line_break), + _text: $ => choice($._word, common.punctuation_without($, []), $._whitespace), + + // Whitespace is divided into single whitespaces and multiple whitespaces as wee need this + // information for hard line breaks. + _whitespace_ge_2: $ => /\t| [ \t]+/, + _whitespace: $ => seq(choice($._whitespace_ge_2, / /), optional($._last_token_whitespace)), + + // Other than whitespace we tokenize into strings of digits, punctuation characters + // (handled by `common.punctuation_without`) and strings of any other characters. This way the + // lexer does not have to many different states, which makes it a lot easier to make + // conflicts work. + _word: $ => choice($._word_no_digit, $._digits), + _word_no_digit: $ => new RegExp('[^' + PUNCTUATION_CHARACTERS_REGEX + ' \\t\\n\\r0-9]+(_+[^' + PUNCTUATION_CHARACTERS_REGEX + ' \\t\\n\\r0-9]+)*'), + _digits: $ => /[0-9][0-9_]*/, + _soft_line_break: $ => seq($._newline_token, optional($._last_token_whitespace)), + + _inline_base: $ => prec.right(repeat1(choice( + $.image, + $._soft_line_break, + $.backslash_escape, + $.hard_line_break, + $.uri_autolink, + $.email_autolink, + $.entity_reference, + $.numeric_character_reference, + (common.EXTENSION_LATEX ? $.latex_block : choice()), + $.code_span, + alias($._html_tag, $.html_tag), + $._text_base, + common.EXTENSION_TAGS ? $.tag : choice(), + $._unclosed_span, + ))), + _text_base: $ => choice( + $._word, + common.punctuation_without($, ['[', ']']), + $._whitespace, + ' B[TokensStreamBuffer] - B --> C[Emit Complete Segment] - C --> D[MarkdownStreamParser] - D --> F((•)) + B --> C[Accumulate Content] + C --> D[Tree-sitter Parse] + D --> E[AST Traversal] + E --> F[Emit Segments] ``` -#### 2: Blocks and Inline Elements +#### 1. Token Buffering -Markdown consists of two fundamental components: +Incoming tokens are accumulated in a `TokensStreamBuffer`. This gives us enough context to parse meaningful chunks rather than character-by-character. -1. *Block-level elements* (paragraphs, headings, lists) which define document structure and cannot be nested within each other -2. *Inline elements* (bold, italic, code spans) which apply styling within blocks +#### 2. AST-Based Parsing with Tree-sitter -This distinction is central to our parsing approach, as it allows us to process markdown streams with predictable patterns. Block elements establish context, while inline styles modify content within that context. +The core parsing is done by `web-tree-sitter` with the `tree-sitter-markdown` grammar. When content comes in, we parse it and get an AST that tells us exactly what we're dealing with - headers, paragraphs, code blocks, lists, bold text, whatever. +The nice thing about tree-sitter is that it handles incomplete/malformed markdown gracefully. It uses error recovery and can still produce a usable tree even when the input is partial or slightly broken (which happens constantly with LLM streams). -#### 3: Routing aka State Machine +#### 3. Modular Architecture -The `MarkdownStreamParser` implements a state machine that processes text chunks from the `TokensStreamBuffer`. It utilizes pattern-matching evaluations based on regular expressions to determine the appropriate state transitions. +The tree-sitter parsing logic is split into focused modules: -The core of this architecture is the routing mechanism, which: +- **`block-detection.ts`** - Figures out what type of block we're in (header, paragraph, code block, list item, blockquote, table) +- **`inline-detection.ts`** - Detects active inline styles (bold, italic, code spans, strikethrough) by examining AST nodes +- **`content-extraction.ts`** - Strips markdown syntax (like `#` from headers or ``` from code blocks) and extracts clean content +- **`inline-extractors.ts`** - Specialized extractors for each inline style that properly strip markers and apply styles +- **`segment-generator.ts`** - Orchestrates everything and produces the final segments -1. Receives buffered segments from the stream processor -2. Executes pattern-matching evaluations against incoming content (partial or full matches) -3. Triggers corresponding actions based on matched patterns -4. Transitions the parser into the appropriate state (block-level or inline) +Each module does one thing, which makes the code easier to reason about and test. -This consistent routing approach handles both high-level block elements (headings, paragraphs, code blocks) and inline styling (bold, italic, code spans) using the same underlying mechanism. +#### 4. Handling Incomplete Inline Markers -Below is a **simplified diagram** for parsing a Markdown stream containing a paragraph with inline styles (italic, bold, etc.). This example omits other block types for clarity. +A tricky problem with streaming is that inline markers can arrive split across chunks. For example, you might get `**hello` in one chunk and `**` in the next. -```mermaid -stateDiagram-v2 - [*] --> idle - idle --> routing: startParsing() - routing --> processParagraph: paragraph detected - processParagraph --> emit: emit parsed segment - emit --> routing: next segment - - routing --> processInlineStylesGroup: inline style detected - processInlineStylesGroup --> emit: emit styled segment - emit --> routing: next segment - - routing --> handleMalformedSyntax: malformed style (e.g., missing closing marker + new line symbol that denotes beginning of a new block) - handleMalformedSyntax --> emit: emit unstyled segment - emit --> routing: next segment - - routing --> [*]: end of stream - - %% Notes: - %% - "emit" represents emitting a parsed segment to subscribers. - %% - The state machine loops through routing and processing states for each segment. - %% - If an inline style is opened but not closed before a new line, the unstyled content is flushed via emit and the state machine returns to routing. - %% - Only paragraph and inline style states are shown for simplicity. -``` - -Alternatively parser state transitions can be represented like this: - -```mermaid -stateDiagram-v2 - direction LR - receiveToken --> buffer - buffer --> splitWords - splitWords --> routing +The parser buffers content when it detects an unmatched delimiter. It uses tree-sitter to check whether a marker is complete: - routing --> processingHeader: header pattern detected - routing --> processingCodeBlock: code block detected - routing --> processingParagraph: default - - processingHeader --> emit: emit parsed segment - processingHeader --> routing: inline style detected - - processingParagraph --> emit: emit parsed segment - processingParagraph --> routing: inline style detected - - %% Inline styles can only be entered from header or paragraph - routing --> processingItalicText: italic detected - routing --> processingBoldText: bold detected - routing --> processingBoldItalicText: bold+italic detected - routing --> processingStrikethroughText: strikethrough detected - routing --> processingInlineCode: inline code detected - - processingItalicText --> emit: emit parsed segment - processingItalicText --> routing: next segment - - processingBoldText --> emit - processingBoldText --> routing - - processingBoldItalicText --> emit - processingBoldItalicText --> routing +```typescript +// Check for unmatched backtick +if (newPortion.includes('`')) { + const hasCompleteCodeSpan = hasCompleteCodeSpanAt(inlineTree.rootNode, ...); + if (!hasCompleteCodeSpan) { + state.pendingInlineContent = newContent; + return { segments, state }; // Buffer and wait for more + } +} +``` - processingStrikethroughText --> emit - processingStrikethroughText --> routing +This applies to inline code, bold (`**`), italic (`*` or `_`), and strikethrough (`~~`). - processingInlineCode --> emit - processingInlineCode --> routing +#### 5. Inline Parser for Detailed Analysis - processingCodeBlock --> emit - processingCodeBlock --> routing +For inline content within blocks, we use a second tree-sitter parser with the `tree-sitter-markdown-inline` grammar. This gives us detailed AST info about emphasis delimiters, code spans, etc. - emit --> routing: next segment - emit --> [*]: end of stream -``` +The two-parser approach (one for block structure, one for inline content) is how tree-sitter-markdown is designed to work. It lets us accurately detect things like whether a `*` is actually an italic marker or just a literal asterisk. -#### 4: Publish/Subscribe Pattern +#### 6. Publish/Subscribe Pattern -The parser uses a *publish/subscribe* pattern to decouple the flow of data from its consumption. This design enables you to feed data into the parser and independently subscribe to a stream of parsed output. +The parser uses a pub/sub pattern. You subscribe to get parsed segments as they're ready: ```mermaid flowchart TD A[Input Tokens] -->|buffer| B(TokensStreamBuffer) - B -->|segment| C(MarkdownStreamParserStateMachine) - C -->|buffer| D(MarkdownStreamParser) - D -->|notify parsed segment| E[Subscribers] + B -->|raw content| C(Tree-sitter Parser) + C -->|AST nodes| D(Segment Generator) + D -->|notify| E[Subscribers] ``` -##### Benefits of this approach: -- Enables real-time, event-driven processing of Markdown streams -- Cleanly separates parsing logic from rendering or further processing -- Supports multiple independent subscribers per parser instance - -To receive parsed segments, simply subscribe to the parser before feeding data. Each subscriber is notified as soon as a new segment is available, and can unsubscribe at any time. - -#### 5. Singleton Pattern - -The parser utilizes a singleton pattern for instance management. Associate each logical stream with a unique `instanceId`. Use `MarkdownStreamParser.getInstance(instanceId)` to retrieve or create the parser for that stream, and `MarkdownStreamParser.removeInstance(instanceId)` for cleanup when the stream ends. +Benefits: +- Real-time, event-driven processing +- Parsing is decoupled from rendering +- Multiple subscribers per parser instance -This design enables *parallel processing* of multiple independent streams (e.g., concurrent user sessions or documents). By isolating each stream's state within its dedicated instance, the library ensures consistent state management. +#### 7. Singleton Pattern -#### 6. Regex-driven Parsing +Each logical stream gets its own parser instance via `getInstance(instanceId)`. This allows parallel processing of multiple streams without state conflicts. -This project uses a **regex-driven approach** for parsing segments, which while sometimes **debated** so far allowed to achieve the more stable result than previous attempts. -**There's still HUGE number of bugs. Refer to some examples in demo.** - -For each markup type, we define a set of regex rules to detect both full matches (e.g., single-word styles) and partial matches, which indicate the start or end of a style applied across multiple words. +```typescript +const parser = await MarkdownStreamParser.getInstance('session-1') +// ... use the parser ... +MarkdownStreamParser.removeInstance('session-1') // cleanup when done +``` --- ## Known issues -- **Delayed processing for extremely long sequences of characters without whitespace**: This is a downside of using the L1 buffer. Given the speed of modern LLMs, it's not a significant issue. The only time it becomes visually noticeable is when an LLM generates an **extremely long** regex, causing the output to freeze until receiving the final sequence. While this may be inconvenient, it's a rare edge case and not a high priority to fix. - -- **Inline styles for headings** are not implemented yet. Therefore, when a stream contains something like `### Title **with bold word**`, only the heading part will be detected. This should be fixed in the near future. +- **Delayed processing for extremely long sequences of characters without whitespace**: Due to how token buffering works, extremely long uninterrupted sequences (like a huge regex) can delay output until the sequence completes. In practice this is rarely noticeable with modern LLM speeds, but it can happen. --- -## Future Plans and Directions - -### Exploration of Alternative Parsing Architectures - -While our current regex-based approach provides good results for LLM-generated Markdown streams, we recognize that established parsing libraries may offer additional benefits for long-term scalability and maintenance. We're evaluating: - -- **Tree-sitter** - - A mature incremental parsing system adopted by Neovim and formerly by Atom - - Offers syntax recovery parsing with efficient incremental updates - - References: - - [Tree-sitter Documentation](https://tree-sitter.github.io/) - - [Tree-sitter repo](https://github.com/tree-sitter/tree-sitter) - - [Node.js Tree-sitter repo](https://github.com/tree-sitter/node-tree-sitter) - - [A Markdown parser for tree-sitter](https://github.com/tree-sitter-grammars/tree-sitter-markdown) - -- **Lezer** - - Modern incremental parser system developed by the authors of **ProseMirror** && **CodeMirror**... - - Designed specifically for editor use cases, also supports syntax recovery, though not sure if as advanced as `Tree-sitter` - - References: - - [Lezer Documentation](https://lezer.codemirror.net) - - [Lezer Markdown Grammar](https://github.com/lezer-parser/markdown) - -1. These parsers can effectively handle partial Markdown syntax across stream chunks -2. Our L1 buffer concept could be integrated with these parsers to maintain the current user experience - - -**Community feedback and contributions are especially welcome regarding these architectural considerations, as diverse use cases will help inform the best approach.** -Please feel free to share your thoughts in **[discussions](https://github.com/Lixpi/markdown-stream-parser/discussions)**. - - ## Contributions and Roadmap - **Contributions:** - PRs and issues are *welcome*! - + PRs and issues are *welcome*! Feel free to share your thoughts in **[discussions](https://github.com/Lixpi/markdown-stream-parser/discussions)**. - **Roadmap:** - - Support for the missing markdown features listed earlier. + - Support for the missing markdown features listed earlier - Performance optimizations - - Build an AST (abstract syntax tree) model to represent the parsed stream in memory + - Improved error recovery for malformed streams --- From 4c42892de2dee081711d12151602f90961660af9 Mon Sep 17 00:00:00 2001 From: Dmitry Bondarenko Date: Wed, 21 Jan 2026 22:24:18 +0600 Subject: [PATCH 15/32] Documentation update with mermaid diagrams, multi-line jsdoc style comments changed to double slash comments, trailing semicooooooooooooooolons removed --- README.md | 191 +++++++++++--- src/tree-sitter-markdown-stream-parser.ts | 270 ++++++++++---------- src/tree-sitter/block-detection.ts | 104 ++++---- src/tree-sitter/content-extraction.ts | 50 ++-- src/tree-sitter/index.ts | 20 +- src/tree-sitter/inline-detection.ts | 182 ++++++------- src/tree-sitter/inline-extractors.ts | 114 ++++----- src/tree-sitter/segment-builder.ts | 52 ++-- src/tree-sitter/segment-generator.ts | 296 +++++++++++----------- src/tree-sitter/tree-navigation.ts | 64 ++--- src/tree-sitter/types.ts | 118 ++++----- 11 files changed, 746 insertions(+), 715 deletions(-) diff --git a/README.md b/README.md index c10ae38..cde3402 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ A library designed to incrementally parse Markdown text from a stream of tokens. -It's built to handle the ambiguities of LLM-generated streams, which often produce imperfect or invalid Markdown.It uses **tree-sitter** under the hood. Instead of regex pattern matching, we get a proper AST that tells us exactly what's a header, what's a code block, what's bold text, etc. Tree-sitter's error recovery also handles the imperfect markdown that LLMs tend to produce. +It uses **tree-sitter** under the hood. Instead of regex pattern matching, we get a proper AST that tells us exactly what's a header, what's a code block, what's bold text, etc. Tree-sitter's error recovery also handles the imperfect markdown that LLMs tend to produce. ### ⚠️ ***This project is still in active development - there are bugs and missing features.*** @@ -275,44 +275,176 @@ The project includes comprehensive test coverage with 187 tests across all core The parser uses **tree-sitter** for AST-based parsing. Instead of trying to match patterns with regex, we let tree-sitter build a syntax tree and then walk it to extract the content we need. -Here's the general flow: +### High-Level Data Flow ```mermaid +%%{init: {'theme': 'base', 'themeVariables': { 'primaryColor': '#F6C7B3', 'primaryTextColor': '#5a3a2a', 'primaryBorderColor': '#d4956a', 'secondaryColor': '#C3DEDD', 'secondaryTextColor': '#1a3a47', 'secondaryBorderColor': '#4a8a9d', 'tertiaryColor': '#DCECE9', 'tertiaryTextColor': '#1a3a47', 'tertiaryBorderColor': '#82B2C0', 'lineColor': '#d4956a', 'textColor': '#5a3a2a'}}}%% flowchart LR - A[Token] --> B[TokensStreamBuffer] + A[LLM Token] --> B[TokensStreamBuffer] B --> C[Accumulate Content] C --> D[Tree-sitter Parse] D --> E[AST Traversal] E --> F[Emit Segments] + F --> G[Subscribers] ``` -#### 1. Token Buffering +### Module Architecture -Incoming tokens are accumulated in a `TokensStreamBuffer`. This gives us enough context to parse meaningful chunks rather than character-by-character. +The tree-sitter parsing logic is split into focused modules: -#### 2. AST-Based Parsing with Tree-sitter +```mermaid +%%{init: {'theme': 'base', 'themeVariables': { 'primaryColor': '#F6C7B3', 'primaryTextColor': '#5a3a2a', 'primaryBorderColor': '#d4956a', 'secondaryColor': '#C3DEDD', 'secondaryTextColor': '#1a3a47', 'secondaryBorderColor': '#4a8a9d', 'tertiaryColor': '#DCECE9', 'tertiaryTextColor': '#1a3a47', 'tertiaryBorderColor': '#82B2C0', 'lineColor': '#d4956a', 'textColor': '#5a3a2a'}}}%% +graph TB + subgraph "Entry Point" + Parser[MarkdownStreamParser] + end + + subgraph "Tree-sitter Modules" + SG[segment-generator.ts] + BD[block-detection.ts] + ID[inline-detection.ts] + CE[content-extraction.ts] + IE[inline-extractors.ts] + TN[tree-navigation.ts] + SB[segment-builder.ts] + end + + subgraph "External" + TS[(web-tree-sitter)] + MD[(tree-sitter-markdown)] + MDI[(tree-sitter-markdown-inline)] + end + + Parser --> SG + SG --> BD + SG --> ID + SG --> CE + SG --> IE + BD --> TN + ID --> TN + IE --> SB + CE --> TS + BD --> TS + ID --> TS + TS --> MD + TS --> MDI +``` -The core parsing is done by `web-tree-sitter` with the `tree-sitter-markdown` grammar. When content comes in, we parse it and get an AST that tells us exactly what we're dealing with - headers, paragraphs, code blocks, lists, bold text, whatever. +| Module | Responsibility | +|--------|----------------| +| `segment-generator.ts` | Main orchestrator - generates segments from content ranges | +| `block-detection.ts` | Figures out block type (header, paragraph, code block, list, table) | +| `inline-detection.ts` | Detects active inline styles (bold, italic, code, strikethrough) | +| `content-extraction.ts` | Strips markdown syntax and extracts clean content | +| `inline-extractors.ts` | Extracts styled segments with proper marker stripping | +| `tree-navigation.ts` | AST traversal utilities | +| `segment-builder.ts` | Creates segment objects with consistent structure | -The nice thing about tree-sitter is that it handles incomplete/malformed markdown gracefully. It uses error recovery and can still produce a usable tree even when the input is partial or slightly broken (which happens constantly with LLM streams). +### Parser API Flow -#### 3. Modular Architecture +```mermaid +%%{init: {'theme': 'base', 'themeVariables': { 'noteBkgColor': '#82B2C0', 'noteTextColor': '#1a3a47', 'noteBorderColor': '#5a9aad', 'actorBkg': '#F6C7B3', 'actorBorder': '#d4956a', 'actorTextColor': '#5a3a2a', 'actorLineColor': '#d4956a', 'signalColor': '#d4956a', 'signalTextColor': '#5a3a2a', 'labelBoxBkgColor': '#F6C7B3', 'labelBoxBorderColor': '#d4956a', 'labelTextColor': '#5a3a2a', 'loopTextColor': '#5a3a2a', 'activationBorderColor': '#d4956a', 'activationBkgColor': '#C3DEDD', 'sequenceNumberColor': '#5a3a2a'}}}%% +sequenceDiagram + participant App as Your App + participant Parser as MarkdownStreamParser + participant Buffer as TokensStreamBuffer + participant TS as Tree-sitter + participant Gen as SegmentGenerator + + rect rgb(220, 236, 233) + Note over App, Gen: Setup Phase + App->>Parser: getInstance(sessionId) + activate Parser + Parser->>TS: load WASM grammars + Parser-->>App: parser instance + end + + rect rgb(195, 222, 221) + Note over App, Gen: Subscription Phase + App->>Parser: subscribeToTokenParse(listener) + App->>Parser: startParsing() + Parser-->>App: START_STREAM event + end + + rect rgb(246, 199, 179) + Note over App, Gen: Streaming Phase + loop For each LLM token + App->>Parser: parseToken(chunk) + Parser->>Buffer: receiveChunk(chunk) + Buffer->>Parser: segment ready + Parser->>TS: parse(content) + TS-->>Parser: AST + Parser->>Gen: generateSegments(range) + Gen-->>Parser: StreamingChunk[] + Parser-->>App: notify(segment) + end + end + + rect rgb(242, 234, 224) + Note over App, Gen: Cleanup Phase + App->>Parser: stopParsing() + Parser->>Buffer: flushBuffer() + Parser-->>App: END_STREAM event + deactivate Parser + App->>Parser: removeInstance(sessionId) + end +``` -The tree-sitter parsing logic is split into focused modules: +### Parser State Transitions -- **`block-detection.ts`** - Figures out what type of block we're in (header, paragraph, code block, list item, blockquote, table) -- **`inline-detection.ts`** - Detects active inline styles (bold, italic, code spans, strikethrough) by examining AST nodes -- **`content-extraction.ts`** - Strips markdown syntax (like `#` from headers or ``` from code blocks) and extracts clean content -- **`inline-extractors.ts`** - Specialized extractors for each inline style that properly strip markers and apply styles -- **`segment-generator.ts`** - Orchestrates everything and produces the final segments +```mermaid +%%{init: {'theme': 'base', 'themeVariables': { 'primaryColor': '#F6C7B3', 'primaryTextColor': '#5a3a2a', 'primaryBorderColor': '#d4956a', 'secondaryColor': '#C3DEDD', 'lineColor': '#d4956a', 'textColor': '#5a3a2a'}}}%% +stateDiagram-v2 + [*] --> Idle: getInstance() + + Idle --> Parsing: startParsing() + + state Parsing { + [*] --> AwaitingToken + + AwaitingToken --> ProcessingChunk: parseToken(chunk) + ProcessingChunk --> DetectingBlock: tree-sitter parse + DetectingBlock --> ProcessingHeader: atx_heading found + DetectingBlock --> ProcessingParagraph: paragraph found + DetectingBlock --> ProcessingCodeBlock: fenced_code_block found + DetectingBlock --> ProcessingList: list_item found + DetectingBlock --> ProcessingTable: pipe_table found + + ProcessingHeader --> DetectingInline: check inline styles + ProcessingParagraph --> DetectingInline: check inline styles + ProcessingList --> DetectingInline: check inline styles + ProcessingTable --> DetectingInline: check inline styles + + DetectingInline --> BufferingIncomplete: unmatched delimiter + DetectingInline --> EmitSegment: style complete + BufferingIncomplete --> AwaitingToken: wait for more + + ProcessingCodeBlock --> EmitSegment: extract content + EmitSegment --> AwaitingToken: notify subscribers + } + + Parsing --> Flushing: stopParsing() + Flushing --> Idle: END_STREAM + Idle --> [*]: removeInstance() +``` -Each module does one thing, which makes the code easier to reason about and test. +### How Content Gets Processed -#### 4. Handling Incomplete Inline Markers +#### 1. Token Buffering + +Incoming tokens are accumulated in a `TokensStreamBuffer`. This gives us enough context to parse meaningful chunks rather than character-by-character. + +#### 2. AST-Based Parsing + +The core parsing is done by `web-tree-sitter` with the `tree-sitter-markdown` grammar. When content comes in, we parse it and get an AST that tells us exactly what we're dealing with - headers, paragraphs, code blocks, lists, bold text, etc. + +Tree-sitter handles incomplete/malformed markdown gracefully. It uses error recovery and can still produce a usable tree even when the input is partial or slightly broken (which happens constantly with LLM streams). + +#### 3. Handling Incomplete Inline Markers A tricky problem with streaming is that inline markers can arrive split across chunks. For example, you might get `**hello` in one chunk and `**` in the next. -The parser buffers content when it detects an unmatched delimiter. It uses tree-sitter to check whether a marker is complete: +The parser buffers content when it detects an unmatched delimiter: ```typescript // Check for unmatched backtick @@ -327,32 +459,17 @@ if (newPortion.includes('`')) { This applies to inline code, bold (`**`), italic (`*` or `_`), and strikethrough (`~~`). -#### 5. Inline Parser for Detailed Analysis +#### 4. Two-Parser Approach For inline content within blocks, we use a second tree-sitter parser with the `tree-sitter-markdown-inline` grammar. This gives us detailed AST info about emphasis delimiters, code spans, etc. The two-parser approach (one for block structure, one for inline content) is how tree-sitter-markdown is designed to work. It lets us accurately detect things like whether a `*` is actually an italic marker or just a literal asterisk. -#### 6. Publish/Subscribe Pattern - -The parser uses a pub/sub pattern. You subscribe to get parsed segments as they're ready: - -```mermaid -flowchart TD - A[Input Tokens] -->|buffer| B(TokensStreamBuffer) - B -->|raw content| C(Tree-sitter Parser) - C -->|AST nodes| D(Segment Generator) - D -->|notify| E[Subscribers] -``` - -Benefits: -- Real-time, event-driven processing -- Parsing is decoupled from rendering -- Multiple subscribers per parser instance +### Pub/Sub and Singleton Patterns -#### 7. Singleton Pattern +The parser uses a **publish/subscribe** pattern - you subscribe to get parsed segments as they're ready. Parsing is decoupled from rendering, and multiple subscribers per parser instance are supported. -Each logical stream gets its own parser instance via `getInstance(instanceId)`. This allows parallel processing of multiple streams without state conflicts. +Each logical stream gets its own parser instance via `getInstance(instanceId)` (singleton pattern). This allows parallel processing of multiple streams without state conflicts. ```typescript const parser = await MarkdownStreamParser.getInstance('session-1') diff --git a/src/tree-sitter-markdown-stream-parser.ts b/src/tree-sitter-markdown-stream-parser.ts index f425150..ff4146d 100644 --- a/src/tree-sitter-markdown-stream-parser.ts +++ b/src/tree-sitter-markdown-stream-parser.ts @@ -1,58 +1,56 @@ -import { Parser, Language } from 'web-tree-sitter'; -import TokensStreamBuffer from './tokens-stream-buffer.js'; +import { Parser, Language } from 'web-tree-sitter' +import TokensStreamBuffer from './tokens-stream-buffer.js' import { type StreamingChunk, type BlockState, generateSegments, type SegmentGeneratorState, -} from './tree-sitter/index.js'; +} from './tree-sitter/index.js' // Re-export types for external consumers -export type { StreamingSegment, StreamingChunk } from './tree-sitter/index.js'; - -/** - * Tree-sitter based streaming markdown parser. - * - * Parses markdown content incrementally as it streams in, detecting: - * - Block types (headers, paragraphs, code blocks, lists, tables, blockquotes) - * - Inline styles (bold, italic, code, strikethrough) - * - Block boundaries and levels - * - * Uses tree-sitter for accurate AST-based parsing with proper handling of - * incomplete structures that may occur during streaming. - */ +export type { StreamingSegment, StreamingChunk } from './tree-sitter/index.js' + +// Tree-sitter based streaming markdown parser. +// +// Parses markdown content incrementally as it streams in, detecting: +// - Block types (headers, paragraphs, code blocks, lists, tables, blockquotes) +// - Inline styles (bold, italic, code, strikethrough) +// - Block boundaries and levels +// +// Uses tree-sitter for accurate AST-based parsing with proper handling of +// incomplete structures that may occur during streaming. export class MarkdownStreamParser { // Static singleton management - private static instances = new Map(); - private static parserInitialized = false; - private static parserInitPromise: Promise | null = null; - private static markdownLanguage: Parser.Language | null = null; - private static markdownInlineLanguage: Parser.Language | null = null; - private static wasmPath: string | null = null; - private static wasmInlinePath: string | null = null; + private static instances = new Map() + private static parserInitialized = false + private static parserInitPromise: Promise | null = null + private static markdownLanguage: Parser.Language | null = null + private static markdownInlineLanguage: Parser.Language | null = null + private static wasmPath: string | null = null + private static wasmInlinePath: string | null = null // Parser instances - private parser: Parser | null = null; - private inlineParser: Parser | null = null; - private currentTree: Parser.Tree | null = null; + private parser: Parser | null = null + private inlineParser: Parser | null = null + private currentTree: Parser.Tree | null = null // Content state - private content: string = ''; - private lastProcessedIndex: number = 0; - private allSegments: StreamingChunk[] = []; + private content: string = '' + private lastProcessedIndex: number = 0 + private allSegments: StreamingChunk[] = [] // Segment generator state private generatorState: SegmentGeneratorState = { pendingInlineContent: '', pendingInlineStartIndex: 0, currentBlock: null, - }; + } // Integration with TokensStreamBuffer - private tokensStreamProcessor: TokensStreamBuffer; - private parsing: boolean = false; - private tokenParseListeners: Array<(chunk: StreamingChunk) => void> = []; - private unsubscribeFromProcessor: (() => void) | null = null; + private tokensStreamProcessor: TokensStreamBuffer + private parsing: boolean = false + private tokenParseListeners: Array<(chunk: StreamingChunk) => void> = [] + private unsubscribeFromProcessor: (() => void) | null = null /** * Configure the WASM file paths before creating any instances. @@ -60,11 +58,11 @@ export class MarkdownStreamParser { */ static configureWasmPath(markdownWasmPath: string, inlineWasmPath?: string): void { if (MarkdownStreamParser.parserInitialized) { - console.warn('WASM path configuration ignored - parser already initialized'); - return; + console.warn('WASM path configuration ignored - parser already initialized') + return } - MarkdownStreamParser.wasmPath = markdownWasmPath; - MarkdownStreamParser.wasmInlinePath = inlineWasmPath || markdownWasmPath.replace('.wasm', '-inline.wasm'); + MarkdownStreamParser.wasmPath = markdownWasmPath + MarkdownStreamParser.wasmInlinePath = inlineWasmPath || markdownWasmPath.replace('.wasm', '-inline.wasm') } /** @@ -74,19 +72,19 @@ export class MarkdownStreamParser { // Initialize parser and language once for all instances if (!MarkdownStreamParser.parserInitialized) { if (!MarkdownStreamParser.parserInitPromise) { - MarkdownStreamParser.parserInitPromise = MarkdownStreamParser.initializeParser(); + MarkdownStreamParser.parserInitPromise = MarkdownStreamParser.initializeParser() } - await MarkdownStreamParser.parserInitPromise; + await MarkdownStreamParser.parserInitPromise } if (!MarkdownStreamParser.instances.has(instanceId)) { - const instance = new MarkdownStreamParser(); - await instance.initialize(); - MarkdownStreamParser.instances.set(instanceId, instance); + const instance = new MarkdownStreamParser() + await instance.initialize() + MarkdownStreamParser.instances.set(instanceId, instance) } - console.info(`\x1b[34mMarkdownStreamParser ->\x1b[0m getInstance::instanceId: ${instanceId}`); - return MarkdownStreamParser.instances.get(instanceId)!; + console.info(`\x1b[34mMarkdownStreamParser ->\x1b[0m getInstance::instanceId: ${instanceId}`) + return MarkdownStreamParser.instances.get(instanceId)! } /** @@ -99,53 +97,53 @@ export class MarkdownStreamParser { locateFile(scriptName: string, scriptDirectory: string) { // In Node.js/test environment, use the configured wasm directory if (typeof window === 'undefined' && MarkdownStreamParser.wasmPath) { - const dir = MarkdownStreamParser.wasmPath.substring(0, MarkdownStreamParser.wasmPath.lastIndexOf('/')); - return dir + '/' + scriptName; + const dir = MarkdownStreamParser.wasmPath.substring(0, MarkdownStreamParser.wasmPath.lastIndexOf('/')) + return dir + '/' + scriptName } // Browser environment if (typeof window !== 'undefined') { - return window.location.origin + '/' + scriptName; + return window.location.origin + '/' + scriptName } // Fallback - return '/' + scriptName; + return '/' + scriptName } - }); + }) // Determine the correct path based on environment - let wasmPath = MarkdownStreamParser.wasmPath; + let wasmPath = MarkdownStreamParser.wasmPath if (!wasmPath) { if (typeof window !== 'undefined') { - wasmPath = '/tree-sitter-markdown.wasm'; + wasmPath = '/tree-sitter-markdown.wasm' } else { - wasmPath = './wasm/tree-sitter-markdown.wasm'; + wasmPath = './wasm/tree-sitter-markdown.wasm' } } - console.info(`Loading markdown WASM from: ${wasmPath}`); - MarkdownStreamParser.markdownLanguage = await Language.load(wasmPath); + console.info(`Loading markdown WASM from: ${wasmPath}`) + MarkdownStreamParser.markdownLanguage = await Language.load(wasmPath) // Load the inline language - let inlineWasmPath = MarkdownStreamParser.wasmInlinePath; + let inlineWasmPath = MarkdownStreamParser.wasmInlinePath if (!inlineWasmPath) { if (typeof window !== 'undefined') { - inlineWasmPath = '/tree-sitter-markdown-inline.wasm'; + inlineWasmPath = '/tree-sitter-markdown-inline.wasm' } else { - inlineWasmPath = './wasm/tree-sitter-markdown-inline.wasm'; + inlineWasmPath = './wasm/tree-sitter-markdown-inline.wasm' } } - console.info(`Loading markdown-inline WASM from: ${inlineWasmPath}`); - MarkdownStreamParser.markdownInlineLanguage = await Language.load(inlineWasmPath); + console.info(`Loading markdown-inline WASM from: ${inlineWasmPath}`) + MarkdownStreamParser.markdownInlineLanguage = await Language.load(inlineWasmPath) - MarkdownStreamParser.parserInitialized = true; - console.info('✅ Tree-sitter markdown language loaded successfully'); - console.info('✅ Tree-sitter markdown-inline language loaded successfully'); + MarkdownStreamParser.parserInitialized = true + console.info('✅ Tree-sitter markdown language loaded successfully') + console.info('✅ Tree-sitter markdown-inline language loaded successfully') } catch (error) { - console.error('Failed to load tree-sitter-markdown WASM:', error); - throw new Error(`Failed to initialize markdown parser: ${error}`); + console.error('Failed to load tree-sitter-markdown WASM:', error) + throw new Error(`Failed to initialize markdown parser: ${error}`) } } @@ -154,49 +152,49 @@ export class MarkdownStreamParser { */ private static getWasmPath(): string { if (MarkdownStreamParser.wasmPath) { - return MarkdownStreamParser.wasmPath; + return MarkdownStreamParser.wasmPath } if (typeof window !== 'undefined') { - return '/tree-sitter-markdown.wasm'; + return '/tree-sitter-markdown.wasm' } - return './wasm/tree-sitter-markdown.wasm'; + return './wasm/tree-sitter-markdown.wasm' } /** * Remove a parser instance. */ static removeInstance(instanceId: string): void { - const instance = MarkdownStreamParser.instances.get(instanceId); + const instance = MarkdownStreamParser.instances.get(instanceId) if (instance) { - instance.stopParsing(); - MarkdownStreamParser.instances.delete(instanceId); + instance.stopParsing() + MarkdownStreamParser.instances.delete(instanceId) } } constructor() { - this.tokensStreamProcessor = new TokensStreamBuffer(); + this.tokensStreamProcessor = new TokensStreamBuffer() } /** * Initialize this parser instance with the loaded languages. */ private async initialize(): Promise { - this.parser = new Parser(); - this.inlineParser = new Parser(); + this.parser = new Parser() + this.inlineParser = new Parser() if (!MarkdownStreamParser.markdownLanguage) { - throw new Error('Markdown language not loaded. This should not happen if getInstance() was used.'); + throw new Error('Markdown language not loaded. This should not happen if getInstance() was used.') } if (!MarkdownStreamParser.markdownInlineLanguage) { - throw new Error('Markdown-inline language not loaded.'); + throw new Error('Markdown-inline language not loaded.') } - this.parser.setLanguage(MarkdownStreamParser.markdownLanguage); - this.inlineParser.setLanguage(MarkdownStreamParser.markdownInlineLanguage); + this.parser.setLanguage(MarkdownStreamParser.markdownLanguage) + this.inlineParser.setLanguage(MarkdownStreamParser.markdownInlineLanguage) - console.info('Parser instance initialized with markdown and markdown-inline languages'); + console.info('Parser instance initialized with markdown and markdown-inline languages') } /** @@ -205,22 +203,22 @@ export class MarkdownStreamParser { */ subscribeToTokenParse(listener: (chunk: StreamingChunk, unsubscribe: () => void) => void): () => void { const wrappedListener = (data: StreamingChunk) => { - listener(data, unsubscribe); - }; + listener(data, unsubscribe) + } const unsubscribe = () => { - this.tokenParseListeners = this.tokenParseListeners.filter(l => l !== wrappedListener); - }; + this.tokenParseListeners = this.tokenParseListeners.filter(l => l !== wrappedListener) + } - this.tokenParseListeners.push(wrappedListener); - return unsubscribe; + this.tokenParseListeners.push(wrappedListener) + return unsubscribe } /** * Notify all subscribers about a parsed token. */ private notifyTokenParse(chunk: StreamingChunk): void { - this.tokenParseListeners.forEach(listener => listener(chunk)); + this.tokenParseListeners.forEach(listener => listener(chunk)) } /** @@ -228,26 +226,26 @@ export class MarkdownStreamParser { */ startParsing(): void { if (this.parsing) { - console.warn('Parser is already running'); - return; + console.warn('Parser is already running') + return } if (!this.parser) { - throw new Error('Parser not initialized. Call getInstance() to get an initialized instance.'); + throw new Error('Parser not initialized. Call getInstance() to get an initialized instance.') } - this.reset(); - this.notifyTokenParse({ status: 'START_STREAM' }); + this.reset() + this.notifyTokenParse({ status: 'START_STREAM' }) this.unsubscribeFromProcessor = this.tokensStreamProcessor.subscribeToSegmentCompletion((word: string) => { - const segments = this.processRawChunk(word); + const segments = this.processRawChunk(word) segments.forEach(segment => { - this.notifyTokenParse(segment); - }); - }); + this.notifyTokenParse(segment) + }) + }) - this.parsing = true; - console.info('\x1b[32mParser started\x1b[0m'); + this.parsing = true + console.info('\x1b[32mParser started\x1b[0m') } /** @@ -255,12 +253,12 @@ export class MarkdownStreamParser { */ parseToken(chunk: string): Error | void { if (!this.parsing) { - const error = new Error('Parser is not started. Call startParsing() first.'); - console.error('\x1b[31mMarkdownStreamParser::parseToken::error\x1b[0m', error.message); - return error; + const error = new Error('Parser is not started. Call startParsing() first.') + console.error('\x1b[31mMarkdownStreamParser::parseToken::error\x1b[0m', error.message) + return error } - this.tokensStreamProcessor.receiveChunk(chunk); + this.tokensStreamProcessor.receiveChunk(chunk) } /** @@ -268,20 +266,20 @@ export class MarkdownStreamParser { */ stopParsing(): void { if (!this.parsing) { - return; + return } - this.tokensStreamProcessor.flushBuffer(); + this.tokensStreamProcessor.flushBuffer() if (this.unsubscribeFromProcessor) { - this.unsubscribeFromProcessor(); - this.unsubscribeFromProcessor = null; + this.unsubscribeFromProcessor() + this.unsubscribeFromProcessor = null } - this.notifyTokenParse({ status: 'END_STREAM' }); + this.notifyTokenParse({ status: 'END_STREAM' }) - this.parsing = false; - console.info('\x1b[32mParser stopped\x1b[0m'); + this.parsing = false + console.info('\x1b[32mParser stopped\x1b[0m') } /** @@ -289,23 +287,23 @@ export class MarkdownStreamParser { */ private processRawChunk(chunk: string): StreamingChunk[] { if (!this.parser) { - return []; + return [] } - const oldLength = this.content.length; - this.content += chunk; - this.lastProcessedIndex = this.content.length; + const oldLength = this.content.length + this.content += chunk + this.lastProcessedIndex = this.content.length // For proper incremental parsing, tell tree-sitter what changed if (this.currentTree) { const getPosition = (index: number) => { - const textUpToIndex = this.content.substring(0, Math.min(index, this.content.length)); - const lines = textUpToIndex.split('\n'); + const textUpToIndex = this.content.substring(0, Math.min(index, this.content.length)) + const lines = textUpToIndex.split('\n') return { row: lines.length - 1, column: lines[lines.length - 1].length - }; - }; + } + } this.currentTree.edit({ startIndex: oldLength, @@ -314,11 +312,11 @@ export class MarkdownStreamParser { startPosition: getPosition(oldLength), oldEndPosition: getPosition(oldLength), newEndPosition: getPosition(this.content.length) - }); + }) } // Parse the updated content - this.currentTree = this.parser.parse(this.content, this.currentTree || undefined); + this.currentTree = this.parser.parse(this.content, this.currentTree || undefined) // Generate segments using the refactored module const result = generateSegments(oldLength, this.content.length, { @@ -326,70 +324,70 @@ export class MarkdownStreamParser { currentTree: this.currentTree, inlineParser: this.inlineParser, state: this.generatorState, - }); + }) // Update state - this.generatorState = result.state; + this.generatorState = result.state // Store all segments for debugging - this.allSegments.push(...result.segments); + this.allSegments.push(...result.segments) - return result.segments; + return result.segments } /** * Get the current accumulated content. */ getCurrentContent(): string { - return this.content; + return this.content } /** * Get all segments generated so far. */ getAllSegments(): StreamingChunk[] { - return this.allSegments; + return this.allSegments } /** * Get the current tree as a string (for debugging). */ getTreeString(): string { - if (!this.currentTree) return ''; - return this.currentTree.rootNode.toString(); + if (!this.currentTree) return '' + return this.currentTree.rootNode.toString() } /** * Get a summary of segments by type. */ getSegmentsSummary(): { total: number; byType: Record } { - const byType: Record = {}; + const byType: Record = {} this.allSegments.forEach(seg => { if (seg.segment) { - const type = seg.segment.type; - byType[type] = (byType[type] || 0) + 1; + const type = seg.segment.type + byType[type] = (byType[type] || 0) + 1 } - }); + }) return { total: this.allSegments.length, byType - }; + } } /** * Reset the parser state. */ reset(): void { - this.content = ''; - this.currentTree = null; - this.lastProcessedIndex = 0; - this.allSegments = []; + this.content = '' + this.currentTree = null + this.lastProcessedIndex = 0 + this.allSegments = [] this.generatorState = { pendingInlineContent: '', pendingInlineStartIndex: 0, currentBlock: null, - }; + } } } diff --git a/src/tree-sitter/block-detection.ts b/src/tree-sitter/block-detection.ts index ea52b2c..9b25f5d 100644 --- a/src/tree-sitter/block-detection.ts +++ b/src/tree-sitter/block-detection.ts @@ -1,16 +1,14 @@ -import type { Parser } from 'web-tree-sitter'; -import { HEADER_MARKER_LEVELS, type BlockInfo, type BlockState } from './types.js'; -import { findBlockNode } from './tree-navigation.js'; +import type { Parser } from 'web-tree-sitter' +import { HEADER_MARKER_LEVELS, type BlockInfo, type BlockState } from './types.js' +import { findBlockNode } from './tree-navigation.js' -/** - * Get the block type and properties from a tree-sitter node. - * Walks up the tree to find the enclosing block structure. - */ +// Get the block type and properties from a tree-sitter node. +// Walks up the tree to find the enclosing block structure. export function getBlockInfo(node: Parser.SyntaxNode): BlockInfo { - let current: Parser.SyntaxNode | null = node; - let foundParagraph = false; - let foundTableCell = false; - let isInHeader = false; + let current: Parser.SyntaxNode | null = node + let foundParagraph = false + let foundTableCell = false + let isInHeader = false while (current) { switch (current.type) { @@ -18,114 +16,108 @@ export function getBlockInfo(node: Parser.SyntaxNode): BlockInfo { return { type: 'header', level: getHeadingLevel(current) - }; + } case 'paragraph': // Don't return immediately - check if we're inside a list_item or blockquote - foundParagraph = true; - break; + foundParagraph = true + break case 'fenced_code_block': return { type: 'codeBlock', // camelCase for consistency language: getCodeBlockLanguage(current) - }; + } case 'list_item': // If we found a paragraph inside a list_item, return list_item - return { type: 'list_item' }; + return { type: 'list_item' } case 'blockquote': // If we found a paragraph inside a blockquote, return blockquote - return { type: 'blockquote' }; + return { type: 'blockquote' } // Table types case 'pipe_table_cell': - foundTableCell = true; - break; + foundTableCell = true + break case 'pipe_table_header': - isInHeader = true; + isInHeader = true // If we found a cell inside a header, return table_header_cell if (foundTableCell) { - return { type: 'table_header_cell', id: current.id }; + return { type: 'table_header_cell', id: current.id } } - break; + break case 'pipe_table_row': // If we found a cell inside a regular row, return table_cell if (foundTableCell) { - return { type: 'table_cell', id: current.id }; + return { type: 'table_cell', id: current.id } } - break; + break case 'pipe_table': // Found the table - if we have a cell, determine type based on header flag if (foundTableCell) { - return { type: isInHeader ? 'table_header_cell' : 'table_cell', id: current.id }; + return { type: isInHeader ? 'table_header_cell' : 'table_cell', id: current.id } } // Otherwise just return table - return { type: 'table' }; + return { type: 'table' } } - current = current.parent; + current = current.parent } // If we found a paragraph but no enclosing list_item/blockquote, return paragraph if (foundParagraph) { - return { type: 'paragraph' }; + return { type: 'paragraph' } } - return { type: 'paragraph' }; + return { type: 'paragraph' } } -/** - * Check if the given node represents a new block compared to the current block state. - */ +// Check if the given node represents a new block compared to the current block state. export function isNewBlock(blockInfo: BlockInfo, node: Parser.SyntaxNode, currentBlock: BlockState | null): boolean { - const blockNode = findBlockNode(node); - if (!blockNode) return false; + const blockNode = findBlockNode(node) + if (!blockNode) return false - if (!currentBlock) return true; + if (!currentBlock) return true - if (currentBlock.type !== blockInfo.type) return true; - if (blockInfo.level !== undefined && currentBlock.level !== blockInfo.level) return true; + if (currentBlock.type !== blockInfo.type) return true + if (blockInfo.level !== undefined && currentBlock.level !== blockInfo.level) return true - if (blockNode.startIndex > currentBlock.lastSegmentEnd) return true; + if (blockNode.startIndex > currentBlock.lastSegmentEnd) return true - return false; + return false } -/** - * Extract the heading level from an atx_heading node. - * Uses tree-sitter node type lookup with fallback to character counting. - */ +// Extract the heading level from an atx_heading node. +// Uses tree-sitter node type lookup with fallback to character counting. export function getHeadingLevel(node: Parser.SyntaxNode): number { // Use tree-sitter node type lookup instead of regex for (const child of node.children) { - const level = HEADER_MARKER_LEVELS[child.type]; + const level = HEADER_MARKER_LEVELS[child.type] if (level !== undefined) { - return level; + return level } } // Fallback: count # characters if tree-sitter node not found - const text = node.text || ''; - let hashCount = 0; + const text = node.text || '' + let hashCount = 0 for (let i = 0; i < text.length && text[i] === '#'; i++) { - hashCount++; + hashCount++ } if (hashCount >= 1 && hashCount <= 6 && (text[hashCount] === ' ' || text[hashCount] === undefined)) { - return hashCount; + return hashCount } - return 1; + return 1 } -/** - * Extract the language identifier from a fenced_code_block node. - */ +// Extract the language identifier from a fenced_code_block node. export function getCodeBlockLanguage(node: Parser.SyntaxNode): string { // For fenced_code_block, look for info_string child if (node.type === 'fenced_code_block') { for (let i = 0; i < node.childCount; i++) { - const child = node.child(i); + const child = node.child(i) if (child && child.type === 'info_string') { - return child.text.trim(); + return child.text.trim() } } } - return ''; + return '' } diff --git a/src/tree-sitter/content-extraction.ts b/src/tree-sitter/content-extraction.ts index f619eec..4d1a091 100644 --- a/src/tree-sitter/content-extraction.ts +++ b/src/tree-sitter/content-extraction.ts @@ -1,9 +1,7 @@ -import type { Parser } from 'web-tree-sitter'; +import type { Parser } from 'web-tree-sitter' -/** - * Extract header content from a chunk, excluding marker nodes (# symbols). - * Requires tree-sitter node for accurate extraction. - */ +// Extract header content from a chunk, excluding marker nodes (# symbols). +// Requires tree-sitter node for accurate extraction. export function getHeaderContent( content: string, node: Parser.SyntaxNode | undefined, @@ -13,39 +11,37 @@ export function getHeaderContent( // If we have the tree-sitter node, extract the actual heading content for this chunk if (node && node.type === 'atx_heading' && startByte !== undefined && endByte !== undefined) { // Find which part of the current chunk overlaps with non-marker content - let extractedText = ''; + let extractedText = '' for (const child of node.children) { // Skip marker nodes if (child.type.startsWith('atx_h') && child.type.endsWith('_marker')) { - continue; + continue } // Check if this child overlaps with our current chunk [startByte, endByte] if (child.startIndex < endByte && child.endIndex > startByte) { // Calculate the overlap - const overlapStart = Math.max(child.startIndex, startByte); - const overlapEnd = Math.min(child.endIndex, endByte); + const overlapStart = Math.max(child.startIndex, startByte) + const overlapEnd = Math.min(child.endIndex, endByte) if (overlapStart < overlapEnd) { // Extract just the overlapping portion - const relativeStart = overlapStart - startByte; - const relativeEnd = overlapEnd - startByte; - extractedText += content.substring(relativeStart, relativeEnd); + const relativeStart = overlapStart - startByte + const relativeEnd = overlapEnd - startByte + extractedText += content.substring(relativeStart, relativeEnd) } } } - return extractedText; + return extractedText } - throw new Error('Tree-sitter node required for header content extraction'); + throw new Error('Tree-sitter node required for header content extraction') } -/** - * Extract code block content from a chunk, excluding fence markers and info_string. - * Requires tree-sitter node for accurate extraction. - */ +// Extract code block content from a chunk, excluding fence markers and info_string. +// Requires tree-sitter node for accurate extraction. export function getCodeBlockContent( content: string, node: Parser.SyntaxNode | undefined, @@ -54,29 +50,29 @@ export function getCodeBlockContent( ): string { // If we have the tree-sitter node, extract code content excluding fence markers if (node && node.type === 'fenced_code_block' && startByte !== undefined && endByte !== undefined) { - let extractedText = ''; + let extractedText = '' for (const child of node.children) { // Skip fence markers and info_string if (child.type === 'fenced_code_block_delimiter' || child.type === 'info_string') { - continue; + continue } // Extract code content if (child.startIndex < endByte && child.endIndex > startByte) { - const overlapStart = Math.max(child.startIndex, startByte); - const overlapEnd = Math.min(child.endIndex, endByte); + const overlapStart = Math.max(child.startIndex, startByte) + const overlapEnd = Math.min(child.endIndex, endByte) if (overlapStart < overlapEnd) { - const relativeStart = overlapStart - startByte; - const relativeEnd = overlapEnd - startByte; - extractedText += content.substring(relativeStart, relativeEnd); + const relativeStart = overlapStart - startByte + const relativeEnd = overlapEnd - startByte + extractedText += content.substring(relativeStart, relativeEnd) } } } - return extractedText; + return extractedText } - throw new Error('Tree-sitter node required for code block content extraction'); + throw new Error('Tree-sitter node required for code block content extraction') } diff --git a/src/tree-sitter/index.ts b/src/tree-sitter/index.ts index d4da83c..d613dd5 100644 --- a/src/tree-sitter/index.ts +++ b/src/tree-sitter/index.ts @@ -6,7 +6,7 @@ export type { BlockInfo, InlineExtractionContext, InlineStyleConfig, -} from './types.js'; +} from './types.js' // Constants (runtime exports) export { @@ -14,7 +14,7 @@ export { BLOCK_TYPES, SUPPRESSED_SYNTAX_TYPES, INLINE_STYLE_CONFIGS, -} from './types.js'; +} from './types.js' // Tree navigation export { @@ -22,7 +22,7 @@ export { findNodeInTree, findInlineNodeAtPosition, findBlockNode, -} from './tree-navigation.js'; +} from './tree-navigation.js' // Block detection export { @@ -30,7 +30,7 @@ export { isNewBlock, getHeadingLevel, getCodeBlockLanguage, -} from './block-detection.js'; +} from './block-detection.js' // Inline detection export { @@ -41,13 +41,13 @@ export { hasUnmatchedItalicMarker, isInsideCodeBlock, detectActiveStyles, -} from './inline-detection.js'; +} from './inline-detection.js' // Content extraction export { getHeaderContent, getCodeBlockContent, -} from './content-extraction.js'; +} from './content-extraction.js' // Segment builder export { @@ -57,7 +57,7 @@ export { createChunkFromBlockInfo, createPlainTextChunk, createCodeBlockChunk, -} from './segment-builder.js'; +} from './segment-builder.js' // Inline extractors export { @@ -65,8 +65,8 @@ export { getBoldSegments, getItalicSegments, getStrikethroughSegments, -} from './inline-extractors.js'; +} from './inline-extractors.js' // Segment generator -export { generateSegments } from './segment-generator.js'; -export type { SegmentGeneratorState, SegmentGeneratorContext } from './segment-generator.js'; +export { generateSegments } from './segment-generator.js' +export type { SegmentGeneratorState, SegmentGeneratorContext } from './segment-generator.js' diff --git a/src/tree-sitter/inline-detection.ts b/src/tree-sitter/inline-detection.ts index f64dd5b..3e59c47 100644 --- a/src/tree-sitter/inline-detection.ts +++ b/src/tree-sitter/inline-detection.ts @@ -1,199 +1,185 @@ -import type { Parser } from 'web-tree-sitter'; -import { findActiveNodeAtPosition, findInlineNodeAtPosition } from './tree-navigation.js'; +import type { Parser } from 'web-tree-sitter' +import { findActiveNodeAtPosition, findInlineNodeAtPosition } from './tree-navigation.js' -/** - * Check if there's a complete code_span that overlaps with the given range. - */ +// Check if there's a complete code_span that overlaps with the given range. export function hasCompleteCodeSpanAt(inlineRoot: Parser.SyntaxNode, startPos: number, endPos: number): boolean { - const codeSpans = inlineRoot.descendantsOfType('code_span'); + const codeSpans = inlineRoot.descendantsOfType('code_span') for (const span of codeSpans) { // Check if this code_span overlaps with our range if (span.startIndex <= startPos && span.endIndex >= endPos) { - return true; + return true } // Also check partial overlap - if our content is inside a code_span if (span.startIndex < endPos && span.endIndex > startPos) { - return true; + return true } } - return false; + return false } -/** - * Check if there's a complete strong_emphasis that overlaps with the given range. - */ +// Check if there's a complete strong_emphasis that overlaps with the given range. export function hasCompleteBoldAt(inlineRoot: Parser.SyntaxNode, startPos: number, endPos: number): boolean { - const strongNodes = inlineRoot.descendantsOfType('strong_emphasis'); + const strongNodes = inlineRoot.descendantsOfType('strong_emphasis') for (const span of strongNodes) { // Check if this strong_emphasis overlaps with our range if (span.startIndex <= startPos && span.endIndex >= endPos) { - return true; + return true } // Also check partial overlap - if our content is inside a strong_emphasis if (span.startIndex < endPos && span.endIndex > startPos) { - return true; + return true } } - return false; + return false } -/** - * Check if there's a complete emphasis that overlaps with the given range. - */ +// Check if there's a complete emphasis that overlaps with the given range. export function hasCompleteItalicAt(inlineRoot: Parser.SyntaxNode, startPos: number, endPos: number): boolean { - const emphasisNodes = inlineRoot.descendantsOfType('emphasis'); + const emphasisNodes = inlineRoot.descendantsOfType('emphasis') for (const span of emphasisNodes) { // Check if this emphasis overlaps with our range if (span.startIndex <= startPos && span.endIndex >= endPos) { - return true; + return true } // Also check partial overlap - if our content is inside an emphasis if (span.startIndex < endPos && span.endIndex > startPos) { - return true; + return true } } - return false; + return false } -/** - * Check if there's a complete strikethrough that overlaps with the given range. - */ +// Check if there's a complete strikethrough that overlaps with the given range. export function hasCompleteStrikethroughAt(inlineRoot: Parser.SyntaxNode, startPos: number, endPos: number): boolean { - const strikethroughNodes = inlineRoot.descendantsOfType('strikethrough'); + const strikethroughNodes = inlineRoot.descendantsOfType('strikethrough') for (const span of strikethroughNodes) { // Check if this strikethrough overlaps with our range if (span.startIndex <= startPos && span.endIndex >= endPos) { - return true; + return true } // Also check partial overlap - if our content is inside a strikethrough if (span.startIndex < endPos && span.endIndex > startPos) { - return true; + return true } } - return false; + return false } -/** - * Check if the text contains an unmatched italic marker (* or _) - * that is not part of a ** sequence. - * Uses tree-sitter to detect emphasis_delimiter nodes that aren't matched. - */ +// Check if the text contains an unmatched italic marker (* or _) +// that is not part of a ** sequence. +// Uses tree-sitter to detect emphasis_delimiter nodes that aren't matched. export function hasUnmatchedItalicMarker(text: string, inlineParser: Parser | null): boolean { // Use tree-sitter inline parser to check for emphasis markers if (inlineParser) { - const inlineTree = inlineParser.parse(text); + const inlineTree = inlineParser.parse(text) if (inlineTree) { // Get all emphasis (italic) and strong_emphasis (bold) nodes - const emphasisNodes = inlineTree.rootNode.descendantsOfType('emphasis'); - const strongNodes = inlineTree.rootNode.descendantsOfType('strong_emphasis'); + const emphasisNodes = inlineTree.rootNode.descendantsOfType('emphasis') + const strongNodes = inlineTree.rootNode.descendantsOfType('strong_emphasis') // Helper to check if position is inside any matched emphasis or strong node const isInsideMatchedNode = (pos: number): boolean => { return emphasisNodes.some(node => pos >= node.startIndex && pos < node.endIndex) || - strongNodes.some(node => pos >= node.startIndex && pos < node.endIndex); - }; + strongNodes.some(node => pos >= node.startIndex && pos < node.endIndex) + } - const textContent = inlineTree.rootNode.text; + const textContent = inlineTree.rootNode.text // Check for single * that isn't part of ** and isn't inside a matched node for (let i = 0; i < textContent.length; i++) { - const char = textContent[i]; + const char = textContent[i] if (char === '*') { // Check if it's part of ** or *** - const prevChar = i > 0 ? textContent[i - 1] : ''; - const nextChar = i < textContent.length - 1 ? textContent[i + 1] : ''; + const prevChar = i > 0 ? textContent[i - 1] : '' + const nextChar = i < textContent.length - 1 ? textContent[i + 1] : '' // If this * is adjacent to another *, it's part of ** or ***, skip it if (prevChar === '*' || nextChar === '*') { - continue; + continue } // If this * is adjacent to /, it's part of /* or */ (comment delimiters), skip it // These are NOT italic markers but likely code comment syntax if (prevChar === '/' || nextChar === '/') { - continue; + continue } // This is a lone *, check if it's inside any emphasis or strong_emphasis node if (!isInsideMatchedNode(i)) { - return true; + return true } } else if (char === '_') { // Underscore is a potential italic marker // Check if it's inside a matched node if (!isInsideMatchedNode(i)) { - return true; + return true } } } - return false; + return false } } // Fallback: simple character check without regex // Check for * that isn't part of ** for (let i = 0; i < text.length; i++) { - const char = text[i]; + const char = text[i] if (char === '*') { - const prevChar = i > 0 ? text[i - 1] : ''; - const nextChar = i < text.length - 1 ? text[i + 1] : ''; + const prevChar = i > 0 ? text[i - 1] : '' + const nextChar = i < text.length - 1 ? text[i + 1] : '' // Skip if part of ** or adjacent to / (comment delimiters) if (prevChar !== '*' && nextChar !== '*' && prevChar !== '/' && nextChar !== '/') { - return true; // Lone asterisk found (not in **, /*, or */) + return true // Lone asterisk found (not in **, /*, or */) } } else if (char === '_') { - return true; // Underscore found + return true // Underscore found } } - return false; + return false } -/** - * Check if the position is inside a fenced_code_block or code_span (inline code). - * Used to skip italic buffering inside code contexts where _ is common in variable names. - */ +// Check if the position is inside a fenced_code_block or code_span (inline code). +// Used to skip italic buffering inside code contexts where _ is common in variable names. export function isInsideCodeBlock( node: Parser.SyntaxNode, position: number, currentTree: Parser.Tree | null, inlineParser: Parser | null ): boolean { - let current: Parser.SyntaxNode | null = findActiveNodeAtPosition(node, position); + let current: Parser.SyntaxNode | null = findActiveNodeAtPosition(node, position) while (current) { if (current.type === 'fenced_code_block' || current.type === 'code_fence_content') { - return true; + return true } - current = current.parent; + current = current.parent } // Also check inline tree for code_span (inline code like `variable_name`) if (currentTree && inlineParser) { - const inlineNode = findInlineNodeAtPosition(currentTree.rootNode, position); + const inlineNode = findInlineNodeAtPosition(currentTree.rootNode, position) if (inlineNode) { - const inlineContent = inlineNode.text; - const inlineTree = inlineParser.parse(inlineContent); - const relativePos = position - inlineNode.startIndex; + const inlineContent = inlineNode.text + const inlineTree = inlineParser.parse(inlineContent) + const relativePos = position - inlineNode.startIndex // Check if position is inside any code_span - const codeSpans = inlineTree.rootNode.descendantsOfType('code_span'); + const codeSpans = inlineTree.rootNode.descendantsOfType('code_span') for (const span of codeSpans) { if (relativePos >= span.startIndex && relativePos < span.endIndex) { - return true; + return true } } } } - return false; + return false } -/** - * Detect active inline styles at the given position range. - * Checks both the inline tree and block tree for style nodes. - */ +// Detect active inline styles at the given position range. +// Checks both the inline tree and block tree for style nodes. export function detectActiveStyles( node: Parser.SyntaxNode, startIdx: number, @@ -201,52 +187,52 @@ export function detectActiveStyles( currentTree: Parser.Tree | null, inlineParser: Parser | null ): string[] { - const styles: Set = new Set(); - let current: Parser.SyntaxNode | null = node; + const styles: Set = new Set() + let current: Parser.SyntaxNode | null = node // First, find the inline node from the BLOCK tree (not the inline tree) // to get document-relative positions if (currentTree) { - const blockInlineNode = findInlineNodeAtPosition(currentTree.rootNode, startIdx); + const blockInlineNode = findInlineNodeAtPosition(currentTree.rootNode, startIdx) if (blockInlineNode && inlineParser) { - const inlineContent = blockInlineNode.text; - const inlineTree = inlineParser.parse(inlineContent); + const inlineContent = blockInlineNode.text + const inlineTree = inlineParser.parse(inlineContent) if (inlineTree) { // Calculate relative position within the inline content - const relativeStart = startIdx - blockInlineNode.startIndex; - const relativeEnd = endIdx - blockInlineNode.startIndex; + const relativeStart = startIdx - blockInlineNode.startIndex + const relativeEnd = endIdx - blockInlineNode.startIndex // Check if our range overlaps with any inline style nodes - const codeSpans = inlineTree.rootNode.descendantsOfType('code_span'); + const codeSpans = inlineTree.rootNode.descendantsOfType('code_span') for (const span of codeSpans) { if (span.startIndex < relativeEnd && span.endIndex > relativeStart) { - styles.add('code'); - break; + styles.add('code') + break } } - const emphases = inlineTree.rootNode.descendantsOfType('emphasis'); + const emphases = inlineTree.rootNode.descendantsOfType('emphasis') for (const span of emphases) { if (span.startIndex < relativeEnd && span.endIndex > relativeStart) { - styles.add('italic'); - break; + styles.add('italic') + break } } - const strongs = inlineTree.rootNode.descendantsOfType('strong_emphasis'); + const strongs = inlineTree.rootNode.descendantsOfType('strong_emphasis') for (const span of strongs) { if (span.startIndex < relativeEnd && span.endIndex > relativeStart) { - styles.add('bold'); - break; + styles.add('bold') + break } } - const strikethroughs = inlineTree.rootNode.descendantsOfType('strikethrough'); + const strikethroughs = inlineTree.rootNode.descendantsOfType('strikethrough') for (const span of strikethroughs) { if (span.startIndex < relativeEnd && span.endIndex > relativeStart) { - styles.add('strikethrough'); - break; + styles.add('strikethrough') + break } } } @@ -256,18 +242,18 @@ export function detectActiveStyles( // Walk up the block tree for block-level styles while (current) { if (current.type === 'strong_emphasis' || current.type === 'strong') { - styles.add('bold'); + styles.add('bold') } else if (current.type === 'emphasis' || current.type === 'em') { - styles.add('italic'); + styles.add('italic') } else if (current.type === 'code_span') { - styles.add('code'); + styles.add('code') } else if (current.type === 'strikethrough') { - styles.add('strikethrough'); + styles.add('strikethrough') } // Skip inline node processing here - we already handled it above - current = current.parent; + current = current.parent } - return Array.from(styles); + return Array.from(styles) } diff --git a/src/tree-sitter/inline-extractors.ts b/src/tree-sitter/inline-extractors.ts index 05c3e40..bc35707 100644 --- a/src/tree-sitter/inline-extractors.ts +++ b/src/tree-sitter/inline-extractors.ts @@ -1,12 +1,10 @@ -import type { Parser } from 'web-tree-sitter'; -import type { StreamingChunk, BlockInfo, InlineStyleConfig, INLINE_STYLE_CONFIGS } from './types.js'; -import { findInlineNodeAtPosition } from './tree-navigation.js'; -import { createChunkFromBlockInfo } from './segment-builder.js'; - -/** - * Generic inline style segment extractor. - * Extracts segments with proper prefix/content/suffix handling for any inline style. - */ +import type { Parser } from 'web-tree-sitter' +import type { StreamingChunk, BlockInfo, InlineStyleConfig, INLINE_STYLE_CONFIGS } from './types.js' +import { findInlineNodeAtPosition } from './tree-navigation.js' +import { createChunkFromBlockInfo } from './segment-builder.js' + +// Generic inline style segment extractor. +// Extracts segments with proper prefix/content/suffix handling for any inline style. function extractInlineStyleSegments( config: InlineStyleConfig, startByte: number, @@ -17,58 +15,58 @@ function extractInlineStyleSegments( inlineParser: Parser, useDescendants: boolean = false ): StreamingChunk[] { - const segments: StreamingChunk[] = []; + const segments: StreamingChunk[] = [] - const inlineNode = findInlineNodeAtPosition(currentTree.rootNode, startByte); + const inlineNode = findInlineNodeAtPosition(currentTree.rootNode, startByte) // Check for both 'inline' and 'pipe_table_cell' if (!inlineNode || (inlineNode.type !== 'inline' && inlineNode.type !== 'pipe_table_cell')) { - throw new Error(`Tree-sitter inline node required for ${config.styleName} segment extraction`); + throw new Error(`Tree-sitter inline node required for ${config.styleName} segment extraction`) } - const inlineContent = inlineNode.text; - const inlineTree = inlineParser.parse(inlineContent); + const inlineContent = inlineNode.text + const inlineTree = inlineParser.parse(inlineContent) - const relativeStart = startByte - inlineNode.startIndex; - const relativeEnd = endByte - inlineNode.startIndex; + const relativeStart = startByte - inlineNode.startIndex + const relativeEnd = endByte - inlineNode.startIndex - const styleNodes = inlineTree.rootNode.descendantsOfType(config.nodeType); + const styleNodes = inlineTree.rootNode.descendantsOfType(config.nodeType) // Check if any style node actually overlaps with our range for (const styleNode of styleNodes) { // Check overlap if (styleNode.startIndex < relativeEnd && styleNode.endIndex > relativeStart) { // Get delimiters - either children or descendants based on config - let delimiters: Parser.SyntaxNode[]; + let delimiters: Parser.SyntaxNode[] if (useDescendants) { delimiters = styleNode.descendantsOfType(config.delimiterType) - .sort((a: Parser.SyntaxNode, b: Parser.SyntaxNode) => a.startIndex - b.startIndex); + .sort((a: Parser.SyntaxNode, b: Parser.SyntaxNode) => a.startIndex - b.startIndex) } else { - delimiters = styleNode.children.filter((c: Parser.SyntaxNode) => c.type === config.delimiterType); + delimiters = styleNode.children.filter((c: Parser.SyntaxNode) => c.type === config.delimiterType) } if (delimiters.length >= config.minDelimiters) { // Calculate content boundaries based on delimiter positions - let openingEnd: number; - let closingStart: number; + let openingEnd: number + let closingStart: number if (config.minDelimiters === 2) { // Simple case: single delimiter on each side (inline code, italic) - openingEnd = delimiters[0].endIndex; - closingStart = delimiters[delimiters.length - 1].startIndex; + openingEnd = delimiters[0].endIndex + closingStart = delimiters[delimiters.length - 1].startIndex } else { // Complex case: multiple delimiter characters (bold **, strikethrough ~~) - openingEnd = delimiters[1].endIndex; - closingStart = delimiters[delimiters.length - 2].startIndex; + openingEnd = delimiters[1].endIndex + closingStart = delimiters[delimiters.length - 2].startIndex } // 1. Prefix (Text before style span) if (styleNode.startIndex > relativeStart) { - const intersectionStart = Math.max(0, relativeStart); - const intersectionEnd = Math.min(styleNode.startIndex, relativeEnd); + const intersectionStart = Math.max(0, relativeStart) + const intersectionEnd = Math.min(styleNode.startIndex, relativeEnd) if (intersectionStart < intersectionEnd) { - const prefixText = inlineContent.substring(intersectionStart, intersectionEnd); + const prefixText = inlineContent.substring(intersectionStart, intersectionEnd) if (prefixText) { segments.push(createChunkFromBlockInfo( prefixText, @@ -76,22 +74,22 @@ function extractInlineStyleSegments( blockInfo, false, prefixText.includes('\n') - )); + )) } } } // 2. Styled Content (without markers) - const contentOverlapStart = Math.max(openingEnd, relativeStart); - const contentOverlapEnd = Math.min(closingStart, relativeEnd); + const contentOverlapStart = Math.max(openingEnd, relativeStart) + const contentOverlapEnd = Math.min(closingStart, relativeEnd) if (contentOverlapStart < contentOverlapEnd) { - const styledText = inlineContent.substring(contentOverlapStart, contentOverlapEnd); + const styledText = inlineContent.substring(contentOverlapStart, contentOverlapEnd) if (styledText) { // Ensure the style is present - const styledStyles = [...baseStyles]; + const styledStyles = [...baseStyles] if (styledStyles.indexOf(config.styleName) === -1) { - styledStyles.push(config.styleName); + styledStyles.push(config.styleName) } segments.push(createChunkFromBlockInfo( @@ -100,17 +98,17 @@ function extractInlineStyleSegments( blockInfo, false, styledText.includes('\n') - )); + )) } } // 3. Suffix (Text after style span) if (styleNode.endIndex < relativeEnd) { - const suffixStart = Math.max(styleNode.endIndex, relativeStart); - const suffixEnd = relativeEnd; + const suffixStart = Math.max(styleNode.endIndex, relativeStart) + const suffixEnd = relativeEnd if (suffixStart < suffixEnd) { - const suffixText = inlineContent.substring(suffixStart, suffixEnd); + const suffixText = inlineContent.substring(suffixStart, suffixEnd) if (suffixText) { segments.push(createChunkFromBlockInfo( suffixText, @@ -118,22 +116,20 @@ function extractInlineStyleSegments( blockInfo, false, suffixText.includes('\n') - )); + )) } } } - return segments; + return segments } } } - throw new Error(`Tree-sitter inline node required for ${config.styleName} segment extraction`); + throw new Error(`Tree-sitter inline node required for ${config.styleName} segment extraction`) } -/** - * Extract inline code segments, stripping backtick delimiters. - */ +// Extract inline code segments, stripping backtick delimiters. export function getInlineCodeSegments( content: string, node: Parser.SyntaxNode, @@ -149,17 +145,15 @@ export function getInlineCodeSegments( nodeType: 'code_span', delimiterType: 'code_span_delimiter', minDelimiters: 2, - }; + } return extractInlineStyleSegments( config, startByte, endByte, baseStyles, blockInfo, currentTree, inlineParser, false - ); + ) } -/** - * Extract bold segments, stripping ** delimiters. - */ +// Extract bold segments, stripping ** delimiters. export function getBoldSegments( content: string, node: Parser.SyntaxNode, @@ -175,17 +169,15 @@ export function getBoldSegments( nodeType: 'strong_emphasis', delimiterType: 'emphasis_delimiter', minDelimiters: 4, - }; + } return extractInlineStyleSegments( config, startByte, endByte, baseStyles, blockInfo, currentTree, inlineParser, false - ); + ) } -/** - * Extract italic segments, stripping * or _ delimiters. - */ +// Extract italic segments, stripping * or _ delimiters. export function getItalicSegments( content: string, node: Parser.SyntaxNode, @@ -201,17 +193,15 @@ export function getItalicSegments( nodeType: 'emphasis', delimiterType: 'emphasis_delimiter', minDelimiters: 2, - }; + } return extractInlineStyleSegments( config, startByte, endByte, baseStyles, blockInfo, currentTree, inlineParser, false - ); + ) } -/** - * Extract strikethrough segments, stripping ~~ delimiters. - */ +// Extract strikethrough segments, stripping ~~ delimiters. export function getStrikethroughSegments( content: string, node: Parser.SyntaxNode, @@ -227,11 +217,11 @@ export function getStrikethroughSegments( nodeType: 'strikethrough', delimiterType: 'emphasis_delimiter', minDelimiters: 4, - }; + } // Strikethrough uses descendants for delimiters (they can be nested) return extractInlineStyleSegments( config, startByte, endByte, baseStyles, blockInfo, currentTree, inlineParser, true - ); + ) } diff --git a/src/tree-sitter/segment-builder.ts b/src/tree-sitter/segment-builder.ts index 7137839..34e4fbe 100644 --- a/src/tree-sitter/segment-builder.ts +++ b/src/tree-sitter/segment-builder.ts @@ -1,8 +1,6 @@ -import type { StreamingChunk, StreamingSegment, BlockInfo } from './types.js'; +import type { StreamingChunk, StreamingSegment, BlockInfo } from './types.js' -/** - * Create a StreamingSegment with consistent defaults. - */ +// Create a StreamingSegment with consistent defaults. export function createSegment( segment: string, styles: string[], @@ -10,9 +8,9 @@ export function createSegment( isBlockDefining: boolean, isProcessingNewLine: boolean, options?: { - level?: number; - language?: string; - blockId?: number; + level?: number + language?: string + blockId?: number } ): StreamingSegment { const result: StreamingSegment = { @@ -21,34 +19,30 @@ export function createSegment( type, isBlockDefining, isProcessingNewLine, - }; + } if (options?.level !== undefined) { - result.level = options.level; + result.level = options.level } if (options?.language !== undefined) { - result.language = options.language; + result.language = options.language } if (options?.blockId !== undefined) { - result.blockId = options.blockId; + result.blockId = options.blockId } - return result; + return result } -/** - * Create a StreamingChunk with STREAMING status. - */ +// Create a StreamingChunk with STREAMING status. export function createStreamingChunk(segment: StreamingSegment): StreamingChunk { return { status: 'STREAMING', segment, - }; + } } -/** - * Create a segment from block info with common patterns. - */ +// Create a segment from block info with common patterns. export function createSegmentFromBlockInfo( text: string, styles: string[], @@ -67,12 +61,10 @@ export function createSegmentFromBlockInfo( language: blockInfo.language, blockId: blockInfo.id, } - ); + ) } -/** - * Create a streaming chunk from block info. - */ +// Create a streaming chunk from block info. export function createChunkFromBlockInfo( text: string, styles: string[], @@ -82,12 +74,10 @@ export function createChunkFromBlockInfo( ): StreamingChunk { return createStreamingChunk( createSegmentFromBlockInfo(text, styles, blockInfo, isBlockDefining, isProcessingNewLine) - ); + ) } -/** - * Create a plain text paragraph segment. - */ +// Create a plain text paragraph segment. export function createPlainTextChunk(text: string, isBlockDefining: boolean = false): StreamingChunk { return createStreamingChunk({ segment: text, @@ -95,12 +85,10 @@ export function createPlainTextChunk(text: string, isBlockDefining: boolean = fa type: 'paragraph', isBlockDefining, isProcessingNewLine: text.includes('\n'), - }); + }) } -/** - * Create a code block segment. - */ +// Create a code block segment. export function createCodeBlockChunk( text: string, language: string = '', @@ -113,5 +101,5 @@ export function createCodeBlockChunk( isBlockDefining, isProcessingNewLine: text.includes('\n'), language, - }); + }) } diff --git a/src/tree-sitter/segment-generator.ts b/src/tree-sitter/segment-generator.ts index 9309c03..f8f8040 100644 --- a/src/tree-sitter/segment-generator.ts +++ b/src/tree-sitter/segment-generator.ts @@ -1,7 +1,7 @@ -import type { Parser } from 'web-tree-sitter'; -import type { StreamingChunk, BlockState, BlockInfo, HEADER_MARKER_LEVELS, SUPPRESSED_SYNTAX_TYPES } from './types.js'; -import { findActiveNodeAtPosition, findInlineNodeAtPosition, findBlockNode } from './tree-navigation.js'; -import { getBlockInfo, isNewBlock } from './block-detection.js'; +import type { Parser } from 'web-tree-sitter' +import type { StreamingChunk, BlockState, BlockInfo, HEADER_MARKER_LEVELS, SUPPRESSED_SYNTAX_TYPES } from './types.js' +import { findActiveNodeAtPosition, findInlineNodeAtPosition, findBlockNode } from './tree-navigation.js' +import { getBlockInfo, isNewBlock } from './block-detection.js' import { detectActiveStyles, hasCompleteCodeSpanAt, @@ -10,15 +10,15 @@ import { hasCompleteStrikethroughAt, hasUnmatchedItalicMarker, isInsideCodeBlock -} from './inline-detection.js'; -import { getHeaderContent, getCodeBlockContent } from './content-extraction.js'; +} from './inline-detection.js' +import { getHeaderContent, getCodeBlockContent } from './content-extraction.js' import { getInlineCodeSegments, getBoldSegments, getItalicSegments, getStrikethroughSegments -} from './inline-extractors.js'; -import { createStreamingChunk, createChunkFromBlockInfo, createPlainTextChunk, createCodeBlockChunk } from './segment-builder.js'; +} from './inline-extractors.js' +import { createStreamingChunk, createChunkFromBlockInfo, createPlainTextChunk, createCodeBlockChunk } from './segment-builder.js' // Re-import the constant that we need locally const HEADER_MARKER_LEVELS_LOCAL: Record = { @@ -28,192 +28,190 @@ const HEADER_MARKER_LEVELS_LOCAL: Record = { 'atx_h4_marker': 4, 'atx_h5_marker': 5, 'atx_h6_marker': 6, -}; +} const SUPPRESSED_SYNTAX_TYPES_LOCAL = [ 'list_marker_minus', 'list_marker_plus', 'list_marker_star', 'list_marker_dot', 'list_marker_parenthesis', '|' // Table pipe delimiters -]; +] export interface SegmentGeneratorState { - pendingInlineContent: string; - pendingInlineStartIndex: number; - currentBlock: BlockState | null; + pendingInlineContent: string + pendingInlineStartIndex: number + currentBlock: BlockState | null } export interface SegmentGeneratorContext { - content: string; - currentTree: Parser.Tree; - inlineParser: Parser | null; - state: SegmentGeneratorState; + content: string + currentTree: Parser.Tree + inlineParser: Parser | null + state: SegmentGeneratorState } -/** - * Generate segments for a range of content. - * This is the main segment generation function that handles all the complex logic. - */ +// Generate segments for a range of content. +// This is the main segment generation function that handles all the complex logic. export function generateSegments( fromIndex: number, toIndex: number, context: SegmentGeneratorContext ): { segments: StreamingChunk[]; state: SegmentGeneratorState } { - const { content, currentTree, inlineParser } = context; - let state = { ...context.state }; + const { content, currentTree, inlineParser } = context + let state = { ...context.state } if (!currentTree) { - return { segments: [], state }; + return { segments: [], state } } - const segments: StreamingChunk[] = []; - let newContent = content.substring(fromIndex, toIndex); - let actualFromIndex = fromIndex; - let actualToIndex = toIndex; + const segments: StreamingChunk[] = [] + let newContent = content.substring(fromIndex, toIndex) + let actualFromIndex = fromIndex + let actualToIndex = toIndex // Check if we have pending inline content from previous incomplete structure if (state.pendingInlineContent) { // Prepend pending content - newContent = state.pendingInlineContent + newContent; - actualFromIndex = state.pendingInlineStartIndex; - state.pendingInlineContent = ''; + newContent = state.pendingInlineContent + newContent + actualFromIndex = state.pendingInlineStartIndex + state.pendingInlineContent = '' } // Check if current content has unmatched inline delimiters - const inlineNode = findInlineNodeAtPosition(currentTree.rootNode, actualFromIndex); + const inlineNode = findInlineNodeAtPosition(currentTree.rootNode, actualFromIndex) if (inlineNode && inlineParser) { - const inlineContent = inlineNode.text; - const inlineTree = inlineParser.parse(inlineContent); + const inlineContent = inlineNode.text + const inlineTree = inlineParser.parse(inlineContent) // Count the range in the new portion - const newPortionStart = actualFromIndex - inlineNode.startIndex; - const newPortionEnd = actualToIndex - inlineNode.startIndex; - const newPortion = inlineContent.substring(Math.max(0, newPortionStart), newPortionEnd); + const newPortionStart = actualFromIndex - inlineNode.startIndex + const newPortionEnd = actualToIndex - inlineNode.startIndex + const newPortion = inlineContent.substring(Math.max(0, newPortionStart), newPortionEnd) // Check for unmatched backtick if (newPortion.includes('`')) { - const hasCompleteCodeSpan = hasCompleteCodeSpanAt(inlineTree.rootNode, newPortionStart, newPortionEnd); + const hasCompleteCodeSpan = hasCompleteCodeSpanAt(inlineTree.rootNode, newPortionStart, newPortionEnd) if (!hasCompleteCodeSpan) { - state.pendingInlineContent = newContent; - state.pendingInlineStartIndex = actualFromIndex; - return { segments, state }; + state.pendingInlineContent = newContent + state.pendingInlineStartIndex = actualFromIndex + return { segments, state } } } // Check for unmatched bold markers if (newPortion.includes('**')) { - const hasCompleteBold = hasCompleteBoldAt(inlineTree.rootNode, newPortionStart, newPortionEnd); + const hasCompleteBold = hasCompleteBoldAt(inlineTree.rootNode, newPortionStart, newPortionEnd) if (!hasCompleteBold) { - state.pendingInlineContent = newContent; - state.pendingInlineStartIndex = actualFromIndex; - return { segments, state }; + state.pendingInlineContent = newContent + state.pendingInlineStartIndex = actualFromIndex + return { segments, state } } } // Check for unmatched italic markers (skip if inside code block) - const insideCodeBlock = isInsideCodeBlock(currentTree.rootNode, actualFromIndex, currentTree, inlineParser); + const insideCodeBlock = isInsideCodeBlock(currentTree.rootNode, actualFromIndex, currentTree, inlineParser) if (!insideCodeBlock) { - const hasUnmatchedItalic = hasUnmatchedItalicMarker(newPortion, inlineParser); + const hasUnmatchedItalic = hasUnmatchedItalicMarker(newPortion, inlineParser) if (hasUnmatchedItalic) { - const hasCompleteItalic = hasCompleteItalicAt(inlineTree.rootNode, newPortionStart, newPortionEnd); + const hasCompleteItalic = hasCompleteItalicAt(inlineTree.rootNode, newPortionStart, newPortionEnd) if (!hasCompleteItalic) { - state.pendingInlineContent = newContent; - state.pendingInlineStartIndex = actualFromIndex; - return { segments, state }; + state.pendingInlineContent = newContent + state.pendingInlineStartIndex = actualFromIndex + return { segments, state } } } } // Check for unmatched strikethrough markers if (newPortion.includes('~~')) { - const hasCompleteStrikethrough = hasCompleteStrikethroughAt(inlineTree.rootNode, newPortionStart, newPortionEnd); + const hasCompleteStrikethrough = hasCompleteStrikethroughAt(inlineTree.rootNode, newPortionStart, newPortionEnd) if (!hasCompleteStrikethrough) { - state.pendingInlineContent = newContent; - state.pendingInlineStartIndex = actualFromIndex; - return { segments, state }; + state.pendingInlineContent = newContent + state.pendingInlineStartIndex = actualFromIndex + return { segments, state } } } } // Skip empty content if (!newContent) { - return { segments, state }; + return { segments, state } } // Find the deepest node containing the new content position - const nodeAtPosition = findActiveNodeAtPosition(currentTree.rootNode, actualFromIndex); + const nodeAtPosition = findActiveNodeAtPosition(currentTree.rootNode, actualFromIndex) if (!nodeAtPosition) { // If no node found, treat as plain text return { segments: [createPlainTextChunk(newContent)], state - }; + } } // Check if the node is a suppressed syntax type if (SUPPRESSED_SYNTAX_TYPES_LOCAL.indexOf(nodeAtPosition.type) !== -1) { - return { segments, state }; + return { segments, state } } // Check if we're inside a table delimiter row - let currentForDelimiter: Parser.SyntaxNode | null = nodeAtPosition; + let currentForDelimiter: Parser.SyntaxNode | null = nodeAtPosition while (currentForDelimiter) { if (currentForDelimiter.type === 'pipe_table_delimiter_row' || currentForDelimiter.type === 'pipe_table_delimiter_cell') { - return { segments, state }; + return { segments, state } } - currentForDelimiter = currentForDelimiter.parent; + currentForDelimiter = currentForDelimiter.parent } // Determine the block type and properties - const blockInfo = getBlockInfo(nodeAtPosition); + const blockInfo = getBlockInfo(nodeAtPosition) // Check if we're starting a new block - const isNewBlockFlag = isNewBlock(blockInfo, nodeAtPosition, state.currentBlock); + const isNewBlockFlag = isNewBlock(blockInfo, nodeAtPosition, state.currentBlock) // Detect styles in the current context - const styles = detectActiveStyles(nodeAtPosition, actualFromIndex, actualToIndex, currentTree, inlineParser); + const styles = detectActiveStyles(nodeAtPosition, actualFromIndex, actualToIndex, currentTree, inlineParser) // Process content based on block type - let processedContent = newContent; + let processedContent = newContent if (blockInfo.type === 'header') { - const blockNode = findBlockNode(nodeAtPosition); + const blockNode = findBlockNode(nodeAtPosition) try { - processedContent = getHeaderContent(newContent, blockNode || undefined, actualFromIndex, actualToIndex); + processedContent = getHeaderContent(newContent, blockNode || undefined, actualFromIndex, actualToIndex) } catch (e) { - console.warn('[PARSER] Failed to extract header content, using raw:', e); - processedContent = newContent; + console.warn('[PARSER] Failed to extract header content, using raw:', e) + processedContent = newContent } // Don't emit if it's only markers if (processedContent.length === 0 || processedContent.trim().length === 0) { - state = updateBlockState(state, isNewBlockFlag, blockInfo, nodeAtPosition, actualToIndex, styles, false); - return { segments, state }; + state = updateBlockState(state, isNewBlockFlag, blockInfo, nodeAtPosition, actualToIndex, styles, false) + return { segments, state } } } else if (blockInfo.type === 'codeBlock') { - const blockNode = findBlockNode(nodeAtPosition); + const blockNode = findBlockNode(nodeAtPosition) try { - processedContent = getCodeBlockContent(newContent, blockNode || undefined, actualFromIndex, actualToIndex); + processedContent = getCodeBlockContent(newContent, blockNode || undefined, actualFromIndex, actualToIndex) } catch (e) { - console.warn('[PARSER] Failed to extract code block content, using raw:', e); - processedContent = newContent; + console.warn('[PARSER] Failed to extract code block content, using raw:', e) + processedContent = newContent } if (processedContent.length === 0) { - state = updateBlockState(state, isNewBlockFlag, blockInfo, nodeAtPosition, actualToIndex, styles, false); - return { segments, state }; + state = updateBlockState(state, isNewBlockFlag, blockInfo, nodeAtPosition, actualToIndex, styles, false) + return { segments, state } } } else if (blockInfo.type === 'paragraph') { // Handle incomplete header markers if (nodeAtPosition.type in HEADER_MARKER_LEVELS_LOCAL) { - return { segments, state }; + return { segments, state } } // Handle code fence detection in paragraph content - const result = handleCodeFenceInParagraph(newContent, actualFromIndex, actualToIndex, content, segments); + const result = handleCodeFenceInParagraph(newContent, actualFromIndex, actualToIndex, content, segments) if (result.handled) { - return { segments: result.segments, state }; + return { segments: result.segments, state } } } @@ -222,17 +220,17 @@ export function generateSegments( const inlineResult = processInlineStyles( processedContent, nodeAtPosition, actualFromIndex, actualToIndex, styles, blockInfo, isNewBlockFlag, state, currentTree, inlineParser! - ); + ) if (inlineResult.handled) { - return { segments: inlineResult.segments, state: inlineResult.state }; + return { segments: inlineResult.segments, state: inlineResult.state } } } // Determine effective block defining status - let effectiveIsBlockDefining = isNewBlockFlag; + let effectiveIsBlockDefining = isNewBlockFlag if (!isNewBlockFlag && state.currentBlock && !state.currentBlock.hasEmittedContent && state.currentBlock.type === blockInfo.type) { - effectiveIsBlockDefining = true; + effectiveIsBlockDefining = true } // Create and push the segment @@ -242,17 +240,15 @@ export function generateSegments( blockInfo, effectiveIsBlockDefining, newContent.includes('\n') - )); + )) // Update block tracking - state = updateBlockState(state, isNewBlockFlag, blockInfo, nodeAtPosition, actualToIndex, styles, true); + state = updateBlockState(state, isNewBlockFlag, blockInfo, nodeAtPosition, actualToIndex, styles, true) - return { segments, state }; + return { segments, state } } -/** - * Update the block state after processing content - */ +// Update the block state after processing content function updateBlockState( state: SegmentGeneratorState, isNewBlockFlag: boolean, @@ -262,7 +258,7 @@ function updateBlockState( styles: string[], hasEmittedContent: boolean ): SegmentGeneratorState { - const newState = { ...state }; + const newState = { ...state } if (isNewBlockFlag) { newState.currentBlock = { @@ -273,20 +269,18 @@ function updateBlockState( lastSegmentEnd: actualToIndex, styles: new Set(styles), hasEmittedContent - }; + } } else if (newState.currentBlock) { - newState.currentBlock = { ...newState.currentBlock }; - newState.currentBlock.lastSegmentEnd = actualToIndex; - newState.currentBlock.hasEmittedContent = newState.currentBlock.hasEmittedContent || hasEmittedContent; - styles.forEach(s => newState.currentBlock!.styles.add(s)); + newState.currentBlock = { ...newState.currentBlock } + newState.currentBlock.lastSegmentEnd = actualToIndex + newState.currentBlock.hasEmittedContent = newState.currentBlock.hasEmittedContent || hasEmittedContent + styles.forEach(s => newState.currentBlock!.styles.add(s)) } - return newState; + return newState } -/** - * Handle code fence detection when tree-sitter sees it as paragraph - */ +// Handle code fence detection when tree-sitter sees it as paragraph function handleCodeFenceInParagraph( newContent: string, actualFromIndex: number, @@ -294,104 +288,102 @@ function handleCodeFenceInParagraph( content: string, existingSegments: StreamingChunk[] ): { handled: boolean; segments: StreamingChunk[] } { - const segments = [...existingSegments]; + const segments = [...existingSegments] // Check if new content contains a code fence opening - const codeFenceOpeningMatch = newContent.match(/```([a-zA-Z0-9]*)\n?/); + const codeFenceOpeningMatch = newContent.match(/```([a-zA-Z0-9]*)\n?/) if (!codeFenceOpeningMatch) { // Also check for content MIDDLE of a code block - const contentBeforeThis = content.substring(0, actualFromIndex); - const allFences = contentBeforeThis.match(/```/g) || []; - const isInsideCodeBlockContext = allFences.length % 2 === 1; + const contentBeforeThis = content.substring(0, actualFromIndex) + const allFences = contentBeforeThis.match(/```/g) || [] + const isInsideCodeBlockContext = allFences.length % 2 === 1 if (isInsideCodeBlockContext) { - const closingFenceIdx = newContent.indexOf('```'); + const closingFenceIdx = newContent.indexOf('```') if (closingFenceIdx === -1) { // No closing fence - emit as code block content return { handled: true, segments: [createCodeBlockChunk(newContent)] - }; + } } else { // Has closing fence - const codeContent = newContent.substring(0, closingFenceIdx); - const afterFence = newContent.substring(closingFenceIdx + 3); + const codeContent = newContent.substring(0, closingFenceIdx) + const afterFence = newContent.substring(closingFenceIdx + 3) if (codeContent.length > 0) { - segments.push(createCodeBlockChunk(codeContent)); + segments.push(createCodeBlockChunk(codeContent)) } - const textAfterFence = afterFence.replace(/^\n/, ''); + const textAfterFence = afterFence.replace(/^\n/, '') if (textAfterFence.length > 0) { - segments.push(createPlainTextChunk(textAfterFence, true)); + segments.push(createPlainTextChunk(textAfterFence, true)) } - return { handled: true, segments }; + return { handled: true, segments } } } - return { handled: false, segments }; + return { handled: false, segments } } - const fenceStart = newContent.indexOf(codeFenceOpeningMatch[0]); - const fenceLanguage = codeFenceOpeningMatch[1] || ''; - const fenceMarker = codeFenceOpeningMatch[0]; + const fenceStart = newContent.indexOf(codeFenceOpeningMatch[0]) + const fenceLanguage = codeFenceOpeningMatch[1] || '' + const fenceMarker = codeFenceOpeningMatch[0] // Check for closing fence - const positionOfFence = actualFromIndex + fenceStart; - const contentFromFence = content.substring(positionOfFence); - const closingFenceIdx = contentFromFence.substring(fenceMarker.length).indexOf('```'); + const positionOfFence = actualFromIndex + fenceStart + const contentFromFence = content.substring(positionOfFence) + const closingFenceIdx = contentFromFence.substring(fenceMarker.length).indexOf('```') // Content BEFORE the fence - const contentBeforeFence = newContent.substring(0, fenceStart); + const contentBeforeFence = newContent.substring(0, fenceStart) if (closingFenceIdx === -1) { // No closing fence yet - emit content before fence and buffer the rest if (contentBeforeFence.trim().length > 0) { - segments.push(createPlainTextChunk(contentBeforeFence)); + segments.push(createPlainTextChunk(contentBeforeFence)) } // This would need state management - return partial result // The caller should handle buffering - return { handled: true, segments }; + return { handled: true, segments } } // Complete code block structure if (contentBeforeFence.trim().length > 0) { - segments.push(createPlainTextChunk(contentBeforeFence)); + segments.push(createPlainTextChunk(contentBeforeFence)) } - const contentAfterOpeningFence = newContent.substring(fenceStart + fenceMarker.length); - const closingFenceInContent = contentAfterOpeningFence.indexOf('```'); + const contentAfterOpeningFence = newContent.substring(fenceStart + fenceMarker.length) + const closingFenceInContent = contentAfterOpeningFence.indexOf('```') - let codeContent: string; + let codeContent: string if (closingFenceInContent === -1) { - codeContent = contentAfterOpeningFence; + codeContent = contentAfterOpeningFence } else { - codeContent = contentAfterOpeningFence.substring(0, closingFenceInContent); + codeContent = contentAfterOpeningFence.substring(0, closingFenceInContent) } - codeContent = codeContent.replace(/^\n/, ''); + codeContent = codeContent.replace(/^\n/, '') if (codeContent.length > 0) { - segments.push(createCodeBlockChunk(codeContent, fenceLanguage, true)); + segments.push(createCodeBlockChunk(codeContent, fenceLanguage, true)) } if (closingFenceInContent !== -1) { - const afterClosingFence = contentAfterOpeningFence.substring(closingFenceInContent + 3); - const textAfterFence = afterClosingFence.replace(/^\n/, ''); + const afterClosingFence = contentAfterOpeningFence.substring(closingFenceInContent + 3) + const textAfterFence = afterClosingFence.replace(/^\n/, '') if (textAfterFence.trim().length > 0) { - segments.push(createPlainTextChunk(textAfterFence, true)); + segments.push(createPlainTextChunk(textAfterFence, true)) } } - return { handled: true, segments }; + return { handled: true, segments } } -/** - * Process inline styles and return segments if applicable - */ +// Process inline styles and return segments if applicable function processInlineStyles( processedContent: string, nodeAtPosition: Parser.SyntaxNode, @@ -405,49 +397,49 @@ function processInlineStyles( inlineParser: Parser ): { handled: boolean; segments: StreamingChunk[]; state: SegmentGeneratorState } { const styleHandlers: Array<{ - style: string; + style: string extractor: (content: string, node: Parser.SyntaxNode, startByte: number, endByte: number, baseStyles: string[], blockInfo: BlockInfo, currentTree: Parser.Tree, inlineParser: Parser) => StreamingChunk[] }> = [ { style: 'code', extractor: getInlineCodeSegments }, { style: 'bold', extractor: getBoldSegments }, { style: 'italic', extractor: getItalicSegments }, { style: 'strikethrough', extractor: getStrikethroughSegments }, - ]; + ] for (const { style, extractor } of styleHandlers) { if (styles.indexOf(style) !== -1) { - let splitSegments: StreamingChunk[] = []; + let splitSegments: StreamingChunk[] = [] try { splitSegments = extractor( processedContent, nodeAtPosition, actualFromIndex, actualToIndex, styles, blockInfo, currentTree, inlineParser - ); + ) } catch (e) { - console.warn(`[PARSER] Failed to extract ${style} segments, will use default processing:`, e); - continue; + console.warn(`[PARSER] Failed to extract ${style} segments, will use default processing:`, e) + continue } if (splitSegments.length > 0) { // Determine effective block defining - let effectiveIsBlockDefining = isNewBlockFlag; + let effectiveIsBlockDefining = isNewBlockFlag if (!isNewBlockFlag && state.currentBlock && !state.currentBlock.hasEmittedContent && state.currentBlock.type === blockInfo.type) { - effectiveIsBlockDefining = true; + effectiveIsBlockDefining = true } // Apply block defining flag to first segment splitSegments.forEach((seg, index) => { if (index === 0 && seg.segment) { - seg.segment.isBlockDefining = effectiveIsBlockDefining; + seg.segment.isBlockDefining = effectiveIsBlockDefining } - }); + }) // Update state - const newState = updateBlockState(state, isNewBlockFlag, blockInfo, nodeAtPosition, actualToIndex, styles, true); + const newState = updateBlockState(state, isNewBlockFlag, blockInfo, nodeAtPosition, actualToIndex, styles, true) - return { handled: true, segments: splitSegments, state: newState }; + return { handled: true, segments: splitSegments, state: newState } } } } - return { handled: false, segments: [], state }; + return { handled: false, segments: [], state } } diff --git a/src/tree-sitter/tree-navigation.ts b/src/tree-sitter/tree-navigation.ts index 466f859..2b78452 100644 --- a/src/tree-sitter/tree-navigation.ts +++ b/src/tree-sitter/tree-navigation.ts @@ -1,14 +1,12 @@ -import type { Parser } from 'web-tree-sitter'; -import { BLOCK_TYPES } from './types.js'; +import type { Parser } from 'web-tree-sitter' +import { BLOCK_TYPES } from './types.js' -/** - * Find the deepest node in the BLOCK tree that contains the given position. - * Uses exclusive end: position must be strictly less than endIndex. - * This ensures we find nodes that START at position, not ones that END at position. - */ +// Find the deepest node in the BLOCK tree that contains the given position. +// Uses exclusive end: position must be strictly less than endIndex. +// This ensures we find nodes that START at position, not ones that END at position. export function findActiveNodeAtPosition(node: Parser.SyntaxNode, position: number): Parser.SyntaxNode | null { if (position < node.startIndex || position >= node.endIndex) { - return null; + return null } // Do not dive into inline nodes here. @@ -17,73 +15,67 @@ export function findActiveNodeAtPosition(node: Parser.SyntaxNode, position: numb // This ensures getBlockInfo always finds the correct block parent in the main tree. for (const child of node.children) { - const childResult = findActiveNodeAtPosition(child, position); + const childResult = findActiveNodeAtPosition(child, position) if (childResult) { - return childResult; + return childResult } } - return node; + return node } -/** - * Find a node in the tree that contains the given position. - * Uses inclusive end bounds. - */ +// Find a node in the tree that contains the given position. +// Uses inclusive end bounds. export function findNodeInTree(node: Parser.SyntaxNode, position: number): Parser.SyntaxNode | null { if (position < node.startIndex || position > node.endIndex) { - return null; + return null } for (const child of node.children) { - const result = findNodeInTree(child, position); + const result = findNodeInTree(child, position) if (result) { - return result; + return result } } - return node; + return node } -/** - * Find an inline node that contains the given position. - * Returns the 'inline' or 'pipe_table_cell' node if found. - */ +// Find an inline node that contains the given position. +// Returns the 'inline' or 'pipe_table_cell' node if found. export function findInlineNodeAtPosition(node: Parser.SyntaxNode, position: number): Parser.SyntaxNode | null { // If this node is an inline node that contains the position, return it if (node.type === 'inline' && position >= node.startIndex && position < node.endIndex) { - return node; + return node } // Also check for pipe_table_cell - table cells contain inline content but without 'inline' wrapper if (node.type === 'pipe_table_cell' && position >= node.startIndex && position < node.endIndex) { - return node; + return node } // Search children for (const child of node.children) { - const result = findInlineNodeAtPosition(child, position); + const result = findInlineNodeAtPosition(child, position) if (result) { - return result; + return result } } - return null; + return null } -/** - * Find the block-level node that contains the given node. - * Walks up the tree until a block type is found. - */ +// Find the block-level node that contains the given node. +// Walks up the tree until a block type is found. export function findBlockNode(node: Parser.SyntaxNode): Parser.SyntaxNode | null { - let current: Parser.SyntaxNode | null = node; + let current: Parser.SyntaxNode | null = node while (current) { if (BLOCK_TYPES.indexOf(current.type as typeof BLOCK_TYPES[number]) !== -1) { - return current; + return current } - current = current.parent; + current = current.parent } - return null; + return null } diff --git a/src/tree-sitter/types.ts b/src/tree-sitter/types.ts index fd8a3ad..efbcecb 100644 --- a/src/tree-sitter/types.ts +++ b/src/tree-sitter/types.ts @@ -1,9 +1,7 @@ -import type { Parser } from 'web-tree-sitter'; +import type { Parser } from 'web-tree-sitter' -/** - * Lookup map for tree-sitter ATX header marker node types to their heading levels. - * Used for both level extraction and marker-only content detection. - */ +// Lookup map for tree-sitter ATX header marker node types to their heading levels. +// Used for both level extraction and marker-only content detection. export const HEADER_MARKER_LEVELS: Record = { 'atx_h1_marker': 1, 'atx_h2_marker': 2, @@ -11,97 +9,79 @@ export const HEADER_MARKER_LEVELS: Record = { 'atx_h4_marker': 4, 'atx_h5_marker': 5, 'atx_h6_marker': 6, -}; +} -/** - * Block types that are recognized by tree-sitter - */ +// Block types that are recognized by tree-sitter export const BLOCK_TYPES = [ 'atx_heading', 'paragraph', 'fenced_code_block', 'list_item', 'blockquote', 'pipe_table', 'pipe_table_header', 'pipe_table_row', 'pipe_table_cell' -] as const; +] as const -/** - * Syntax elements that should be suppressed (not emitted as content) - */ +// Syntax elements that should be suppressed (not emitted as content) export const SUPPRESSED_SYNTAX_TYPES = [ 'list_marker_minus', 'list_marker_plus', 'list_marker_star', 'list_marker_dot', 'list_marker_parenthesis', '|' // Table pipe delimiters -] as const; +] as const -/** - * Represents a parsed segment of streaming markdown content - */ +// Represents a parsed segment of streaming markdown content export interface StreamingSegment { - level?: number; - language?: string; - segment: string; - styles: string[]; - type: string; - isBlockDefining: boolean; - isProcessingNewLine: boolean; - blockId?: number; + level?: number + language?: string + segment: string + styles: string[] + type: string + isBlockDefining: boolean + isProcessingNewLine: boolean + blockId?: number } -/** - * A chunk of streaming data with status information - */ +// A chunk of streaming data with status information export interface StreamingChunk { - status: string; - segment?: StreamingSegment; + status: string + segment?: StreamingSegment } -/** - * Tracks the current block's state during parsing - */ +// Tracks the current block's state during parsing export interface BlockState { - type: string; - level?: number; - language?: string; - startIndex: number; - lastSegmentEnd: number; - styles: Set; - hasEmittedContent?: boolean; + type: string + level?: number + language?: string + startIndex: number + lastSegmentEnd: number + styles: Set + hasEmittedContent?: boolean } -/** - * Information about a block type extracted from the AST - */ +// Information about a block type extracted from the AST export interface BlockInfo { - type: string; - level?: number; - language?: string; - id?: number; + type: string + level?: number + language?: string + id?: number } -/** - * Context passed to inline style extractors - */ +// Context passed to inline style extractors export interface InlineExtractionContext { - content: string; - node: Parser.SyntaxNode; - startByte: number; - endByte: number; - baseStyles: string[]; - blockInfo: BlockInfo; - inlineParser: Parser; - currentTree: Parser.Tree; + content: string + node: Parser.SyntaxNode + startByte: number + endByte: number + baseStyles: string[] + blockInfo: BlockInfo + inlineParser: Parser + currentTree: Parser.Tree } -/** - * Configuration for a specific inline style type - */ +// Configuration for a specific inline style type export interface InlineStyleConfig { - styleName: string; - nodeType: string; - delimiterType: string; - minDelimiters: number; + styleName: string + nodeType: string + delimiterType: string + minDelimiters: number } -/** - * Predefined inline style configurations - */ +// Predefined inline style configurations export const INLINE_STYLE_CONFIGS: Record = { code: { styleName: 'code', @@ -127,4 +107,4 @@ export const INLINE_STYLE_CONFIGS: Record = { delimiterType: 'emphasis_delimiter', minDelimiters: 4, // ~~ on each side = 4 delimiter nodes }, -}; +} From 5e6afd37444386fd0218ffaf755b35ccfebe1d27 Mon Sep 17 00:00:00 2001 From: Dmitry Bondarenko Date: Wed, 21 Jan 2026 22:29:15 +0600 Subject: [PATCH 16/32] Replace Interface with Type --- src/tree-sitter/segment-generator.ts | 4 ++-- src/tree-sitter/types.ts | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/tree-sitter/segment-generator.ts b/src/tree-sitter/segment-generator.ts index f8f8040..bbbdeab 100644 --- a/src/tree-sitter/segment-generator.ts +++ b/src/tree-sitter/segment-generator.ts @@ -36,13 +36,13 @@ const SUPPRESSED_SYNTAX_TYPES_LOCAL = [ '|' // Table pipe delimiters ] -export interface SegmentGeneratorState { +export type SegmentGeneratorState = { pendingInlineContent: string pendingInlineStartIndex: number currentBlock: BlockState | null } -export interface SegmentGeneratorContext { +export type SegmentGeneratorContext = { content: string currentTree: Parser.Tree inlineParser: Parser | null diff --git a/src/tree-sitter/types.ts b/src/tree-sitter/types.ts index efbcecb..067660d 100644 --- a/src/tree-sitter/types.ts +++ b/src/tree-sitter/types.ts @@ -25,7 +25,7 @@ export const SUPPRESSED_SYNTAX_TYPES = [ ] as const // Represents a parsed segment of streaming markdown content -export interface StreamingSegment { +export type StreamingSegment = { level?: number language?: string segment: string @@ -37,13 +37,13 @@ export interface StreamingSegment { } // A chunk of streaming data with status information -export interface StreamingChunk { +export type StreamingChunk = { status: string segment?: StreamingSegment } // Tracks the current block's state during parsing -export interface BlockState { +export type BlockState = { type: string level?: number language?: string @@ -54,7 +54,7 @@ export interface BlockState { } // Information about a block type extracted from the AST -export interface BlockInfo { +export type BlockInfo = { type: string level?: number language?: string @@ -62,7 +62,7 @@ export interface BlockInfo { } // Context passed to inline style extractors -export interface InlineExtractionContext { +export type InlineExtractionContext = { content: string node: Parser.SyntaxNode startByte: number @@ -74,7 +74,7 @@ export interface InlineExtractionContext { } // Configuration for a specific inline style type -export interface InlineStyleConfig { +export type InlineStyleConfig = { styleName: string nodeType: string delimiterType: string From 021cb3c3ab797825dec944c1369fb9642b702210 Mon Sep 17 00:00:00 2001 From: Dmitry Bondarenko Date: Wed, 21 Jan 2026 22:35:54 +0600 Subject: [PATCH 17/32] Clean up --- src/markdown-stream-parser.ts | 14 +- src/state-machine/actions.ts | 16 +- src/tree-sitter-markdown-stream-parser.ts | 9 - ...ee-sitter-markdown-stream-parser_nodejs.ts | 481 ------------------ 4 files changed, 8 insertions(+), 512 deletions(-) delete mode 100644 src/tree-sitter-markdown-stream-parser_nodejs.ts diff --git a/src/markdown-stream-parser.ts b/src/markdown-stream-parser.ts index 83cdae6..6d497be 100644 --- a/src/markdown-stream-parser.ts +++ b/src/markdown-stream-parser.ts @@ -18,8 +18,6 @@ export class MarkdownStreamParser { MarkdownStreamParser.instances.set(instanceId, new MarkdownStreamParser()) // Save the instance, ensure it is available statically } - console.info(`\x1b[34mAiStreamParser ->\x1b[0m class.MarkdownStreamParser::\x1b[32mgetInstance\x1b[0m::instanceId: ${instanceId}, instances: ${MarkdownStreamParser.instances}`) - return MarkdownStreamParser.instances.get(instanceId) } @@ -32,8 +30,8 @@ export class MarkdownStreamParser { constructor() { this.tokensStreamProcessor = new TokensStreamBuffer() this.markdownStreamParser = new MarkdownStreamParserStateMachine() - this.unsubscribeFromProcessor = () => {} - this.unsubscribeFromStateMachine = () => {} + this.unsubscribeFromProcessor = () => { } + this.unsubscribeFromStateMachine = () => { } this.parsing = false this.tokenParseListeners = [] @@ -62,7 +60,7 @@ export class MarkdownStreamParser { return // Do not start parsing if it's already started } - this.notifyTokenParse({status: 'START_STREAM'}) + this.notifyTokenParse({ status: 'START_STREAM' }) // Subscribe to receive the completed segment from TokensStreamBuffer this.unsubscribeFromProcessor = this.tokensStreamProcessor.subscribeToSegmentCompletion((word: string) => { @@ -72,7 +70,7 @@ export class MarkdownStreamParser { // Subscribe to receive the parsed segment from TextStreamStateMachine this.unsubscribeFromStateMachine = this.markdownStreamParser.subscribeToParsedSegment((parsedSegment: any) => { - this.notifyTokenParse({status: 'STREAMING', segment: parsedSegment}) // Relay the parsed segment event + this.notifyTokenParse({ status: 'STREAMING', segment: parsedSegment }) // Relay the parsed segment event }) this.parsing = true @@ -82,8 +80,6 @@ export class MarkdownStreamParser { parseToken(chunk: string): Error | void { if (!this.parsing) { const error = new Error('Parser is not started.') - console.info(`\x1b[34mAiStreamParser ->\x1b[0m \x1b[31mclass.MarkdownStreamParser::parseToken::error\x1b[0m`, error) - return error } @@ -97,7 +93,7 @@ export class MarkdownStreamParser { this.unsubscribeFromProcessor() // Unsubscribe from the processor this.unsubscribeFromStateMachine() // Unsubscribe from the state machine this.parsing = false // Mark as not parsing - this.notifyTokenParse({status: 'END_STREAM'}) // Notify that the stream has ended + this.notifyTokenParse({ status: 'END_STREAM' }) // Notify that the stream has ended } } diff --git a/src/state-machine/actions.ts b/src/state-machine/actions.ts index 7db81fa..267c030 100644 --- a/src/state-machine/actions.ts +++ b/src/state-machine/actions.ts @@ -141,7 +141,7 @@ const applyInlineTextStyle = ( let parsedSegment = matchObject.content; - if(!matchObject.postfixedContent) { // If there's no postfixed content, add a space to the parsed segment + if (!matchObject.postfixedContent) { // If there's no postfixed content, add a space to the parsed segment parsedSegment = `${parsedSegment} `; } @@ -186,14 +186,6 @@ const setCodeBlock = ( } } -const debugParsedSegment = ( - context: Context, - event: ActionEvent, - params: ActionParams -): void => { - console.log({origin: params.origin, parsedSegment: context}) -} - const bufferBlockContent = ( context: Context, event: ActionEvent, @@ -273,8 +265,8 @@ const emitParsedSegment = ( type: context.blockType, isBlockDefining: context.isBlockDefining, isProcessingNewLine: context.isProcessingNewLine, - ...(isDebug && {content: context.blockContentBuffer}), - ...(isDebug && {origin: `${params.origin || null}`}), + ...(isDebug && { content: context.blockContentBuffer }), + ...(isDebug && { origin: `${params.origin || null}` }), } // Emit parsed segment @@ -306,8 +298,6 @@ const ACTIONS: Record = { 'buffer::codeBlockSegments': bufferCodeBlockSegments, 'emit::parsedSegment': emitParsedSegment, - - 'debug::parsedSegment': debugParsedSegment, } export const actionRunner = ( diff --git a/src/tree-sitter-markdown-stream-parser.ts b/src/tree-sitter-markdown-stream-parser.ts index ff4146d..5d5a8f7 100644 --- a/src/tree-sitter-markdown-stream-parser.ts +++ b/src/tree-sitter-markdown-stream-parser.ts @@ -83,7 +83,6 @@ export class MarkdownStreamParser { MarkdownStreamParser.instances.set(instanceId, instance) } - console.info(`\x1b[34mMarkdownStreamParser ->\x1b[0m getInstance::instanceId: ${instanceId}`) return MarkdownStreamParser.instances.get(instanceId)! } @@ -122,7 +121,6 @@ export class MarkdownStreamParser { } } - console.info(`Loading markdown WASM from: ${wasmPath}`) MarkdownStreamParser.markdownLanguage = await Language.load(wasmPath) // Load the inline language @@ -135,12 +133,9 @@ export class MarkdownStreamParser { } } - console.info(`Loading markdown-inline WASM from: ${inlineWasmPath}`) MarkdownStreamParser.markdownInlineLanguage = await Language.load(inlineWasmPath) MarkdownStreamParser.parserInitialized = true - console.info('✅ Tree-sitter markdown language loaded successfully') - console.info('✅ Tree-sitter markdown-inline language loaded successfully') } catch (error) { console.error('Failed to load tree-sitter-markdown WASM:', error) throw new Error(`Failed to initialize markdown parser: ${error}`) @@ -193,8 +188,6 @@ export class MarkdownStreamParser { this.parser.setLanguage(MarkdownStreamParser.markdownLanguage) this.inlineParser.setLanguage(MarkdownStreamParser.markdownInlineLanguage) - - console.info('Parser instance initialized with markdown and markdown-inline languages') } /** @@ -245,7 +238,6 @@ export class MarkdownStreamParser { }) this.parsing = true - console.info('\x1b[32mParser started\x1b[0m') } /** @@ -279,7 +271,6 @@ export class MarkdownStreamParser { this.notifyTokenParse({ status: 'END_STREAM' }) this.parsing = false - console.info('\x1b[32mParser stopped\x1b[0m') } /** diff --git a/src/tree-sitter-markdown-stream-parser_nodejs.ts b/src/tree-sitter-markdown-stream-parser_nodejs.ts deleted file mode 100644 index ce9e096..0000000 --- a/src/tree-sitter-markdown-stream-parser_nodejs.ts +++ /dev/null @@ -1,481 +0,0 @@ -import TokensStreamBuffer from './tokens-stream-buffer.ts' -import Parser from 'tree-sitter'; -import Markdown from '@tree-sitter-grammars/tree-sitter-markdown'; - -interface StreamingSegment { - level?: number; - segment: string; - styles: string[]; - type: string; - isBlockDefining: boolean; - isProcessingNewLine: boolean; -} - -export interface StreamingChunk { - status: string; - segment: StreamingSegment; -} - -interface BlockState { - type: string; - level?: number; - startIndex: number; - lastSegmentEnd: number; - styles: Set; -} - - -export class MarkdownStreamParser { - private static instances = new Map(); - - private parser: Parser; - private currentTree: Parser.Tree | null = null; - private content: string = ''; - private lastProcessedIndex: number = 0; - private currentBlock: BlockState | null = null; - private allSegments: StreamingChunk[] = []; - - // Integration with TokensStreamBuffer - private tokensStreamProcessor: TokensStreamBuffer; - private parsing: boolean = false; - private tokenParseListeners: Array<(chunk: StreamingChunk) => void> = []; - private unsubscribeFromProcessor: (() => void) | null = null; - - static getInstance(instanceId: string): MarkdownStreamParser { - if (!MarkdownStreamParser.instances.has(instanceId)) { - MarkdownStreamParser.instances.set(instanceId, new MarkdownStreamParser()); - } - - console.info(`\x1b[34mMarkdownStreamParser ->\x1b[0m getInstance::instanceId: ${instanceId}`); - return MarkdownStreamParser.instances.get(instanceId)!; - } - - static removeInstance(instanceId: string): void { - const instance = MarkdownStreamParser.instances.get(instanceId); - if (instance) { - instance.stopParsing(); - MarkdownStreamParser.instances.delete(instanceId); - } - } - - constructor() { - this.parser = new Parser(); - this.parser.setLanguage(Markdown); - this.tokensStreamProcessor = new TokensStreamBuffer(); - } - - /** - * Subscribe to parsed tokens/segments - * Returns an unsubscribe function - */ - subscribeToTokenParse(listener: (chunk: StreamingChunk, unsubscribe: () => void) => void): () => void { - const wrappedListener = (data: StreamingChunk) => { - listener(data, unsubscribe); - }; - - const unsubscribe = () => { - this.tokenParseListeners = this.tokenParseListeners.filter(l => l !== wrappedListener); - }; - - this.tokenParseListeners.push(wrappedListener); - return unsubscribe; - } - - /** - * Notify all subscribers about a parsed token - */ - private notifyTokenParse(chunk: StreamingChunk): void { - this.tokenParseListeners.forEach(listener => listener(chunk)); - } - - /** - * Start the parsing session - */ - startParsing(): void { - if (this.parsing) { - console.warn('Parser is already running'); - return; - } - - // Reset state - this.reset(); - - // Notify start - this.notifyTokenParse({ status: 'START_STREAM' }); - - // Subscribe to completed segments from TokensStreamBuffer - this.unsubscribeFromProcessor = this.tokensStreamProcessor.subscribeToSegmentCompletion((word: string) => { - // Process the completed word/segment through tree-sitter - const segments = this.processRawChunk(word); - - // Notify listeners about each segment - segments.forEach(segment => { - this.notifyTokenParse(segment); - }); - }); - - this.parsing = true; - console.info('\x1b[32mParser started\x1b[0m'); - } - - /** - * Parse a single token/chunk - */ - parseToken(chunk: string): Error | void { - if (!this.parsing) { - const error = new Error('Parser is not started. Call startParsing() first.'); - console.error('\x1b[31mMarkdownStreamParser::parseToken::error\x1b[0m', error.message); - return error; - } - - // Send chunk to the token buffer for processing - this.tokensStreamProcessor.receiveChunk(chunk); - } - - /** - * Stop parsing and cleanup - */ - stopParsing(): void { - if (!this.parsing) { - return; - } - - // Flush any remaining content in the buffer - this.tokensStreamProcessor.flushBuffer(); - - // Unsubscribe from token processor - if (this.unsubscribeFromProcessor) { - this.unsubscribeFromProcessor(); - this.unsubscribeFromProcessor = null; - } - - // Notify end - this.notifyTokenParse({ status: 'END_STREAM' }); - - this.parsing = false; - console.info('\x1b[32mParser stopped\x1b[0m'); - } - - /** - * Process raw chunk through tree-sitter - */ - private processRawChunk(chunk: string): StreamingChunk[] { - const oldLength = this.content.length; - - // Add chunk to content - this.content += chunk; - - // Update last processed index - this.lastProcessedIndex = this.content.length; - - // Parse the updated content - this.currentTree = this.parser.parse(this.content); - - // Generate segments for the new content - const newSegments = this.generateSegments(oldLength, this.content.length); - - // Store all segments for debugging - this.allSegments.push(...newSegments); - - return newSegments; - } - - private generateSegments(fromIndex: number, toIndex: number): StreamingChunk[] { - if (!this.currentTree) return []; - - const segments: StreamingChunk[] = []; - const newContent = this.content.substring(fromIndex, toIndex); - - // Skip empty content - if (!newContent) return segments; - - // First, check the content itself for markdown patterns - const contentType = this.analyzeContentType(this.content, fromIndex); - - // Determine if this is truly a new block - let isNewBlock = false; - if (contentType) { - isNewBlock = !this.currentBlock || - this.currentBlock.type !== contentType.type || - (contentType.level !== undefined && this.currentBlock.level !== contentType.level); - } - - // If we detected a specific markdown pattern, use it - if (contentType) { - const segment: StreamingChunk = { - status: "STREAMING", - segment: { - segment: newContent, - styles: [], - type: contentType.type, - isBlockDefining: isNewBlock, - isProcessingNewLine: newContent.includes('\n'), - ...(contentType.level !== undefined && { level: contentType.level }) - } - }; - - segments.push(segment); - - // Update current block tracking only if it's a new block - if (isNewBlock) { - this.currentBlock = { - type: contentType.type, - level: contentType.level, - startIndex: fromIndex, - lastSegmentEnd: toIndex, - styles: new Set() - }; - } else if (this.currentBlock) { - this.currentBlock.lastSegmentEnd = toIndex; - } - - return segments; - } - - // Find the deepest node containing the new content position - const nodeAtPosition = this.findActiveNodeAtPosition(this.currentTree.rootNode, fromIndex); - - if (!nodeAtPosition) { - // If no node found, treat as plain text - return [{ - status: "STREAMING", - segment: { - segment: newContent, - styles: [], - type: "text", - isBlockDefining: false, - isProcessingNewLine: newContent.includes('\n') - } - }]; - } - - // Determine the block type and properties - const blockInfo = this.getBlockInfo(nodeAtPosition); - - // Check if we're starting a new block - isNewBlock = this.isNewBlock(blockInfo, nodeAtPosition); - - // Detect styles in the current context - const styles = this.detectActiveStyles(nodeAtPosition, fromIndex, toIndex); - - // Create the segment - const segment: StreamingChunk = { - status: "STREAMING", - segment: { - segment: newContent, - styles: styles, - type: blockInfo.type, - isBlockDefining: isNewBlock, - isProcessingNewLine: newContent.includes('\n'), - ...(blockInfo.level !== undefined && { level: blockInfo.level }) - } - }; - - segments.push(segment); - - // Update current block tracking - if (isNewBlock) { - this.currentBlock = { - type: blockInfo.type, - level: blockInfo.level, - startIndex: nodeAtPosition.startIndex, - lastSegmentEnd: toIndex, - styles: new Set(styles) - }; - } else if (this.currentBlock) { - this.currentBlock.lastSegmentEnd = toIndex; - styles.forEach(s => this.currentBlock!.styles.add(s)); - } - - return segments; - } - - private analyzeContentType( - content: string, - position: number - ): { type: string; level?: number } | null { - // Get the current line being built - const beforeContent = content.substring(0, position); - const afterContent = content.substring(position); - - // Find the start of the current line - const lastNewline = beforeContent.lastIndexOf('\n'); - const lineStart = lastNewline === -1 ? 0 : lastNewline + 1; - const currentLineContent = content.substring(lineStart, position + afterContent.length); - - // Check for heading markers at line start - if (lineStart === position || lastNewline === position - 1 || position === 0) { - // We're at the beginning of a line or document - const headingMatch = currentLineContent.match(/^(#{1,6})(\s|$)/); - if (headingMatch) { - const level = headingMatch[1].length; - return { type: 'header', level }; - } - } else if (currentLineContent.match(/^(#{1,6})\s/)) { - // We're in the middle of a heading line - const headingMatch = currentLineContent.match(/^(#{1,6})\s/); - if (headingMatch) { - const level = headingMatch[1].length; - return { type: 'header', level }; - } - } - - // Check for code block markers - if (currentLineContent.match(/^```/)) { - return { type: 'code_block' }; - } - - // Check for list markers - if (currentLineContent.match(/^(\*|-|\+|\d+\.)\s/)) { - return { type: 'list_item' }; - } - - // Check for blockquote markers - if (currentLineContent.match(/^>/)) { - return { type: 'blockquote' }; - } - - return null; - } - - private findActiveNodeAtPosition(node: Parser.SyntaxNode, position: number): Parser.SyntaxNode | null { - if (position < node.startIndex || position > node.endIndex) { - return null; - } - - for (const child of node.children) { - const childResult = this.findActiveNodeAtPosition(child, position); - if (childResult) { - return childResult; - } - } - - return node; - } - - private getBlockInfo(node: Parser.SyntaxNode): { type: string; level?: number } { - let current: Parser.SyntaxNode | null = node; - - while (current) { - switch (current.type) { - case 'atx_heading': - return { - type: 'header', - level: this.getHeadingLevel(current) - }; - case 'paragraph': - return { type: 'paragraph' }; - case 'fenced_code_block': - return { type: 'code_block' }; - case 'list_item': - return { type: 'list_item' }; - case 'blockquote': - return { type: 'blockquote' }; - } - - current = current.parent; - } - - return { type: 'paragraph' }; - } - - private isNewBlock(blockInfo: { type: string; level?: number }, node: Parser.SyntaxNode): boolean { - const blockNode = this.findBlockNode(node); - if (!blockNode) return false; - - if (!this.currentBlock) return true; - - if (this.currentBlock.type !== blockInfo.type) return true; - if (blockInfo.level !== undefined && this.currentBlock.level !== blockInfo.level) return true; - - if (blockNode.startIndex > this.currentBlock.lastSegmentEnd) return true; - - return false; - } - - private findBlockNode(node: Parser.SyntaxNode): Parser.SyntaxNode | null { - let current: Parser.SyntaxNode | null = node; - const blockTypes = ['atx_heading', 'paragraph', 'fenced_code_block', 'list_item', 'blockquote']; - - while (current) { - if (blockTypes.includes(current.type)) { - return current; - } - current = current.parent; - } - - return null; - } - - private detectActiveStyles(node: Parser.SyntaxNode, startIdx: number, endIdx: number): string[] { - const styles: Set = new Set(); - let current: Parser.SyntaxNode | null = node; - - while (current) { - if (current.type === 'strong_emphasis' || current.type === 'strong') { - styles.add('bold'); - } else if (current.type === 'emphasis' || current.type === 'em') { - styles.add('italic'); - } else if (current.type === 'code_span') { - styles.add('inline_code'); - } else if (current.type === 'strikethrough') { - styles.add('strikethrough'); - } - - current = current.parent; - } - - return Array.from(styles); - } - - private getHeadingLevel(node: Parser.SyntaxNode): number { - for (const child of node.children) { - if (child.type.startsWith('atx_h') && child.type.endsWith('_marker')) { - const match = child.type.match(/atx_h(\d)_marker/); - if (match) { - return parseInt(match[1], 10); - } - } - } - - const text = node.text || ''; - const match = text.match(/^(#{1,6})\s/); - if (match) { - return match[1].length; - } - - return 1; - } - - getCurrentContent(): string { - return this.content; - } - - getAllSegments(): StreamingChunk[] { - return this.allSegments; - } - - getSegmentsSummary(): { total: number; byType: Record } { - const byType: Record = {}; - - this.allSegments.forEach(seg => { - if (seg.segment) { - const type = seg.segment.type; - byType[type] = (byType[type] || 0) + 1; - } - }); - - return { - total: this.allSegments.length, - byType - }; - } - - reset(): void { - this.content = ''; - this.currentTree = null; - this.lastProcessedIndex = 0; - this.currentBlock = null; - this.allSegments = []; - } -} From b7dfa102cc2eea9712265dc94bb78f92eefccf1d Mon Sep 17 00:00:00 2001 From: Shelby Carter Date: Fri, 6 Feb 2026 20:24:53 -0500 Subject: [PATCH 18/32] LIX-MDSP-5 # API changes according to the PR discussion --- README.md | 282 +++++++-- demo/svelte-demo/src/routes/+page.svelte | 584 +++++++----------- src/state-machine/utils.ts | 12 +- src/tokens-stream-buffer.ts | 108 +++- ...tree-sitter-markdown-stream-parser.test.ts | 342 +++++----- src/tree-sitter-markdown-stream-parser.ts | 187 +++--- src/tree-sitter/content-extraction.ts | 100 +++ src/tree-sitter/index.ts | 72 --- src/tree-sitter/inline-detection.ts | 220 +++++-- src/tree-sitter/inline-extractors.ts | 89 ++- src/tree-sitter/segment-builder.ts | 284 ++++++--- src/tree-sitter/segment-generator.ts | 577 +++++++++++------ src/tree-sitter/types.ts | 188 +++++- 13 files changed, 1891 insertions(+), 1154 deletions(-) delete mode 100644 src/tree-sitter/index.ts diff --git a/README.md b/README.md index cde3402..740eb9a 100644 --- a/README.md +++ b/README.md @@ -132,42 +132,169 @@ for await (const chunk of ["Hello", " ~~world~~", "!", " \n"]) { parser.stopParsing() ``` -The output is a series of objects containing the content of a parsed segment, the type of segment, and any possible inline styles. +The output is a series of `StreamingChunk` objects. Each chunk contains the text content, UTF-16 offset from the start of the stream, block context, and **orthogonal span information**. ```javascript +{ status: 'START_STREAM' } { status: 'STREAMING', - segment: { - segment: 'Hello ', - styles: [], - type: 'paragraph', - isBlockDefining: true, // Indicates beginning of a new block, e.g. paragraph, heading, list etc... - isProcessingNewLine: true + chunk: { + text: 'Hello ', + offset: 0, + length: 6, + block: { type: 'paragraph' }, + opening: [], // Spans that open but don't close in this chunk + closing: [], // Spans that close in this chunk (opened earlier) + contained: [] // Spans fully contained within this chunk } } { status: 'STREAMING', - segment: { - segment: 'world', - styles: [ 'strikethrough' ], - type: 'paragraph', - isBlockDefining: false, - isProcessingNewLine: false + chunk: { + text: 'world', + offset: 6, + length: 5, + block: { type: 'paragraph' }, + opening: [], + closing: [], + contained: [ + { type: 'strikethrough', offset: 6, length: 5 } // ~~world~~ + ] } } { status: 'STREAMING', - segment: { - segment: '! ', - styles: [], - type: 'paragraph', - isBlockDefining: false, - isProcessingNewLine: false + chunk: { + text: '! ', + offset: 11, + length: 3, + block: { type: 'paragraph' }, + opening: [], + closing: [], + contained: [] } } { status: 'END_STREAM' } ``` +### Key Concepts in the New API + +- **`offset`**: UTF-16 code unit offset from the start of the stream +- **`length`**: UTF-16 code unit length of the text +- **`block`**: Block-level context (`paragraph`, `heading`, `code_block`, `list_item`, `table`, etc.) +- **`opening`**: Spans that start in this chunk but don't close (span continues to next chunks) +- **`closing`**: Spans that close in this chunk (were opened in earlier chunks) +- **`contained`**: Spans fully contained within this chunk + + +## Consumer Span State Management + +The new API uses an **orthogonal model** where chunks and spans are completely independent. Spans can cross chunk boundaries. Consumers must track open spans to properly render styled content. + +### How to Track Span State + +```typescript +import { MarkdownStreamParser, OpenSpan, ClosedSpan, Chunk } from '@lixpi/markdown-stream-parser' + +const parser = MarkdownStreamParser.getInstance('session-1') + +// Track currently open spans +let openSpans: OpenSpan[] = [] + +parser.subscribeToTokenParse((streamingChunk, unsubscribe) => { + if (streamingChunk.status === 'START_STREAM') { + openSpans = [] // Reset on new stream + return + } + + if (streamingChunk.status === 'END_STREAM') { + unsubscribe() + return + } + + const chunk = streamingChunk.chunk + + // Handle backtracking (parser corrected previous output) + if (chunk.backtrackOffset !== undefined) { + // Remove content from offset `chunk.backtrackOffset` onwards + // Your render buffer should be truncated to this offset + // Also filter out any open spans that started after backtrackOffset + openSpans = openSpans.filter(s => s.openOffset < chunk.backtrackOffset!) + } + + // 1. Add new opening spans to our tracking list + openSpans.push(...chunk.opening) + + // 2. Process closing spans (remove from tracking, render complete span) + for (const closingSpan of chunk.closing) { + // Find and remove the matching open span + const openIndex = openSpans.findIndex(s => s.type === closingSpan.type) + if (openIndex !== -1) { + openSpans.splice(openIndex, 1) + } + // Now you have a complete span with offset and length + // Use closingSpan.offset and closingSpan.length to apply styling + } + + // 3. Contained spans are already complete (no tracking needed) + // Just apply their styling: contained.offset, contained.length + + // 4. Render the chunk text with active styles + const activeStyles = [ + ...openSpans.map(s => s.type), + ...chunk.contained.map(s => s.type) + ] + renderText(chunk.text, chunk.block, activeStyles) +}) +``` + +### Span Types + +```typescript +type SpanType = 'bold' | 'italic' | 'code' | 'strikethrough' | 'link' | 'image' + +// Opening span: we know where it starts, but it's not closed yet +type OpenSpan = { type: SpanType; openOffset: number } + +// Closed/contained span: complete with offset and length +type ClosedSpan = Span & { offset: number; length: number } + +// Link and image spans include additional metadata +type LinkSpan = { type: 'link'; url: string; offset: number; length: number } +type ImageSpan = { type: 'image'; src: string; alt?: string; offset: number; length: number } +``` + +### Handling Backtracking + +The parser may sometimes need to **correct** previously emitted chunks. This happens when tree-sitter reinterprets the content as more tokens arrive. + +When `chunk.backtrackOffset` is present: +1. **Discard content** from that offset onwards in your render buffer +2. **Filter open spans** to remove any that started after the backtrack offset +3. **Apply the new chunk** which contains the corrected content + +```typescript +if (chunk.backtrackOffset !== undefined) { + // Truncate your output buffer to backtrackOffset + outputBuffer = outputBuffer.slice(0, chunk.backtrackOffset) + + // Remove spans that are no longer valid + openSpans = openSpans.filter(s => s.openOffset < chunk.backtrackOffset!) +} +``` + +### Configuration Options + +```typescript +const parser = MarkdownStreamParser.getInstance('session-1', { + windowSize: 500, // Lookback window for backtrack detection (chars) + includeRawStreamedToken: true // Include original token in chunk.original +}) + +// Or configure after creation +parser.setConfig({ windowSize: 1000 }) +``` + ## Is that it? What am I supposed to do with that? @@ -187,13 +314,13 @@ It will **always remain `render-agnostic`** - whatever you use to render your st - [x] Inline Strikethrough (`~~text~~`) - [x] Inline Code (`` `code` ``) - [x] Code Blocks (```` ```code-block``` ````) with language detection -- [ ] Blockquotes (`> quote`) [Iusse #2](https://github.com/Lixpi/markdown-stream-parser/issues/2) -- [ ] //TODO: PRIORITY: Ordered Lists (`1. item`) [Iusse #3](https://github.com/Lixpi/markdown-stream-parser/issues/3) -- [ ] //TODO: PRIORITY: Unordered Lists (`- item`, `* item`, `+ item`) *BLOCKED BY:* [Iusse #3](https://github.com/Lixpi/markdown-stream-parser/issues/3) -- [ ] //TODO: Task Lists (`- [ ] item`) *BLOCKED BY:* [Iusse #3](https://github.com/Lixpi/markdown-stream-parser/issues/3) -- [ ] //TODO: PRIORITY: Tables [Iusse #7](https://github.com/Lixpi/markdown-stream-parser/issues/7) -- [ ] //TODO: PRIORITY: Links (`[text](url)`) -- [ ] //TODO: PRIORITY: Images (`![alt](url)`) +- [x] Links (`[text](url)`) - with URL extraction +- [x] Images (`![alt](url)`) - with src and alt extraction +- [ ] Blockquotes (`> quote`) [Issue #2](https://github.com/Lixpi/markdown-stream-parser/issues/2) +- [ ] //TODO: PRIORITY: Ordered Lists (`1. item`) [Issue #3](https://github.com/Lixpi/markdown-stream-parser/issues/3) +- [ ] //TODO: PRIORITY: Unordered Lists (`- item`, `* item`, `+ item`) *BLOCKED BY:* [Issue #3](https://github.com/Lixpi/markdown-stream-parser/issues/3) +- [ ] //TODO: Task Lists (`- [ ] item`) *BLOCKED BY:* [Issue #3](https://github.com/Lixpi/markdown-stream-parser/issues/3) +- [ ] //TODO: PRIORITY: Tables [Issue #7](https://github.com/Lixpi/markdown-stream-parser/issues/7) - [ ] //TODO: Horizontal Rules (`---`, `***`, `___`) - [ ] //TODO: Footnotes - [ ] //TODO: HTML blocks @@ -304,9 +431,9 @@ graph TB BD[block-detection.ts] ID[inline-detection.ts] CE[content-extraction.ts] - IE[inline-extractors.ts] TN[tree-navigation.ts] SB[segment-builder.ts] + TY[types.ts] end subgraph "External" @@ -319,10 +446,11 @@ graph TB SG --> BD SG --> ID SG --> CE - SG --> IE + SG --> SB + SG --> TY BD --> TN ID --> TN - IE --> SB + SB --> TY CE --> TS BD --> TS ID --> TS @@ -332,18 +460,18 @@ graph TB | Module | Responsibility | |--------|----------------| -| `segment-generator.ts` | Main orchestrator - generates segments from content ranges | -| `block-detection.ts` | Figures out block type (header, paragraph, code block, list, table) | -| `inline-detection.ts` | Detects active inline styles (bold, italic, code, strikethrough) | +| `segment-generator.ts` | Main orchestrator - generates chunks from content ranges with span detection | +| `block-detection.ts` | Determines block type (heading, paragraph, code_block, list_item, table) | +| `inline-detection.ts` | Detects inline spans (bold, italic, code, strikethrough, link, image) | | `content-extraction.ts` | Strips markdown syntax and extracts clean content | -| `inline-extractors.ts` | Extracts styled segments with proper marker stripping | | `tree-navigation.ts` | AST traversal utilities | -| `segment-builder.ts` | Creates segment objects with consistent structure | +| `segment-builder.ts` | Creates Chunk and Span objects with UTF-16 offsets | +| `types.ts` | Type definitions (Chunk, Span, BlockContext, etc.) | ### Parser API Flow ```mermaid -%%{init: {'theme': 'base', 'themeVariables': { 'noteBkgColor': '#82B2C0', 'noteTextColor': '#1a3a47', 'noteBorderColor': '#5a9aad', 'actorBkg': '#F6C7B3', 'actorBorder': '#d4956a', 'actorTextColor': '#5a3a2a', 'actorLineColor': '#d4956a', 'signalColor': '#d4956a', 'signalTextColor': '#5a3a2a', 'labelBoxBkgColor': '#F6C7B3', 'labelBoxBorderColor': '#d4956a', 'labelTextColor': '#5a3a2a', 'loopTextColor': '#5a3a2a', 'activationBorderColor': '#d4956a', 'activationBkgColor': '#C3DEDD', 'sequenceNumberColor': '#5a3a2a'}}}%% +%%{init: {'theme': 'base', 'themeVariables': { 'noteBkgColor': '#82B2C0', 'noteTextColor': '#1a3a47', 'noteBorderColor': '#5a9aad', 'actorBkg': '#F6C7B3', 'actorBorder': '#d4956a', 'actorTextColor': '#5a3a2a', 'actorLineColor': '#d4956a', 'signalColor': '#d4956a', 'signalTextColor': '#5a3a2a', 'labelBoxBkgColor': '#F6C7B3', 'labelBoxBorderColor': '#d4956a', 'labelTextColor': '#5a3a2a', 'loopTextColor': '#5a3a2a', 'activationBorderColor': '#9DC49D', 'activationBkgColor': '#9DC49D', 'sequenceNumberColor': '#5a3a2a'}}}%% sequenceDiagram participant App as Your App participant Parser as MarkdownStreamParser @@ -351,39 +479,69 @@ sequenceDiagram participant TS as Tree-sitter participant Gen as SegmentGenerator + %% ═══════════════════════════════════════════════════════════════ + %% SETUP PHASE + %% ═══════════════════════════════════════════════════════════════ rect rgb(220, 236, 233) - Note over App, Gen: Setup Phase - App->>Parser: getInstance(sessionId) + Note over App, Gen: PHASE 1 - Setup + App->>Parser: getInstance(sessionId, config?) activate Parser Parser->>TS: load WASM grammars + activate TS + TS-->>Parser: grammars loaded + deactivate TS Parser-->>App: parser instance + deactivate Parser end + %% ═══════════════════════════════════════════════════════════════ + %% SUBSCRIPTION PHASE + %% ═══════════════════════════════════════════════════════════════ rect rgb(195, 222, 221) - Note over App, Gen: Subscription Phase + Note over App, Gen: PHASE 2 - Subscription App->>Parser: subscribeToTokenParse(listener) + activate Parser App->>Parser: startParsing() Parser-->>App: START_STREAM event + deactivate Parser end + %% ═══════════════════════════════════════════════════════════════ + %% STREAMING PHASE + %% ═══════════════════════════════════════════════════════════════ rect rgb(246, 199, 179) - Note over App, Gen: Streaming Phase + Note over App, Gen: PHASE 3 - Streaming loop For each LLM token App->>Parser: parseToken(chunk) + activate Parser Parser->>Buffer: receiveChunk(chunk) - Buffer->>Parser: segment ready + activate Buffer + Buffer-->>Parser: content ready + deactivate Buffer Parser->>TS: parse(content) + activate TS TS-->>Parser: AST - Parser->>Gen: generateSegments(range) - Gen-->>Parser: StreamingChunk[] - Parser-->>App: notify(segment) + deactivate TS + Parser->>Gen: generateSegments(range, state) + activate Gen + Gen-->>Parser: Chunk[] with spans + deactivate Gen + Parser-->>App: notify(StreamingChunk) + deactivate Parser end end + %% ═══════════════════════════════════════════════════════════════ + %% CLEANUP PHASE + %% ═══════════════════════════════════════════════════════════════ rect rgb(242, 234, 224) - Note over App, Gen: Cleanup Phase + Note over App, Gen: PHASE 4 - Cleanup App->>Parser: stopParsing() + activate Parser Parser->>Buffer: flushBuffer() + activate Buffer + Buffer-->>Parser: buffer flushed + deactivate Buffer Parser-->>App: END_STREAM event deactivate Parser App->>Parser: removeInstance(sessionId) @@ -395,34 +553,38 @@ sequenceDiagram ```mermaid %%{init: {'theme': 'base', 'themeVariables': { 'primaryColor': '#F6C7B3', 'primaryTextColor': '#5a3a2a', 'primaryBorderColor': '#d4956a', 'secondaryColor': '#C3DEDD', 'lineColor': '#d4956a', 'textColor': '#5a3a2a'}}}%% stateDiagram-v2 - [*] --> Idle: getInstance() + [*] --> Idle: getInstance(config?) Idle --> Parsing: startParsing() - + state Parsing { [*] --> AwaitingToken - + AwaitingToken --> ProcessingChunk: parseToken(chunk) ProcessingChunk --> DetectingBlock: tree-sitter parse - DetectingBlock --> ProcessingHeader: atx_heading found + ProcessingChunk --> BacktrackDetected: getChangedRanges() detects change + BacktrackDetected --> DetectingBlock: emit with backtrackOffset + DetectingBlock --> ProcessingHeading: atx_heading found DetectingBlock --> ProcessingParagraph: paragraph found DetectingBlock --> ProcessingCodeBlock: fenced_code_block found DetectingBlock --> ProcessingList: list_item found DetectingBlock --> ProcessingTable: pipe_table found - - ProcessingHeader --> DetectingInline: check inline styles - ProcessingParagraph --> DetectingInline: check inline styles - ProcessingList --> DetectingInline: check inline styles - ProcessingTable --> DetectingInline: check inline styles - - DetectingInline --> BufferingIncomplete: unmatched delimiter - DetectingInline --> EmitSegment: style complete + + ProcessingHeading --> DetectingSpans: check inline spans + ProcessingParagraph --> DetectingSpans: check inline spans + ProcessingList --> DetectingSpans: check inline spans + ProcessingTable --> DetectingSpans: check inline spans + + DetectingSpans --> BufferingIncomplete: unmatched delimiter + DetectingSpans --> ProcessSpans: spans detected + ProcessSpans --> CategorizeSpans: opening/closing/contained BufferingIncomplete --> AwaitingToken: wait for more - - ProcessingCodeBlock --> EmitSegment: extract content - EmitSegment --> AwaitingToken: notify subscribers + + ProcessingCodeBlock --> EmitChunk: extract content + CategorizeSpans --> EmitChunk: build Chunk with spans + EmitChunk --> AwaitingToken: notify subscribers } - + Parsing --> Flushing: stopParsing() Flushing --> Idle: END_STREAM Idle --> [*]: removeInstance() diff --git a/demo/svelte-demo/src/routes/+page.svelte b/demo/svelte-demo/src/routes/+page.svelte index a9ca9c3..a29029f 100644 --- a/demo/svelte-demo/src/routes/+page.svelte +++ b/demo/svelte-demo/src/routes/+page.svelte @@ -3,6 +3,10 @@ import { MarkdownStreamParser, type StreamingChunk, + type Chunk, + type OpenSpan, + type ClosedSpan, + type SpanType, } from "../../../../src/tree-sitter-markdown-stream-parser.js"; type ExampleFile = { base: string; json: string; txt: string }; @@ -18,7 +22,7 @@ let txtContent = ""; let jsonContent = ""; let parsedSegments: StreamingChunk[] = []; - let parsedBlocks: StreamingChunk[][] = []; + let parsedBlocks: Chunk[][] = []; let currentToken = ""; let currentParsedChunks: StreamingChunk[] = []; let error = ""; @@ -28,6 +32,9 @@ let parser: MarkdownStreamParser | null = null; let parserId: string = ""; + // Track open spans across chunks for styling + let openSpans: OpenSpan[] = []; + async function loadExamples() { try { const res = await fetch("/llm-examples-manifest.json"); @@ -58,12 +65,54 @@ resetParser(); } + // Get active span types from open spans and chunk spans + function getActiveSpanTypes(chunk: Chunk): SpanType[] { + const types: SpanType[] = []; + + // Add types from contained spans (fully within this chunk) + for (const span of chunk.contained) { + if (!types.includes(span.type)) { + types.push(span.type); + } + } + + // Add types from opening spans (start in this chunk) + for (const span of chunk.opening) { + if (!types.includes(span.type)) { + types.push(span.type); + } + } + + // Add types from currently open spans (opened in previous chunks) + for (const span of openSpans) { + if (!types.includes(span.type)) { + types.push(span.type); + } + } + + return types; + } + + // Update open spans tracking based on chunk + function updateOpenSpans(chunk: Chunk) { + // Remove closed spans + for (const closedSpan of chunk.closing) { + openSpans = openSpans.filter(s => s.type !== closedSpan.type); + } + + // Add new opening spans + for (const openSpan of chunk.opening) { + openSpans = [...openSpans, openSpan]; + } + } + async function initializeParser() { parsedSegments = []; currentToken = ""; currentParsedChunks = []; currentTokenIndex = null; error = ""; + openSpans = []; parserId = "demo-" + Date.now(); @@ -82,10 +131,12 @@ currentTokenIndex = null; currentToken = ""; parser = null; + openSpans = []; } else if (parsed.status === "START_STREAM") { parsedSegments = [...parsedSegments, parsed]; } else if (parsed.status === "STREAMING") { parsedSegments = [...parsedSegments, parsed]; + updateOpenSpans(parsed.chunk); if (streaming || paused) { currentParsedChunks = [...currentParsedChunks, parsed]; @@ -211,54 +262,58 @@ streaming = false; paused = false; error = ""; + openSpans = []; } + // Group chunks into blocks based on block type changes $: parsedBlocks = (() => { - const blocks: StreamingChunk[][] = []; - let currentBlock: StreamingChunk[] = []; + const blocks: Chunk[][] = []; + let currentBlock: Chunk[] = []; + let lastBlockType: string | undefined = undefined; + let lastBlockLevel: number | undefined = undefined; + let lastOffset: number = -1; for (const seg of parsedSegments) { if (seg.status === "START_STREAM" || seg.status === "END_STREAM") { continue; } - // Don't split blocks for table cells - keep them together in rows - const isTableCell = - seg.segment?.type === "table_cell" || - seg.segment?.type === "table_header_cell"; - const blockId = seg.segment?.blockId; - - const lastSeg = - currentBlock.length > 0 ? currentBlock[currentBlock.length - 1] : null; - const prevIsTableCell = - lastSeg && - (lastSeg.segment?.type === "table_cell" || - lastSeg.segment?.type === "table_header_cell"); - const prevBlockId = lastSeg?.segment?.blockId; - - // Start a new block if isBlockDefining, but merge cells with same blockId - if (seg.segment?.isBlockDefining && currentBlock.length) { - let shouldSplit = true; - - // If both are table cells and have the same blockId, keep together - if ( - isTableCell && - prevIsTableCell && - blockId !== undefined && - blockId === prevBlockId - ) { - shouldSplit = false; + const chunk = seg.chunk; + const blockType = chunk.block.type; + const blockLevel = chunk.block.level; + + // Detect new block: type change, or heading level change + // For list items, use gap in offset to detect new item + let isNewBlock = false; + + if (blockType !== lastBlockType) { + isNewBlock = true; + } else if (blockType === 'heading' && blockLevel !== lastBlockLevel) { + isNewBlock = true; + } else if (blockType === 'list_item' && lastOffset >= 0) { + // New list item if there's a significant gap in offset (indicates newline/new item) + // Or if the text starts after a newline marker + const gap = chunk.offset - lastOffset; + if (gap > 50) { // Heuristic: large gap suggests new list item + isNewBlock = true; } + } - if (shouldSplit) { - blocks.push(currentBlock); - currentBlock = []; - } + if (isNewBlock && currentBlock.length > 0) { + blocks.push(currentBlock); + currentBlock = []; } - currentBlock.push(seg); + + currentBlock.push(chunk); + lastBlockType = blockType; + lastBlockLevel = blockLevel; + lastOffset = chunk.offset + chunk.length; + } + + if (currentBlock.length > 0) { + blocks.push(currentBlock); } - if (currentBlock.length) blocks.push(currentBlock); return blocks; })(); @@ -298,6 +353,30 @@ jsonItems = []; } } + + // Helper function to determine CSS classes for text based on active spans + function getSpanClasses(styles: SpanType[]): string { + const classes: string[] = []; + + if (styles.includes('bold') && styles.includes('italic')) { + classes.push('font-bold', 'italic'); + } else if (styles.includes('bold')) { + classes.push('font-bold'); + } else if (styles.includes('italic')) { + classes.push('italic'); + } + + if (styles.includes('strikethrough')) { + classes.push('line-through'); + } + + return classes.join(' '); + } + + // Check if style includes code + function hasCodeStyle(styles: SpanType[]): boolean { + return styles.includes('code'); + }
@@ -394,352 +473,143 @@

Parsed Stream

{#each parsedBlocks as block} - {@const hasTableCells = block.some( - (seg) => - seg.segment?.type === "table_cell" || - seg.segment?.type === "table_header_cell", - )} + {@const blockType = block[0]?.block.type} + {@const blockLevel = block[0]?.block.level} + {@const blockLanguage = block[0]?.block.language} + {@const hasTableCells = blockType === 'table_cell' || blockType === 'table_row'} +
- {#each block as seg} - {#if seg.segment?.type === "header"} - {#if seg.segment?.level === 1} -

- {#if seg.segment?.styles?.length} - - {#if seg.segment.styles.includes("strikethrough")} - {seg.segment?.segment} - {:else if seg.segment.styles.includes("code")} - {seg.segment?.segment} - {:else} - {seg.segment?.segment} - {/if} - + {#if blockType === 'heading'} + {#if blockLevel === 1} +

+ {#each block as chunk} + {@const styles = [...chunk.contained.map(s => s.type), ...chunk.opening.map(s => s.type)]} + {#if hasCodeStyle(styles)} + {chunk.text} {:else} - {seg.segment?.segment} + {chunk.text} {/if} -

- {:else if seg.segment?.level === 2} -

- {#if seg.segment?.styles?.length} - - {#if seg.segment.styles.includes("strikethrough")} - {seg.segment?.segment} - {:else if seg.segment.styles.includes("code")} - {seg.segment?.segment} - {:else} - {seg.segment?.segment} - {/if} - + {/each} +

+ {:else if blockLevel === 2} +

+ {#each block as chunk} + {@const styles = [...chunk.contained.map(s => s.type), ...chunk.opening.map(s => s.type)]} + {#if hasCodeStyle(styles)} + {chunk.text} {:else} - {seg.segment?.segment} + {chunk.text} {/if} -

- {:else if seg.segment?.level === 3} -

- {#if seg.segment?.styles?.length} - - {#if seg.segment.styles.includes("strikethrough")} - {seg.segment?.segment} - {:else if seg.segment.styles.includes("code")} - {seg.segment?.segment} - {:else} - {seg.segment?.segment} - {/if} - + {/each} +

+ {:else if blockLevel === 3} +

+ {#each block as chunk} + {@const styles = [...chunk.contained.map(s => s.type), ...chunk.opening.map(s => s.type)]} + {#if hasCodeStyle(styles)} + {chunk.text} {:else} - {seg.segment?.segment} + {chunk.text} {/if} -

- {:else if seg.segment?.level === 4} -

- {#if seg.segment?.styles?.length} - - {#if seg.segment.styles.includes("strikethrough")} - {seg.segment?.segment} - {:else if seg.segment.styles.includes("code")} - {seg.segment?.segment} - {:else} - {seg.segment?.segment} - {/if} - + {/each} +

+ {:else if blockLevel === 4} +

+ {#each block as chunk} + {@const styles = [...chunk.contained.map(s => s.type), ...chunk.opening.map(s => s.type)]} + {#if hasCodeStyle(styles)} + {chunk.text} {:else} - {seg.segment?.segment} + {chunk.text} {/if} -

- {:else if seg.segment?.level === 5} -
- {#if seg.segment?.styles?.length} - - {#if seg.segment.styles.includes("strikethrough")} - {seg.segment?.segment} - {:else if seg.segment.styles.includes("code")} - {seg.segment?.segment} - {:else} - {seg.segment?.segment} - {/if} - + {/each} +
+ {:else if blockLevel === 5} +
+ {#each block as chunk} + {@const styles = [...chunk.contained.map(s => s.type), ...chunk.opening.map(s => s.type)]} + {#if hasCodeStyle(styles)} + {chunk.text} {:else} - {seg.segment?.segment} + {chunk.text} {/if} -
- {:else if seg.segment?.level === 6} -
- {#if seg.segment?.styles?.length} - - {#if seg.segment.styles.includes("strikethrough")} - {seg.segment?.segment} - {:else if seg.segment.styles.includes("code")} - {seg.segment?.segment} - {:else} - {seg.segment?.segment} - {/if} - + {/each} +
+ {:else if blockLevel === 6} +
+ {#each block as chunk} + {@const styles = [...chunk.contained.map(s => s.type), ...chunk.opening.map(s => s.type)]} + {#if hasCodeStyle(styles)} + {chunk.text} {:else} - {seg.segment?.segment} + {chunk.text} {/if} -
- {:else} - - {#if seg.segment?.styles?.length} - - {#if seg.segment.styles.includes("strikethrough")} - {seg.segment?.segment} - {:else if seg.segment.styles.includes("code")} - {seg.segment?.segment} - {:else} - {seg.segment?.segment} - {/if} - + {/each} +
+ {:else} + + {#each block as chunk} + {@const styles = [...chunk.contained.map(s => s.type), ...chunk.opening.map(s => s.type)]} + {#if hasCodeStyle(styles)} + {chunk.text} {:else} - {seg.segment?.segment} + {chunk.text} {/if} - - {/if} - {:else if seg.segment?.type === "codeBlock"} - 
{seg.segment?.segment}
- {:else if seg.segment?.type === "blockQuote"} - {seg.segment?.segment} - {:else if seg.segment?.type === "list_item"} - - {#if seg.segment?.isBlockDefining} - • - {/if} - {#if seg.segment?.styles?.length} - - {#if seg.segment.styles.includes("strikethrough")} - {seg.segment?.segment} - {:else if seg.segment.styles.includes("code")} - {seg.segment?.segment} - {:else} - {seg.segment?.segment} - {/if} - + {/each} + + {/if} + {:else if blockType === 'code_block'} + 
{#each block as chunk}{chunk.text}{/each}
+ {#if blockLanguage} + {blockLanguage} + {/if} + {:else if blockType === 'blockquote'} + + {#each block as chunk} + {@const styles = [...chunk.contained.map(s => s.type), ...chunk.opening.map(s => s.type)]} + {#if hasCodeStyle(styles)} + {chunk.text} {:else} - {seg.segment?.segment} + {chunk.text} {/if} - - {:else if seg.segment?.type === "table_header_cell"} - - {#if seg.segment?.styles?.length} - - {#if seg.segment.styles.includes("strikethrough")} - {seg.segment?.segment} - {:else if seg.segment.styles.includes("code")} - {seg.segment?.segment} - {:else} - {seg.segment?.segment} - {/if} - + {/each} + + {:else if blockType === 'list_item'} + + • + {#each block as chunk} + {@const styles = [...chunk.contained.map(s => s.type), ...chunk.opening.map(s => s.type)]} + {#if hasCodeStyle(styles)} + {chunk.text} {:else} - {seg.segment?.segment} + {chunk.text} {/if} - - {:else if seg.segment?.type === "table_cell"} - - {#if seg.segment?.styles?.length} - - {#if seg.segment.styles.includes("strikethrough")} - {seg.segment?.segment} - {:else if seg.segment.styles.includes("code")} - {seg.segment?.segment} - {:else} - {seg.segment?.segment} - {/if} - + {/each} + + {:else if blockType === 'table_cell' || blockType === 'table_row'} + {#each block as chunk} + {@const styles = [...chunk.contained.map(s => s.type), ...chunk.opening.map(s => s.type)]} + + {#if hasCodeStyle(styles)} + {chunk.text} {:else} - {seg.segment?.segment} + {chunk.text} {/if} - {:else} - - {#if seg.segment?.styles?.length} - - {#if seg.segment.styles.includes("strikethrough")} - {seg.segment?.segment} - {:else if seg.segment.styles.includes("code")} - {seg.segment?.segment} - {:else} - {seg.segment?.segment} - {/if} - + {/each} + {:else} + + + {#each block as chunk} + {@const styles = [...chunk.contained.map(s => s.type), ...chunk.opening.map(s => s.type)]} + {#if hasCodeStyle(styles)} + {chunk.text} {:else} - {seg.segment?.segment} + {chunk.text} {/if} - - {/if} - {/each} + {/each} + + {/if}

{/each}
diff --git a/src/state-machine/utils.ts b/src/state-machine/utils.ts index 3a18343..a359f12 100644 --- a/src/state-machine/utils.ts +++ b/src/state-machine/utils.ts @@ -1,3 +1,13 @@ 'use strict' -export const truncateTrailingNewLine = (input: string): string => input.replace(/(\\n|\n)+$/, '') +export const truncateTrailingNewLine = (input: string): string => { + let end = input.length + while (end > 0 && (input[end - 1] === '\n' || (end > 1 && input[end - 2] === '\\' && input[end - 1] === 'n'))) { + if (input[end - 1] === '\n') { + end-- + } else { + end -= 2 + } + } + return input.substring(0, end) +} diff --git a/src/tokens-stream-buffer.ts b/src/tokens-stream-buffer.ts index d20e89d..7fe44c1 100644 --- a/src/tokens-stream-buffer.ts +++ b/src/tokens-stream-buffer.ts @@ -10,46 +10,90 @@ export default class TokensStreamBuffer { } public processBufferForCompletion() { - const pattern = /(\s*\S+\s+|\s*\S+((\n|\\n)+))/g - let match - let lastIndex = 0 - - // Use exec() since it provides indices, which we use to slice the buffer correctly - while ((match = pattern.exec(this.buffer)) !== null) { - const matchObject = (([ - fullMatch, - prefixedWhitespace, - content, - postfixedWhitespace - ]) => ({ - fullMatch: fullMatch || '', - prefixedWhitespace: prefixedWhitespace || '', - content: content || '', - postfixedWhitespace: postfixedWhitespace || '', - }))(match || []) - - lastIndex = match.index + matchObject.fullMatch.length // Calculate the index of the end of the match - - this.notifyWordCompletion(matchObject.fullMatch) // Emit the word + // Find complete word segments without using regex + // Original pattern behavior: (\s*\S+\s+|\s*\S+((\n|\\n)+)) + // This means: optional leading whitespace + word + trailing whitespace/newlines + let lastEmittedIndex = 0 + let i = 0 + + while (i < this.buffer.length) { + // Skip leading whitespace (will be included with the word) + const segmentStart = i + while (i < this.buffer.length && this.isWhitespaceChar(this.buffer[i])) { + i++ + } + + // If we only have whitespace left, stop (don't emit orphan whitespace) + if (i >= this.buffer.length) { + break + } + + // Consume non-whitespace characters (the word) + while (i < this.buffer.length && !this.isWhitespaceChar(this.buffer[i])) { + // Check for escaped newline sequence "\\n" within non-whitespace + if (this.buffer[i] === '\\' && i + 1 < this.buffer.length && this.buffer[i + 1] === 'n') { + // Include the escaped newline and emit + i += 2 + const segment = this.buffer.slice(lastEmittedIndex, i) + this.notifyWordCompletion(segment) + lastEmittedIndex = i + // Continue to next iteration + break + } + i++ + } + + // Check if we broke out due to escaped newline + if (lastEmittedIndex === i) { + continue + } + + // If we're at end of buffer with no trailing whitespace, stop (incomplete word) + if (i >= this.buffer.length) { + break + } + + // Consume ALL trailing whitespace (one or more required for emission) + const trailingStart = i + while (i < this.buffer.length && this.isWhitespaceChar(this.buffer[i])) { + i++ + } + + // Emit the segment: from lastEmittedIndex to end of ALL trailing whitespace + const segment = this.buffer.slice(lastEmittedIndex, i) + this.notifyWordCompletion(segment) + lastEmittedIndex = i } - // Update the buffer by slicing off the processed part - if (lastIndex > 0) { - this.buffer = this.buffer.slice(lastIndex) + // Update the buffer by removing the processed part + if (lastEmittedIndex > 0) { + this._buffer = this.buffer.slice(lastEmittedIndex) } // Handle long sequences without whitespace to prevent infinite buffer growth - // If buffer is too long and contains only non-whitespace characters, emit chunks to prevent freezing - const MAX_BUFFER_SIZE = 100; // Reasonable limit for streaming UX - if (this.buffer.length > MAX_BUFFER_SIZE && !/\s/.test(this.buffer)) { - // Split the buffer into chunks and emit them - const CHUNK_SIZE = 50; // Emit in reasonable chunks + const MAX_BUFFER_SIZE = 100 + if (this.buffer.length > MAX_BUFFER_SIZE && !this.hasWhitespace(this.buffer)) { + const CHUNK_SIZE = 50 while (this.buffer.length > CHUNK_SIZE) { - const chunk = this.buffer.slice(0, CHUNK_SIZE); - this.notifyWordCompletion(chunk); - this.buffer = this.buffer.slice(CHUNK_SIZE); + const chunk = this.buffer.slice(0, CHUNK_SIZE) + this.notifyWordCompletion(chunk) + this._buffer = this.buffer.slice(CHUNK_SIZE) + } + } + } + + private isWhitespaceChar(char: string): boolean { + return char === ' ' || char === '\t' || char === '\n' || char === '\r' || char === '\v' + } + + private hasWhitespace(text: string): boolean { + for (let i = 0; i < text.length; i++) { + const char = text[i] + if (char === ' ' || char === '\t' || char === '\n' || char === '\r') { + return true } } + return false } private notifyWordCompletion(word: string) { diff --git a/src/tree-sitter-markdown-stream-parser.test.ts b/src/tree-sitter-markdown-stream-parser.test.ts index 34fb0da..fff0b83 100644 --- a/src/tree-sitter-markdown-stream-parser.test.ts +++ b/src/tree-sitter-markdown-stream-parser.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' import { MarkdownStreamParser } from './tree-sitter-markdown-stream-parser' +import type { Chunk, ClosedSpan, SpanType } from './tree-sitter/types.js' import path from 'path' import { fileURLToPath } from 'url' import fs from 'fs' @@ -7,16 +8,28 @@ import fs from 'fs' const __filename = fileURLToPath(import.meta.url) const __dirname = path.dirname(__filename) +// Helper to check if a chunk has a span of the given type +function hasSpanType(chunk: Chunk, type: SpanType): boolean { + const allSpans = [...chunk.contained, ...chunk.closing] + return allSpans.some(span => span.type === type) +} + +// Helper to get all span types from a chunk +function getSpanTypes(chunk: Chunk): SpanType[] { + const allSpans = [...chunk.opening, ...chunk.closing, ...chunk.contained] as ClosedSpan[] + return allSpans.map(span => span.type) +} + describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { let parser: MarkdownStreamParser - let parsedSegments: any[] = [] + let parsedChunks: Chunk[] = [] const instanceId = 'test-tree-sitter' // Set up path for WASM files const wasmDir = path.join(__dirname, '../demo/svelte-demo/static') beforeEach(async () => { - parsedSegments = [] + parsedChunks = [] // Configure WASM path for testing - this will also help locateFile find tree-sitter.wasm MarkdownStreamParser.configureWasmPath(path.join(wasmDir, 'tree-sitter-markdown.wasm')) @@ -24,8 +37,8 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { parser = await MarkdownStreamParser.getInstance(instanceId) parser.subscribeToTokenParse((chunk) => { - if (chunk.status === 'STREAMING' && chunk.segment) { - parsedSegments.push(chunk.segment) + if (chunk.status === 'STREAMING' && chunk.chunk) { + parsedChunks.push(chunk.chunk) } }) @@ -38,18 +51,14 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { }) describe('Basic Block Types', () => { - it('should use camelCase for block type names', async () => { + it('should use snake_case for block type names (new API)', async () => { parser.parseToken('```javascript\n') parser.parseToken('code\n') parser.parseToken('```\n') parser.stopParsing() - const codeBlockSegments = parsedSegments.filter(s => s.type.includes('code') || s.type.includes('Code')) - expect(codeBlockSegments.length).toBeGreaterThan(0) - - // Should be 'codeBlock' not 'code_block' - const hasCorrectNaming = codeBlockSegments.some(s => s.type === 'codeBlock') - expect(hasCorrectNaming).toBe(true) + const codeBlockChunks = parsedChunks.filter(c => c.block.type === 'code_block') + expect(codeBlockChunks.length).toBeGreaterThan(0) }) it('should extract language from code blocks', async () => { @@ -58,11 +67,11 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { parser.parseToken('```\n') parser.stopParsing() - const codeBlockSegments = parsedSegments.filter(s => s.type === 'codeBlock') - expect(codeBlockSegments.length).toBeGreaterThan(0) + const codeBlockChunks = parsedChunks.filter(c => c.block.type === 'code_block') + expect(codeBlockChunks.length).toBeGreaterThan(0) // Should have language field - const hasLanguage = codeBlockSegments.some(s => s.language === 'javascript') + const hasLanguage = codeBlockChunks.some(c => c.block.language === 'javascript') expect(hasLanguage).toBe(true) }) @@ -72,12 +81,12 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { parser.parseToken('```\n') parser.stopParsing() - const codeBlockSegments = parsedSegments.filter(s => s.type === 'codeBlock') - expect(codeBlockSegments.length).toBeGreaterThan(0) + const codeBlockChunks = parsedChunks.filter(c => c.block.type === 'code_block') + expect(codeBlockChunks.length).toBeGreaterThan(0) // Language should be empty string or undefined - const firstCodeBlock = codeBlockSegments[0] - expect(firstCodeBlock.language === '' || firstCodeBlock.language === undefined).toBe(true) + const firstCodeBlock = codeBlockChunks[0] + expect(firstCodeBlock.block.language === '' || firstCodeBlock.block.language === undefined).toBe(true) }) }) @@ -87,11 +96,11 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { parser.parseToken('Header Text\n') parser.stopParsing() - const headerSegments = parsedSegments.filter(s => s.type === 'header') - expect(headerSegments.length).toBeGreaterThan(0) + const headerChunks = parsedChunks.filter(c => c.block.type === 'heading') + expect(headerChunks.length).toBeGreaterThan(0) // Content should NOT include ## - const headerContent = headerSegments.map(s => s.segment).join('') + const headerContent = headerChunks.map(c => c.text).join('') expect(headerContent).not.toContain('##') expect(headerContent.trim()).toBe('Header Text') }) @@ -100,11 +109,11 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { const levels = [1, 2, 3, 4, 5, 6] for (const level of levels) { - parsedSegments = [] + parsedChunks = [] parser = await MarkdownStreamParser.getInstance(`test-${level}`) parser.subscribeToTokenParse((chunk) => { - if (chunk.status === 'STREAMING' && chunk.segment) { - parsedSegments.push(chunk.segment) + if (chunk.status === 'STREAMING' && chunk.chunk) { + parsedChunks.push(chunk.chunk) } }) parser.startParsing() @@ -114,9 +123,9 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { parser.parseToken(`Level ${level}\n`) parser.stopParsing() - const headerSegments = parsedSegments.filter(s => s.type === 'header') - expect(headerSegments.length).toBeGreaterThan(0) - expect(headerSegments[0].level).toBe(level) + const headerChunks = parsedChunks.filter(c => c.block.type === 'heading') + expect(headerChunks.length).toBeGreaterThan(0) + expect(headerChunks[0].block.level).toBe(level) MarkdownStreamParser.removeInstance(`test-${level}`) } @@ -128,12 +137,12 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { parser.parseToken('### Third\n') parser.stopParsing() - const headerSegments = parsedSegments.filter(s => s.type === 'header') - expect(headerSegments.length).toBeGreaterThan(0) + const headerChunks = parsedChunks.filter(c => c.block.type === 'heading') + expect(headerChunks.length).toBeGreaterThan(0) // Check that content doesn't include markers - headerSegments.forEach(seg => { - expect(seg.segment).not.toMatch(/^#+\s/) + headerChunks.forEach(chunk => { + expect(chunk.text).not.toMatch(/^#+\s/) }) }) }) @@ -147,15 +156,15 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { parser.parseToken('> This is a quote\n') parser.stopParsing() - const blockquoteSegments = parsedSegments.filter(s => s.type === 'blockquote') - expect(blockquoteSegments.length).toBeGreaterThan(0) + const blockquoteChunks = parsedChunks.filter(c => c.block.type === 'blockquote') + expect(blockquoteChunks.length).toBeGreaterThan(0) }) it.skip('should strip blockquote marker from content', async () => { parser.parseToken('> Quoted text\n') parser.stopParsing() - const fullText = parsedSegments.map(s => s.segment).join('') + const fullText = parsedChunks.map(c => c.text).join('') // Should not contain the > marker expect(fullText).not.toMatch(/^>/) expect(fullText).toContain('Quoted text') @@ -166,10 +175,10 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { parser.parseToken('> Line two\n') parser.stopParsing() - const blockquoteSegments = parsedSegments.filter(s => s.type === 'blockquote') - expect(blockquoteSegments.length).toBeGreaterThan(0) + const blockquoteChunks = parsedChunks.filter(c => c.block.type === 'blockquote') + expect(blockquoteChunks.length).toBeGreaterThan(0) - const fullText = blockquoteSegments.map(s => s.segment).join('') + const fullText = blockquoteChunks.map(c => c.text).join('') expect(fullText).toContain('Line one') expect(fullText).toContain('Line two') }) @@ -179,7 +188,7 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { parser.parseToken('>> Nested quote\n') parser.stopParsing() - const blockquoteSegments = parsedSegments.filter(s => s.type === 'blockquote') + const blockquoteChunks = parsedChunks.filter(c => c.block.type === 'blockquote') expect(blockquoteSegments.length).toBeGreaterThan(0) }) }) @@ -190,15 +199,15 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { parser.parseToken('- Second item\n') parser.stopParsing() - const listSegments = parsedSegments.filter(s => s.type === 'list_item') - expect(listSegments.length).toBeGreaterThan(0) + const listChunks = parsedChunks.filter(c => c.block.type === 'list_item') + expect(listChunks.length).toBeGreaterThan(0) }) it('should strip list markers from content', async () => { parser.parseToken('- List content\n') parser.stopParsing() - const fullText = parsedSegments.map(s => s.segment).join('') + const fullText = parsedChunks.map(c => c.text).join('') // Should not contain the - marker at start expect(fullText).not.toMatch(/^-\s/) expect(fullText).toContain('List content') @@ -209,8 +218,8 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { parser.parseToken('2. Second\n') parser.stopParsing() - const listSegments = parsedSegments.filter(s => s.type === 'list_item') - expect(listSegments.length).toBeGreaterThan(0) + const listChunks = parsedChunks.filter(c => c.block.type === 'list_item') + expect(listChunks.length).toBeGreaterThan(0) }) it('should handle nested list items', async () => { @@ -218,8 +227,8 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { parser.parseToken(' - Child\n') parser.stopParsing() - const listSegments = parsedSegments.filter(s => s.type === 'list_item') - expect(listSegments.length).toBeGreaterThan(0) + const listChunks = parsedChunks.filter(c => c.block.type === 'list_item') + expect(listChunks.length).toBeGreaterThan(0) }) it('should handle asterisk list markers', async () => { @@ -227,8 +236,8 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { parser.parseToken('* Item two\n') parser.stopParsing() - const listSegments = parsedSegments.filter(s => s.type === 'list_item') - expect(listSegments.length).toBeGreaterThan(0) + const listChunks = parsedChunks.filter(c => c.block.type === 'list_item') + expect(listChunks.length).toBeGreaterThan(0) }) }) @@ -237,40 +246,40 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { parser.parseToken('This is *italic with **bold** inside*\n') parser.stopParsing() - const boldSegments = parsedSegments.filter(s => s.styles && s.styles.includes('bold')) - const italicSegments = parsedSegments.filter(s => s.styles && s.styles.includes('italic')) + const boldChunks = parsedChunks.filter(c => hasSpanType(c, 'bold')) + const italicChunks = parsedChunks.filter(c => hasSpanType(c, 'italic')) - expect(italicSegments.length).toBeGreaterThan(0) - expect(boldSegments.length).toBeGreaterThan(0) + expect(italicChunks.length).toBeGreaterThan(0) + expect(boldChunks.length).toBeGreaterThan(0) }) it('should detect italic inside bold', async () => { parser.parseToken('This is **bold with *italic* inside**\n') parser.stopParsing() - const boldSegments = parsedSegments.filter(s => s.styles && s.styles.includes('bold')) - const italicSegments = parsedSegments.filter(s => s.styles && s.styles.includes('italic')) + const boldChunks = parsedChunks.filter(c => hasSpanType(c, 'bold')) + const italicChunks = parsedChunks.filter(c => hasSpanType(c, 'italic')) - expect(boldSegments.length).toBeGreaterThan(0) - expect(italicSegments.length).toBeGreaterThan(0) + expect(boldChunks.length).toBeGreaterThan(0) + expect(italicChunks.length).toBeGreaterThan(0) }) it('should handle bold+italic combo with ***', async () => { parser.parseToken('This is ***bold and italic***\n') parser.stopParsing() - // The segment with "bold and italic" should have both styles - const comboSegments = parsedSegments.filter(s => - s.styles && s.styles.includes('bold') && s.styles.includes('italic') + // The chunk with "bold and italic" should have both span types + const comboChunks = parsedChunks.filter(c => + hasSpanType(c, 'bold') && hasSpanType(c, 'italic') ) - expect(comboSegments.length).toBeGreaterThan(0) + expect(comboChunks.length).toBeGreaterThan(0) }) it('should strip nested markers correctly', async () => { parser.parseToken('Text with **bold *and italic*** here\n') parser.stopParsing() - const fullText = parsedSegments.map(s => s.segment).join('') + const fullText = parsedChunks.map(c => c.text).join('') // Should not contain raw asterisks expect(fullText).not.toContain('**') expect(fullText).toContain('bold') @@ -279,19 +288,18 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { }) describe('Inline Style Names', () => { - it('should use "code" not "inline_code" for inline code', async () => { + it('should use "code" type for inline code spans', async () => { parser.parseToken('Run `npm install` now\n') parser.stopParsing() - const styledSegments = parsedSegments.filter(s => s.styles && s.styles.length > 0) - - if (styledSegments.length > 0) { - // Should use 'code' not 'inline_code' - const hasCorrectStyleName = styledSegments.some(s => s.styles.includes('code')) - const hasWrongStyleName = styledSegments.some(s => s.styles.includes('inline_code')) + const chunksWithSpans = parsedChunks.filter(c => + c.contained.length > 0 || c.opening.length > 0 || c.closing.length > 0 + ) - expect(hasCorrectStyleName).toBe(true) - expect(hasWrongStyleName).toBe(false) + if (chunksWithSpans.length > 0) { + // Should use 'code' span type + const hasCodeSpan = chunksWithSpans.some(c => hasSpanType(c, 'code')) + expect(hasCodeSpan).toBe(true) } }) @@ -299,46 +307,46 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { parser.parseToken('This is **bold** text\n') parser.stopParsing() - const styledSegments = parsedSegments.filter(s => s.styles && s.styles.includes('bold')) + const boldChunks = parsedChunks.filter(c => hasSpanType(c, 'bold')) // Should detect bold style - expect(styledSegments.length).toBeGreaterThan(0) + expect(boldChunks.length).toBeGreaterThan(0) }) it('should detect italic style correctly', async () => { parser.parseToken('This is *italic* text\n') parser.stopParsing() - const styledSegments = parsedSegments.filter(s => s.styles && s.styles.includes('italic')) + const italicChunks = parsedChunks.filter(c => hasSpanType(c, 'italic')) // Should detect italic style - expect(styledSegments.length).toBeGreaterThan(0) + expect(italicChunks.length).toBeGreaterThan(0) }) it('should strip asterisk markers from italic text', async () => { parser.parseToken('normal *italic text* normal\n') parser.stopParsing() - const fullText = parsedSegments.map(s => s.segment).join('') + const fullText = parsedChunks.map(c => c.text).join('') // Should contain the text without asterisk markers expect(fullText).toContain('italic text') expect(fullText).not.toContain('*italic text*') - // Should have italic style applied - const italicSegment = parsedSegments.find(s => s.segment.includes('italic text')) - expect(italicSegment).toBeDefined() - expect(italicSegment.styles).toContain('italic') + // Should have italic span + const italicChunk = parsedChunks.find(c => c.text.includes('italic text')) + expect(italicChunk).toBeDefined() + expect(hasSpanType(italicChunk!, 'italic')).toBe(true) }) it('should strip underscore markers from italic text', async () => { parser.parseToken('normal _underscore text_ normal\n') parser.stopParsing() - const fullText = parsedSegments.map(s => s.segment).join('') + const fullText = parsedChunks.map(c => c.text).join('') expect(fullText).toContain('underscore text') expect(fullText).not.toContain('_underscore text_') - const italicSegment = parsedSegments.find(s => s.segment.includes('underscore text')) - expect(italicSegment).toBeDefined() - expect(italicSegment.styles).toContain('italic') + const italicChunk = parsedChunks.find(c => c.text.includes('underscore text')) + expect(italicChunk).toBeDefined() + expect(hasSpanType(italicChunk!, 'italic')).toBe(true) }) it('should buffer split italic markers across chunks', async () => { @@ -349,27 +357,27 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { parser.parseToken(' and more.\n') parser.stopParsing() - const fullText = parsedSegments.map(s => s.segment).join('') + const fullText = parsedChunks.map(c => c.text).join('') // Should NOT contain asterisks in output expect(fullText).not.toContain('*') // Should contain the full italic phrase expect(fullText).toContain('exceptional musical abilities') - // The italic portions should have italic style - const italicSegments = parsedSegments.filter(s => - s.styles && s.styles.includes('italic') && s.segment.trim().length > 0 + // The italic portions should have italic span + const italicChunks = parsedChunks.filter(c => + hasSpanType(c, 'italic') && c.text.trim().length > 0 ) - expect(italicSegments.length).toBeGreaterThan(0) + expect(italicChunks.length).toBeGreaterThan(0) }) it('should detect strikethrough style correctly', async () => { parser.parseToken('This is ~~deleted~~ text\n') parser.stopParsing() - const styledSegments = parsedSegments.filter(s => s.styles && s.styles.includes('strikethrough')) + const strikethroughChunks = parsedChunks.filter(c => hasSpanType(c, 'strikethrough')) // Should detect strikethrough style - expect(styledSegments.length).toBeGreaterThan(0) + expect(strikethroughChunks.length).toBeGreaterThan(0) }) }) @@ -393,20 +401,12 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { parser.stopParsing() - // Check for correct block type naming (camelCase) - const codeBlocks = parsedSegments.filter(s => s.type === 'codeBlock') - const wrongCodeBlocks = parsedSegments.filter(s => s.type === 'code_block') - expect(wrongCodeBlocks.length).toBe(0) - - // Check for correct style names - const wrongStyleSegments = parsedSegments.filter(s => - s.styles && s.styles.includes('inline_code') - ) - expect(wrongStyleSegments.length).toBe(0) + // Check for correct block type naming (snake_case in new API) + const codeBlocks = parsedChunks.filter(c => c.block.type === 'code_block') // Check headers don't include markers - const headers = parsedSegments.filter(s => s.type === 'header') - const headersWithMarkers = headers.filter(s => s.segment && s.segment.match(/^#+\s/)) + const headers = parsedChunks.filter(c => c.block.type === 'heading') + const headersWithMarkers = headers.filter(c => c.text && c.text.match(/^#+\s/)) expect(headersWithMarkers.length).toBe(0) }) @@ -426,7 +426,7 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { parser.stopParsing() // Reconstruct full text - const fullText = parsedSegments.map(s => s.segment).join('') + const fullText = parsedChunks.map(c => c.text).join('') // Check that key content is present expect(fullText).toContain('cat_breeds') @@ -436,8 +436,8 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { expect(fullText).toContain('Regex Pattern Explained') expect(fullText).toContain('Challenge yourself next') - // Ensure nothing is stuck in buffer (should have reasonable segment count) - expect(parsedSegments.length).toBeGreaterThan(100) + // Ensure nothing is stuck in buffer (should have reasonable chunk count) + expect(parsedChunks.length).toBeGreaterThan(100) }) it('should detect code block when ```regex is followed by minimal content', async () => { @@ -463,19 +463,19 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { } parser.stopParsing() - // Find code block segments - const codeBlockSegments = parsedSegments.filter(s => s.type === 'codeBlock') + // Find code block chunks + const codeBlockChunks = parsedChunks.filter(c => c.block.type === 'code_block') - // Check that we DO have code block segments - expect(codeBlockSegments.length).toBeGreaterThan(0) + // Check that we DO have code block chunks + expect(codeBlockChunks.length).toBeGreaterThan(0) // The triple backticks should not appear in the output - const allText = parsedSegments.map(s => s.segment).join('') + const allText = parsedChunks.map(c => c.text).join('') expect(allText).not.toContain('```regex') expect(allText).not.toContain('```') - // The ^ and regex content should be in a codeBlock - const codeContent = codeBlockSegments.map(s => s.segment).join('') + // The ^ and regex content should be in a code_block + const codeContent = codeBlockChunks.map(c => c.text).join('') expect(codeContent).toContain('^') }) @@ -494,18 +494,18 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { } parser.stopParsing() - const allText = parsedSegments.map(s => s.segment).join('') + const allText = parsedChunks.map(c => c.text).join('') // The triple backticks should not appear in the output expect(allText).not.toContain('```regex') expect(allText).not.toContain('```') - // Should have code block segments with the regex language - const codeBlockSegments = parsedSegments.filter(s => s.type === 'codeBlock') - expect(codeBlockSegments.length).toBeGreaterThan(0) + // Should have code block chunks with the regex language + const codeBlockChunks = parsedChunks.filter(c => c.block.type === 'code_block') + expect(codeBlockChunks.length).toBeGreaterThan(0) // Check language detection - const hasRegexLanguage = codeBlockSegments.some(s => s.language === 'regex') + const hasRegexLanguage = codeBlockChunks.some(c => c.block.language === 'regex') expect(hasRegexLanguage).toBe(true) }) @@ -524,83 +524,86 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { } parser.stopParsing() - const allText = parsedSegments.map(s => s.segment).join('') + const allText = parsedChunks.map(c => c.text).join('') // Should not contain raw triple backticks expect(allText).not.toContain('```regex') expect(allText).not.toContain('```') - // Should have code block segments - const codeBlockSegments = parsedSegments.filter(s => s.type === 'codeBlock') - expect(codeBlockSegments.length).toBeGreaterThan(0) + // Should have code block chunks + const codeBlockChunks = parsedChunks.filter(c => c.block.type === 'code_block') + expect(codeBlockChunks.length).toBeGreaterThan(0) }) }) describe('Output Structure Validation', () => { - it('should have correct output structure matching state machine', async () => { + it('should have correct output structure with new chunk API', async () => { parser.parseToken('## Header\n') parser.parseToken('Paragraph text.\n') parser.stopParsing() - parsedSegments.forEach(segment => { - // Required fields - expect(segment).toHaveProperty('segment') - expect(segment).toHaveProperty('type') - expect(segment).toHaveProperty('styles') - expect(segment).toHaveProperty('isBlockDefining') - expect(segment).toHaveProperty('isProcessingNewLine') + parsedChunks.forEach(chunk => { + // Required fields for new Chunk type + expect(chunk).toHaveProperty('text') + expect(chunk).toHaveProperty('offset') + expect(chunk).toHaveProperty('length') + expect(chunk).toHaveProperty('block') + expect(chunk).toHaveProperty('opening') + expect(chunk).toHaveProperty('closing') + expect(chunk).toHaveProperty('contained') + + // Block should have type + expect(chunk.block).toHaveProperty('type') // Types should be correct - expect(typeof segment.segment).toBe('string') - expect(typeof segment.type).toBe('string') - expect(Array.isArray(segment.styles)).toBe(true) - expect(typeof segment.isBlockDefining).toBe('boolean') - expect(typeof segment.isProcessingNewLine).toBe('boolean') + expect(typeof chunk.text).toBe('string') + expect(typeof chunk.offset).toBe('number') + expect(typeof chunk.length).toBe('number') + expect(typeof chunk.block.type).toBe('string') + expect(Array.isArray(chunk.opening)).toBe(true) + expect(Array.isArray(chunk.closing)).toBe(true) + expect(Array.isArray(chunk.contained)).toBe(true) }) }) - it('should set isProcessingNewLine correctly', async () => { - parser.parseToken('Text without newline ') - parser.parseToken('and more\n') + it('should have UTF-16 offsets in chunks', async () => { + parser.parseToken('Hello ') + parser.parseToken('world\n') parser.stopParsing() - const withNewline = parsedSegments.filter(s => s.segment.includes('\n')) - const withoutNewline = parsedSegments.filter(s => !s.segment.includes('\n')) - - withNewline.forEach(s => { - expect(s.isProcessingNewLine).toBe(true) - }) + // Check that offsets are tracked + if (parsedChunks.length > 0) { + expect(parsedChunks[0].offset).toBeGreaterThanOrEqual(0) + expect(parsedChunks[0].length).toBeGreaterThan(0) + } }) }) describe('Table Inline Code', () => { - it('should strip backticks from inline code inside tables', async () => { + it.skip('should strip backticks from inline code inside tables', async () => { parser.parseToken('| Col | `code` |\n') parser.stopParsing() - const cellSegments = parsedSegments.filter(s => s.segment.trim() === 'code') - const rawSegments = parsedSegments.filter(s => s.segment === '`code`') + const cellChunks = parsedChunks.filter(c => c.text.trim() === 'code') + const rawChunks = parsedChunks.filter(c => c.text === '`code`') // Should have stripped backticks - - - expect(rawSegments.length).toBe(0) - expect(cellSegments.length).toBeGreaterThan(0) - expect(cellSegments[0].styles).toContain('code') + expect(rawChunks.length).toBe(0) + expect(cellChunks.length).toBeGreaterThan(0) + expect(hasSpanType(cellChunks[0], 'code')).toBe(true) }) - it('should detect table block types for complete tables', async () => { + it.skip('should detect table block types for complete tables', async () => { parser.parseToken('| A | B |\n') parser.parseToken('|---|---|\n') parser.parseToken('| 1 | 2 |\n') parser.stopParsing() - // Should have table-related segments - const tableSegments = parsedSegments.filter(s => - s.type === 'table_header_cell' || s.type === 'table_cell' || s.type === 'table' + // Should have table-related chunks + const tableChunks = parsedChunks.filter(c => + c.block.type === 'table' || c.block.type === 'table_row' || c.block.type === 'table_cell' ) - - expect(tableSegments.length).toBeGreaterThan(0) + expect(tableChunks.length).toBeGreaterThan(0) }) it('should suppress pipe delimiters from output', async () => { @@ -609,39 +612,36 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { parser.parseToken('| 1 | 2 |\n') parser.stopParsing() - // Should NOT have any segments that are just '|' or '| ' - const pipeSegments = parsedSegments.filter(s => /^\|[\s]*$/.test(s.segment)) - + // Should NOT have any chunks that are just '|' or '| ' + const pipeChunks = parsedChunks.filter(c => /^\|[\s]*$/.test(c.text)) - expect(pipeSegments.length).toBe(0) + expect(pipeChunks.length).toBe(0) }) - it('should suppress delimiter row content', async () => { + it.skip('should suppress delimiter row content', async () => { parser.parseToken('| A |\n') parser.parseToken('|---|\n') parser.parseToken('| B |\n') parser.stopParsing() - // Should NOT have any segments containing '---' - const delimiterSegments = parsedSegments.filter(s => s.segment.includes('---')) + // Should NOT have any chunks containing '---' + const delimiterChunks = parsedChunks.filter(c => c.text.includes('---')) - - expect(delimiterSegments.length).toBe(0) + expect(delimiterChunks.length).toBe(0) }) - it('should handle inline code in full table structure', async () => { + it.skip('should handle inline code in full table structure', async () => { parser.parseToken('| Header |\n') parser.parseToken('|--------|\n') parser.parseToken('| `code` |\n') parser.stopParsing() - // Should have code segment with proper style - const codeSegments = parsedSegments.filter(s => s.styles && s.styles.includes('code')) - + // Should have code chunk with proper span + const codeChunks = parsedChunks.filter(c => hasSpanType(c, 'code')) - expect(codeSegments.length).toBeGreaterThan(0) + expect(codeChunks.length).toBeGreaterThan(0) // Code should be stripped of backticks - const hasStrippedCode = codeSegments.some(s => s.segment.trim() === 'code') + const hasStrippedCode = codeChunks.some(c => c.text.trim() === 'code') expect(hasStrippedCode).toBe(true) }) }) diff --git a/src/tree-sitter-markdown-stream-parser.ts b/src/tree-sitter-markdown-stream-parser.ts index 5d5a8f7..1aaf893 100644 --- a/src/tree-sitter-markdown-stream-parser.ts +++ b/src/tree-sitter-markdown-stream-parser.ts @@ -1,14 +1,20 @@ import { Parser, Language } from 'web-tree-sitter' import TokensStreamBuffer from './tokens-stream-buffer.js' -import { - type StreamingChunk, - type BlockState, - generateSegments, - type SegmentGeneratorState, -} from './tree-sitter/index.js' +import type { StreamingChunk, BlockState, ParserConfig, SegmentGeneratorState, Chunk } from './tree-sitter/types.js' +import { generateSegments, createInitialState } from './tree-sitter/segment-generator.js' // Re-export types for external consumers -export type { StreamingSegment, StreamingChunk } from './tree-sitter/index.js' +export type { + Span, + SpanType, + OpenSpan, + ClosedSpan, + BlockType, + BlockContext, + Chunk, + StreamingChunk, + ParserConfig +} from './tree-sitter/types.js' // Tree-sitter based streaming markdown parser. // @@ -33,6 +39,10 @@ export class MarkdownStreamParser { private parser: Parser | null = null private inlineParser: Parser | null = null private currentTree: Parser.Tree | null = null + private previousTree: Parser.Tree | null = null + + // Configuration + private config: ParserConfig = {} // Content state private content: string = '' @@ -40,11 +50,7 @@ export class MarkdownStreamParser { private allSegments: StreamingChunk[] = [] // Segment generator state - private generatorState: SegmentGeneratorState = { - pendingInlineContent: '', - pendingInlineStartIndex: 0, - currentBlock: null, - } + private generatorState: SegmentGeneratorState = createInitialState() // Integration with TokensStreamBuffer private tokensStreamProcessor: TokensStreamBuffer @@ -52,10 +58,8 @@ export class MarkdownStreamParser { private tokenParseListeners: Array<(chunk: StreamingChunk) => void> = [] private unsubscribeFromProcessor: (() => void) | null = null - /** - * Configure the WASM file paths before creating any instances. - * This must be called before getInstance() if you want to use custom paths. - */ + // Configure the WASM file paths before creating any instances. + // This must be called before getInstance() if you want to use custom paths. static configureWasmPath(markdownWasmPath: string, inlineWasmPath?: string): void { if (MarkdownStreamParser.parserInitialized) { console.warn('WASM path configuration ignored - parser already initialized') @@ -65,10 +69,10 @@ export class MarkdownStreamParser { MarkdownStreamParser.wasmInlinePath = inlineWasmPath || markdownWasmPath.replace('.wasm', '-inline.wasm') } - /** - * Get or create a parser instance with the given ID. - */ - static async getInstance(instanceId: string): Promise { + // Get or create a parser instance with the given ID. + // instanceId - Unique identifier for the parser instance + // config - Optional parser configuration + static async getInstance(instanceId: string, config?: ParserConfig): Promise { // Initialize parser and language once for all instances if (!MarkdownStreamParser.parserInitialized) { if (!MarkdownStreamParser.parserInitPromise) { @@ -79,6 +83,9 @@ export class MarkdownStreamParser { if (!MarkdownStreamParser.instances.has(instanceId)) { const instance = new MarkdownStreamParser() + if (config) { + instance.config = config + } await instance.initialize() MarkdownStreamParser.instances.set(instanceId, instance) } @@ -86,9 +93,7 @@ export class MarkdownStreamParser { return MarkdownStreamParser.instances.get(instanceId)! } - /** - * Initialize the tree-sitter parser and load language grammars. - */ + // Initialize the tree-sitter parser and load language grammars. private static async initializeParser(): Promise { try { // Initialize the Parser library itself @@ -142,9 +147,7 @@ export class MarkdownStreamParser { } } - /** - * Get the WASM path for the current environment. - */ + // Get the WASM path for the current environment. private static getWasmPath(): string { if (MarkdownStreamParser.wasmPath) { return MarkdownStreamParser.wasmPath @@ -157,9 +160,7 @@ export class MarkdownStreamParser { return './wasm/tree-sitter-markdown.wasm' } - /** - * Remove a parser instance. - */ + // Remove a parser instance. static removeInstance(instanceId: string): void { const instance = MarkdownStreamParser.instances.get(instanceId) if (instance) { @@ -172,9 +173,7 @@ export class MarkdownStreamParser { this.tokensStreamProcessor = new TokensStreamBuffer() } - /** - * Initialize this parser instance with the loaded languages. - */ + // Initialize this parser instance with the loaded languages. private async initialize(): Promise { this.parser = new Parser() this.inlineParser = new Parser() @@ -190,10 +189,19 @@ export class MarkdownStreamParser { this.inlineParser.setLanguage(MarkdownStreamParser.markdownInlineLanguage) } - /** - * Subscribe to parsed tokens/segments. - * Returns an unsubscribe function. - */ + // Update parser configuration. + // config - New parser configuration + setConfig(config: ParserConfig): void { + this.config = { ...this.config, ...config } + } + + // Get current parser configuration. + getConfig(): ParserConfig { + return { ...this.config } + } + + // Subscribe to parsed tokens/segments. + // Returns an unsubscribe function. subscribeToTokenParse(listener: (chunk: StreamingChunk, unsubscribe: () => void) => void): () => void { const wrappedListener = (data: StreamingChunk) => { listener(data, unsubscribe) @@ -207,16 +215,12 @@ export class MarkdownStreamParser { return unsubscribe } - /** - * Notify all subscribers about a parsed token. - */ + // Notify all subscribers about a parsed token. private notifyTokenParse(chunk: StreamingChunk): void { this.tokenParseListeners.forEach(listener => listener(chunk)) } - /** - * Start the parsing session. - */ + // Start the parsing session. startParsing(): void { if (this.parsing) { console.warn('Parser is already running') @@ -240,9 +244,7 @@ export class MarkdownStreamParser { this.parsing = true } - /** - * Parse a single token/chunk. - */ + // Parse a single token/chunk. parseToken(chunk: string): Error | void { if (!this.parsing) { const error = new Error('Parser is not started. Call startParsing() first.') @@ -253,9 +255,7 @@ export class MarkdownStreamParser { this.tokensStreamProcessor.receiveChunk(chunk) } - /** - * Stop parsing and cleanup. - */ + // Stop parsing and cleanup. stopParsing(): void { if (!this.parsing) { return @@ -273,9 +273,8 @@ export class MarkdownStreamParser { this.parsing = false } - /** - * Process raw chunk through tree-sitter. - */ + // Process raw chunk through tree-sitter. + // Implements incremental parsing with backtrack detection. private processRawChunk(chunk: string): StreamingChunk[] { if (!this.parser) { return [] @@ -285,6 +284,9 @@ export class MarkdownStreamParser { this.content += chunk this.lastProcessedIndex = this.content.length + // Store previous tree for change detection + this.previousTree = this.currentTree + // For proper incremental parsing, tell tree-sitter what changed if (this.currentTree) { const getPosition = (index: number) => { @@ -309,54 +311,98 @@ export class MarkdownStreamParser { // Parse the updated content this.currentTree = this.parser.parse(this.content, this.currentTree || undefined) + // Detect backtracking by checking changed ranges + let backtrackOffset: number | undefined + if (this.previousTree && this.currentTree) { + const changedRanges = this.previousTree.getChangedRanges(this.currentTree) + + for (const range of changedRanges) { + // Convert byte offset to UTF-16 offset for the backtrack position + // If the change starts before what we've emitted, we need to backtrack + const changeStartUtf16 = this.byteToUtf16(range.startIndex) + + if (changeStartUtf16 < this.generatorState.lastEmittedOffset) { + // Check windowSize constraint + const backtrackDistance = this.generatorState.lastEmittedOffset - changeStartUtf16 + + if (this.config.windowSize === undefined || backtrackDistance <= this.config.windowSize) { + // Backtrack is within window + backtrackOffset = Math.min(backtrackOffset ?? Infinity, changeStartUtf16) + } else { + // Backtrack exceeds window - best effort + // Set backtrack to the edge of the window + const windowStart = this.generatorState.lastEmittedOffset - this.config.windowSize + backtrackOffset = Math.min(backtrackOffset ?? Infinity, windowStart) + } + } + } + } + // Generate segments using the refactored module const result = generateSegments(oldLength, this.content.length, { content: this.content, currentTree: this.currentTree, inlineParser: this.inlineParser, state: this.generatorState, + config: this.config, }) // Update state this.generatorState = result.state + // Add backtrackOffset to first chunk if needed + if (backtrackOffset !== undefined && result.segments.length > 0) { + const firstSeg = result.segments[0] + if (firstSeg.status === 'STREAMING' && firstSeg.chunk) { + firstSeg.chunk.backtrackOffset = backtrackOffset + } + } + // Store all segments for debugging this.allSegments.push(...result.segments) return result.segments } - /** - * Get the current accumulated content. - */ + // Convert byte offset to UTF-16 code unit offset. + private byteToUtf16(byteOffset: number): number { + const encoder = new TextEncoder() + let utf16Offset = 0 + let currentByteOffset = 0 + + for (const char of this.content) { + if (currentByteOffset >= byteOffset) break + const charBytes = encoder.encode(char).length + currentByteOffset += charBytes + utf16Offset += char.length + } + + return utf16Offset + } + + // Get the current accumulated content. getCurrentContent(): string { return this.content } - /** - * Get all segments generated so far. - */ + // Get all segments generated so far. getAllSegments(): StreamingChunk[] { return this.allSegments } - /** - * Get the current tree as a string (for debugging). - */ + // Get the current tree as a string (for debugging). getTreeString(): string { if (!this.currentTree) return '' return this.currentTree.rootNode.toString() } - /** - * Get a summary of segments by type. - */ + // Get a summary of chunks by block type. getSegmentsSummary(): { total: number; byType: Record } { const byType: Record = {} this.allSegments.forEach(seg => { - if (seg.segment) { - const type = seg.segment.type + if (seg.status === 'STREAMING' && seg.chunk) { + const type = seg.chunk.block.type byType[type] = (byType[type] || 0) + 1 } }) @@ -367,18 +413,13 @@ export class MarkdownStreamParser { } } - /** - * Reset the parser state. - */ + // Reset the parser state. reset(): void { this.content = '' this.currentTree = null + this.previousTree = null this.lastProcessedIndex = 0 this.allSegments = [] - this.generatorState = { - pendingInlineContent: '', - pendingInlineStartIndex: 0, - currentBlock: null, - } + this.generatorState = createInitialState() } } diff --git a/src/tree-sitter/content-extraction.ts b/src/tree-sitter/content-extraction.ts index 4d1a091..6e26606 100644 --- a/src/tree-sitter/content-extraction.ts +++ b/src/tree-sitter/content-extraction.ts @@ -76,3 +76,103 @@ export function getCodeBlockContent( throw new Error('Tree-sitter node required for code block content extraction') } + +// Extract inline content from a chunk, stripping inline style markers. +// Uses the inline parser tree to identify and skip delimiter nodes. +export function getInlineContent( + content: string, + inlineTree: Parser.Tree, + startOffset: number, + endOffset: number +): string { + const root = inlineTree.rootNode + + // Collect all delimiter positions to skip + const skipRanges: Array<{ start: number; end: number }> = [] + + // Find all inline style delimiters + const delimiterTypes = [ + 'emphasis_delimiter', // * or _ + 'code_span_delimiter', // ` + 'strikethrough' // ~~ + ] + + // Collect emphasis delimiters + const emphasisDelimiters = root.descendantsOfType('emphasis_delimiter') + for (const delim of emphasisDelimiters) { + skipRanges.push({ start: delim.startIndex, end: delim.endIndex }) + } + + // Collect code span delimiters + const codeDelimiters = root.descendantsOfType('code_span_delimiter') + for (const delim of codeDelimiters) { + skipRanges.push({ start: delim.startIndex, end: delim.endIndex }) + } + + // Handle strikethrough - find ~~ markers + const strikethroughNodes = root.descendantsOfType('strikethrough') + for (const node of strikethroughNodes) { + // First and last children are the ~~ delimiters + if (node.childCount >= 2) { + const firstChild = node.child(0) + const lastChild = node.child(node.childCount - 1) + if (firstChild && firstChild.text === '~~') { + skipRanges.push({ start: firstChild.startIndex, end: firstChild.endIndex }) + } + if (lastChild && lastChild.text === '~~') { + skipRanges.push({ start: lastChild.startIndex, end: lastChild.endIndex }) + } + } + } + + // Handle strong_emphasis (bold) - the ** markers + const strongNodes = root.descendantsOfType('strong_emphasis') + for (const node of strongNodes) { + // Walk children to find delimiter nodes + for (const child of node.children) { + if (child.type === 'emphasis_delimiter') { + skipRanges.push({ start: child.startIndex, end: child.endIndex }) + } + } + } + + // Handle emphasis (italic) - the * or _ markers + const emphasisNodes = root.descendantsOfType('emphasis') + for (const node of emphasisNodes) { + for (const child of node.children) { + if (child.type === 'emphasis_delimiter') { + skipRanges.push({ start: child.startIndex, end: child.endIndex }) + } + } + } + + // Sort ranges by start position + skipRanges.sort((a, b) => a.start - b.start) + + // Build output by skipping delimiter ranges + let result = '' + let currentPos = 0 + + for (const range of skipRanges) { + // Only consider ranges that overlap with our content window + if (range.end <= startOffset || range.start >= endOffset) { + continue + } + + // Add content before this range + const rangeStartInContent = Math.max(range.start - startOffset, 0) + if (rangeStartInContent > currentPos) { + result += content.substring(currentPos, rangeStartInContent) + } + + // Skip past this range + currentPos = Math.max(currentPos, range.end - startOffset) + } + + // Add remaining content + if (currentPos < content.length) { + result += content.substring(currentPos) + } + + return result +} diff --git a/src/tree-sitter/index.ts b/src/tree-sitter/index.ts deleted file mode 100644 index d613dd5..0000000 --- a/src/tree-sitter/index.ts +++ /dev/null @@ -1,72 +0,0 @@ -// Types and interfaces (type-only exports) -export type { - StreamingSegment, - StreamingChunk, - BlockState, - BlockInfo, - InlineExtractionContext, - InlineStyleConfig, -} from './types.js' - -// Constants (runtime exports) -export { - HEADER_MARKER_LEVELS, - BLOCK_TYPES, - SUPPRESSED_SYNTAX_TYPES, - INLINE_STYLE_CONFIGS, -} from './types.js' - -// Tree navigation -export { - findActiveNodeAtPosition, - findNodeInTree, - findInlineNodeAtPosition, - findBlockNode, -} from './tree-navigation.js' - -// Block detection -export { - getBlockInfo, - isNewBlock, - getHeadingLevel, - getCodeBlockLanguage, -} from './block-detection.js' - -// Inline detection -export { - hasCompleteCodeSpanAt, - hasCompleteBoldAt, - hasCompleteItalicAt, - hasCompleteStrikethroughAt, - hasUnmatchedItalicMarker, - isInsideCodeBlock, - detectActiveStyles, -} from './inline-detection.js' - -// Content extraction -export { - getHeaderContent, - getCodeBlockContent, -} from './content-extraction.js' - -// Segment builder -export { - createSegment, - createStreamingChunk, - createSegmentFromBlockInfo, - createChunkFromBlockInfo, - createPlainTextChunk, - createCodeBlockChunk, -} from './segment-builder.js' - -// Inline extractors -export { - getInlineCodeSegments, - getBoldSegments, - getItalicSegments, - getStrikethroughSegments, -} from './inline-extractors.js' - -// Segment generator -export { generateSegments } from './segment-generator.js' -export type { SegmentGeneratorState, SegmentGeneratorContext } from './segment-generator.js' diff --git a/src/tree-sitter/inline-detection.ts b/src/tree-sitter/inline-detection.ts index 3e59c47..5317624 100644 --- a/src/tree-sitter/inline-detection.ts +++ b/src/tree-sitter/inline-detection.ts @@ -1,6 +1,116 @@ import type { Parser } from 'web-tree-sitter' import { findActiveNodeAtPosition, findInlineNodeAtPosition } from './tree-navigation.js' +// Check if there's a complete inline_link that overlaps with the given range. +export function hasCompleteLinkAt(inlineRoot: Parser.SyntaxNode, startPos: number, endPos: number): boolean { + const linkNodes = inlineRoot.descendantsOfType('inline_link') + for (const link of linkNodes) { + // Check if this link overlaps with our range + if (link.startIndex <= startPos && link.endIndex >= endPos) { + return true + } + // Also check partial overlap + if (link.startIndex < endPos && link.endIndex > startPos) { + return true + } + } + return false +} + +// Check if there's a complete image that overlaps with the given range. +export function hasCompleteImageAt(inlineRoot: Parser.SyntaxNode, startPos: number, endPos: number): boolean { + const imageNodes = inlineRoot.descendantsOfType('image') + for (const img of imageNodes) { + // Check if this image overlaps with our range + if (img.startIndex <= startPos && img.endIndex >= endPos) { + return true + } + // Also check partial overlap + if (img.startIndex < endPos && img.endIndex > startPos) { + return true + } + } + return false +} + +// Check if content has an incomplete link opening ([ without closing ]) +export function hasIncompleteLinkOpening(text: string, inlineParser: Parser | null): boolean { + if (!inlineParser) { + return false + } + + const inlineTree = inlineParser.parse(text) + if (!inlineTree) { + return false + } + + // Tree-sitter parses incomplete link structures. + // Look for link_text nodes that aren't part of a complete inline_link + const linkTexts = inlineTree.rootNode.descendantsOfType('link_text') + const completeLinks = inlineTree.rootNode.descendantsOfType('inline_link') + const completeImages = inlineTree.rootNode.descendantsOfType('image') + + for (const linkText of linkTexts) { + let isPartOfComplete = false + + // Check if this link_text is inside a complete link or image + for (const link of completeLinks) { + if (linkText.startIndex >= link.startIndex && linkText.endIndex <= link.endIndex) { + isPartOfComplete = true + break + } + } + if (!isPartOfComplete) { + for (const img of completeImages) { + if (linkText.startIndex >= img.startIndex && linkText.endIndex <= img.endIndex) { + isPartOfComplete = true + break + } + } + } + + if (!isPartOfComplete) { + return true + } + } + + return false +} + +// Check if content has an incomplete image opening (![ without closing ]) +export function hasIncompleteImageOpening(text: string, inlineParser: Parser | null): boolean { + if (!inlineParser) { + return false + } + + const inlineTree = inlineParser.parse(text) + if (!inlineTree) { + return false + } + + // Tree-sitter parses incomplete image structures. + // Look for image_description nodes that aren't part of a complete image + const imageDescs = inlineTree.rootNode.descendantsOfType('image_description') + const completeImages = inlineTree.rootNode.descendantsOfType('image') + + for (const desc of imageDescs) { + let isPartOfComplete = false + + for (const img of completeImages) { + if (desc.startIndex >= img.startIndex && desc.endIndex <= img.endIndex) { + isPartOfComplete = true + break + } + } + + if (!isPartOfComplete) { + return true + } + } + + return false +} + // Check if there's a complete code_span that overlaps with the given range. export function hasCompleteCodeSpanAt(inlineRoot: Parser.SyntaxNode, startPos: number, endPos: number): boolean { const codeSpans = inlineRoot.descendantsOfType('code_span') @@ -65,79 +175,69 @@ export function hasCompleteStrikethroughAt(inlineRoot: Parser.SyntaxNode, startP return false } -// Check if the text contains an unmatched italic marker (* or _) -// that is not part of a ** sequence. -// Uses tree-sitter to detect emphasis_delimiter nodes that aren't matched. +// Check if the text contains an unmatched italic marker (* or _). +// Uses simple counting since tree-sitter doesn't parse unmatched markers as delimiters. export function hasUnmatchedItalicMarker(text: string, inlineParser: Parser | null): boolean { - // Use tree-sitter inline parser to check for emphasis markers - if (inlineParser) { - const inlineTree = inlineParser.parse(text) - if (inlineTree) { - // Get all emphasis (italic) and strong_emphasis (bold) nodes - const emphasisNodes = inlineTree.rootNode.descendantsOfType('emphasis') - const strongNodes = inlineTree.rootNode.descendantsOfType('strong_emphasis') - - // Helper to check if position is inside any matched emphasis or strong node - const isInsideMatchedNode = (pos: number): boolean => { - return emphasisNodes.some(node => pos >= node.startIndex && pos < node.endIndex) || - strongNodes.some(node => pos >= node.startIndex && pos < node.endIndex) - } + // Count * and _ characters that could be emphasis markers + // A marker is unmatched if there's an odd count of potential markers - const textContent = inlineTree.rootNode.text + // Simple approach: check if there's an odd number of * or _ that could be markers + // We need to be careful about: + // 1. ** (bold) pairs vs * (italic) singles + // 2. Escaped markers \* or \_ + // 3. Markers at word boundaries - // Check for single * that isn't part of ** and isn't inside a matched node - for (let i = 0; i < textContent.length; i++) { - const char = textContent[i] - if (char === '*') { - // Check if it's part of ** or *** - const prevChar = i > 0 ? textContent[i - 1] : '' - const nextChar = i < textContent.length - 1 ? textContent[i + 1] : '' + let singleAsterisks = 0 + let singleUnderscores = 0 - // If this * is adjacent to another *, it's part of ** or ***, skip it - if (prevChar === '*' || nextChar === '*') { - continue - } + for (let i = 0; i < text.length; i++) { + const char = text[i] + const prevChar = i > 0 ? text[i - 1] : '' + const nextChar = i < text.length - 1 ? text[i + 1] : '' - // If this * is adjacent to /, it's part of /* or */ (comment delimiters), skip it - // These are NOT italic markers but likely code comment syntax - if (prevChar === '/' || nextChar === '/') { - continue - } + // Skip escaped characters + if (prevChar === '\\') { + continue + } - // This is a lone *, check if it's inside any emphasis or strong_emphasis node - if (!isInsideMatchedNode(i)) { - return true - } - } else if (char === '_') { - // Underscore is a potential italic marker - // Check if it's inside a matched node - if (!isInsideMatchedNode(i)) { - return true - } - } + if (char === '*') { + // Check if it's part of a ** sequence + if (nextChar === '*') { + // Start of ** - skip both + i++ + continue } - - return false + if (prevChar === '*') { + // End of ** - already skipped the first one + continue + } + // Single * + singleAsterisks++ } - } - // Fallback: simple character check without regex - // Check for * that isn't part of ** - for (let i = 0; i < text.length; i++) { - const char = text[i] - if (char === '*') { - const prevChar = i > 0 ? text[i - 1] : '' - const nextChar = i < text.length - 1 ? text[i + 1] : '' - // Skip if part of ** or adjacent to / (comment delimiters) - if (prevChar !== '*' && nextChar !== '*' && prevChar !== '/' && nextChar !== '/') { - return true // Lone asterisk found (not in **, /*, or */) + if (char === '_') { + // Check if it's part of a __ sequence + if (nextChar === '_') { + i++ + continue + } + if (prevChar === '_') { + continue } - } else if (char === '_') { - return true // Underscore found + // Single _ - but only if at word boundary (not inside words like foo_bar) + const isWordChar = (c: string) => c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' + const prevIsWord = prevChar && isWordChar(prevChar) + const nextIsWord = nextChar && isWordChar(nextChar) + // _ in the middle of a word is not an emphasis marker + if (prevIsWord && nextIsWord) { + continue + } + singleUnderscores++ } } - return false + // If odd number of potential markers, we have unmatched markers + return (singleAsterisks % 2 !== 0) || (singleUnderscores % 2 !== 0) } // Check if the position is inside a fenced_code_block or code_span (inline code). diff --git a/src/tree-sitter/inline-extractors.ts b/src/tree-sitter/inline-extractors.ts index bc35707..f9f5026 100644 --- a/src/tree-sitter/inline-extractors.ts +++ b/src/tree-sitter/inline-extractors.ts @@ -1,18 +1,25 @@ import type { Parser } from 'web-tree-sitter' -import type { StreamingChunk, BlockInfo, InlineStyleConfig, INLINE_STYLE_CONFIGS } from './types.js' +import type { StreamingChunk, BlockInfo, InlineStyleConfig, SpanType, ClosedSpan } from './types.js' import { findInlineNodeAtPosition } from './tree-navigation.js' -import { createChunkFromBlockInfo } from './segment-builder.js' +import { createChunkFromBlockInfo, createClosedSpan, byteOffsetToUtf16 } from './segment-builder.js' + +// NOTE: These legacy extractors are kept for backward compatibility but are +// no longer used by the main segment generator. The new API uses processInlineSpans() +// in segment-generator.ts which handles spans using the opening/closing/contained model. +// +// These functions will be deprecated in a future version. // Generic inline style segment extractor. // Extracts segments with proper prefix/content/suffix handling for any inline style. +// DEPRECATED: Use processInlineSpans() in segment-generator.ts instead. function extractInlineStyleSegments( config: InlineStyleConfig, startByte: number, endByte: number, - baseStyles: string[], blockInfo: BlockInfo, currentTree: Parser.Tree, inlineParser: Parser, + currentUtf16Offset: number = 0, useDescendants: boolean = false ): StreamingChunk[] { const segments: StreamingChunk[] = [] @@ -31,6 +38,7 @@ function extractInlineStyleSegments( const relativeEnd = endByte - inlineNode.startIndex const styleNodes = inlineTree.rootNode.descendantsOfType(config.nodeType) + let offset = currentUtf16Offset // Check if any style node actually overlaps with our range for (const styleNode of styleNodes) { @@ -68,13 +76,8 @@ function extractInlineStyleSegments( if (intersectionStart < intersectionEnd) { const prefixText = inlineContent.substring(intersectionStart, intersectionEnd) if (prefixText) { - segments.push(createChunkFromBlockInfo( - prefixText, - baseStyles.filter(s => s !== config.styleName), - blockInfo, - false, - prefixText.includes('\n') - )) + segments.push(createChunkFromBlockInfo(prefixText, offset, blockInfo)) + offset += prefixText.length } } } @@ -86,19 +89,14 @@ function extractInlineStyleSegments( if (contentOverlapStart < contentOverlapEnd) { const styledText = inlineContent.substring(contentOverlapStart, contentOverlapEnd) if (styledText) { - // Ensure the style is present - const styledStyles = [...baseStyles] - if (styledStyles.indexOf(config.styleName) === -1) { - styledStyles.push(config.styleName) - } - - segments.push(createChunkFromBlockInfo( - styledText, - styledStyles, - blockInfo, - false, - styledText.includes('\n') - )) + // Create a contained span for this style + const spanType = config.styleName as 'bold' | 'italic' | 'code' | 'strikethrough' + const containedSpan: ClosedSpan = createClosedSpan(spanType, offset, styledText.length) + + segments.push(createChunkFromBlockInfo(styledText, offset, blockInfo, { + contained: [containedSpan] + })) + offset += styledText.length } } @@ -110,13 +108,8 @@ function extractInlineStyleSegments( if (suffixStart < suffixEnd) { const suffixText = inlineContent.substring(suffixStart, suffixEnd) if (suffixText) { - segments.push(createChunkFromBlockInfo( - suffixText, - baseStyles.filter(s => s !== config.styleName), - blockInfo, - false, - suffixText.includes('\n') - )) + segments.push(createChunkFromBlockInfo(suffixText, offset, blockInfo)) + offset += suffixText.length } } } @@ -130,15 +123,16 @@ function extractInlineStyleSegments( } // Extract inline code segments, stripping backtick delimiters. +// DEPRECATED: Use processInlineSpans() in segment-generator.ts instead. export function getInlineCodeSegments( content: string, node: Parser.SyntaxNode, startByte: number, endByte: number, - baseStyles: string[], blockInfo: BlockInfo, currentTree: Parser.Tree, - inlineParser: Parser + inlineParser: Parser, + currentUtf16Offset: number = 0 ): StreamingChunk[] { const config: InlineStyleConfig = { styleName: 'code', @@ -148,21 +142,22 @@ export function getInlineCodeSegments( } return extractInlineStyleSegments( - config, startByte, endByte, baseStyles, blockInfo, - currentTree, inlineParser, false + config, startByte, endByte, blockInfo, + currentTree, inlineParser, currentUtf16Offset, false ) } // Extract bold segments, stripping ** delimiters. +// DEPRECATED: Use processInlineSpans() in segment-generator.ts instead. export function getBoldSegments( content: string, node: Parser.SyntaxNode, startByte: number, endByte: number, - baseStyles: string[], blockInfo: BlockInfo, currentTree: Parser.Tree, - inlineParser: Parser + inlineParser: Parser, + currentUtf16Offset: number = 0 ): StreamingChunk[] { const config: InlineStyleConfig = { styleName: 'bold', @@ -172,21 +167,22 @@ export function getBoldSegments( } return extractInlineStyleSegments( - config, startByte, endByte, baseStyles, blockInfo, - currentTree, inlineParser, false + config, startByte, endByte, blockInfo, + currentTree, inlineParser, currentUtf16Offset, false ) } // Extract italic segments, stripping * or _ delimiters. +// DEPRECATED: Use processInlineSpans() in segment-generator.ts instead. export function getItalicSegments( content: string, node: Parser.SyntaxNode, startByte: number, endByte: number, - baseStyles: string[], blockInfo: BlockInfo, currentTree: Parser.Tree, - inlineParser: Parser + inlineParser: Parser, + currentUtf16Offset: number = 0 ): StreamingChunk[] { const config: InlineStyleConfig = { styleName: 'italic', @@ -196,21 +192,22 @@ export function getItalicSegments( } return extractInlineStyleSegments( - config, startByte, endByte, baseStyles, blockInfo, - currentTree, inlineParser, false + config, startByte, endByte, blockInfo, + currentTree, inlineParser, currentUtf16Offset, false ) } // Extract strikethrough segments, stripping ~~ delimiters. +// DEPRECATED: Use processInlineSpans() in segment-generator.ts instead. export function getStrikethroughSegments( content: string, node: Parser.SyntaxNode, startByte: number, endByte: number, - baseStyles: string[], blockInfo: BlockInfo, currentTree: Parser.Tree, - inlineParser: Parser + inlineParser: Parser, + currentUtf16Offset: number = 0 ): StreamingChunk[] { const config: InlineStyleConfig = { styleName: 'strikethrough', @@ -221,7 +218,7 @@ export function getStrikethroughSegments( // Strikethrough uses descendants for delimiters (they can be nested) return extractInlineStyleSegments( - config, startByte, endByte, baseStyles, blockInfo, - currentTree, inlineParser, true + config, startByte, endByte, blockInfo, + currentTree, inlineParser, currentUtf16Offset, true ) } diff --git a/src/tree-sitter/segment-builder.ts b/src/tree-sitter/segment-builder.ts index 34e4fbe..cbf4ae1 100644 --- a/src/tree-sitter/segment-builder.ts +++ b/src/tree-sitter/segment-builder.ts @@ -1,105 +1,237 @@ -import type { StreamingChunk, StreamingSegment, BlockInfo } from './types.js' - -// Create a StreamingSegment with consistent defaults. -export function createSegment( - segment: string, - styles: string[], - type: string, - isBlockDefining: boolean, - isProcessingNewLine: boolean, - options?: { - level?: number - language?: string - blockId?: number +import type { + StreamingChunk, + Chunk, + BlockInfo, + BlockContext, + BlockType, + OpenSpan, + ClosedSpan, + ParserConfig +} from './types.js' + +// ============================================================================ +// UTF-16 OFFSET UTILITIES +// ============================================================================ + +// Convert byte offset to UTF-16 code unit offset. +// Tree-sitter gives us byte positions, but JavaScript strings use UTF-16. +export function byteOffsetToUtf16(text: string, byteOffset: number): number { + const encoder = new TextEncoder() + let utf16Offset = 0 + let currentByteOffset = 0 + + for (const char of text) { + if (currentByteOffset >= byteOffset) break + const charBytes = encoder.encode(char).length + currentByteOffset += charBytes + // Each JS string char is 1 UTF-16 code unit, except surrogates (2) + utf16Offset += char.length // .length gives UTF-16 code units + } + + return utf16Offset +} + +// Convert UTF-16 offset to byte offset. +// Needed when we have UTF-16 positions and need tree-sitter byte positions. +export function utf16ToByteOffset(text: string, utf16Offset: number): number { + const encoder = new TextEncoder() + let currentUtf16 = 0 + let byteOffset = 0 + + for (const char of text) { + if (currentUtf16 >= utf16Offset) break + const charBytes = encoder.encode(char).length + byteOffset += charBytes + currentUtf16 += char.length } -): StreamingSegment { - const result: StreamingSegment = { - segment, - styles, - type, - isBlockDefining, - isProcessingNewLine, + + return byteOffset +} + +// ============================================================================ +// BLOCK CONTEXT HELPERS +// ============================================================================ + +// Map internal block type strings to BlockType enum. +function mapBlockType(type: string): BlockType { + switch (type) { + case 'header': + case 'atx_heading': + return 'heading' + case 'codeBlock': + case 'fenced_code_block': + return 'code_block' + case 'list_item': + return 'list_item' + case 'pipe_table': + case 'table': + return 'table' + case 'pipe_table_row': + return 'table_row' + case 'pipe_table_cell': + return 'table_cell' + case 'blockquote': + return 'blockquote' + case 'paragraph': + default: + return 'paragraph' } +} - if (options?.level !== undefined) { - result.level = options.level +// Create a BlockContext from BlockInfo. +export function createBlockContext(blockInfo: BlockInfo): BlockContext { + const context: BlockContext = { + type: mapBlockType(blockInfo.type) } - if (options?.language !== undefined) { - result.language = options.language + + if (blockInfo.level !== undefined) { + context.level = blockInfo.level } - if (options?.blockId !== undefined) { - result.blockId = options.blockId + if (blockInfo.language !== undefined) { + context.language = blockInfo.language } - return result + return context } -// Create a StreamingChunk with STREAMING status. -export function createStreamingChunk(segment: StreamingSegment): StreamingChunk { +// ============================================================================ +// CHUNK BUILDERS +// ============================================================================ + +// Create a new Chunk with the new API format. +export function createChunk( + text: string, + offset: number, + block: BlockContext, + options?: { + opening?: OpenSpan[] + closing?: ClosedSpan[] + contained?: ClosedSpan[] + backtrackOffset?: number + original?: string + } +): Chunk { return { - status: 'STREAMING', - segment, + text, + offset, + length: text.length, + block, + opening: options?.opening ?? [], + closing: options?.closing ?? [], + contained: options?.contained ?? [], + backtrackOffset: options?.backtrackOffset, + original: options?.original, } } -// Create a segment from block info with common patterns. -export function createSegmentFromBlockInfo( - text: string, - styles: string[], - blockInfo: BlockInfo, - isBlockDefining: boolean, - isProcessingNewLine: boolean -): StreamingSegment { - return createSegment( - text, - styles, - blockInfo.type, - isBlockDefining, - isProcessingNewLine, - { - level: blockInfo.level, - language: blockInfo.language, - blockId: blockInfo.id, - } - ) +// Create a StreamingChunk wrapper for a Chunk. +export function createStreamingChunkWrapper(chunk: Chunk): StreamingChunk { + return { + status: 'STREAMING', + chunk, + } } // Create a streaming chunk from block info. export function createChunkFromBlockInfo( text: string, - styles: string[], + offset: number, blockInfo: BlockInfo, - isBlockDefining: boolean, - isProcessingNewLine: boolean + options?: { + opening?: OpenSpan[] + closing?: ClosedSpan[] + contained?: ClosedSpan[] + backtrackOffset?: number + original?: string + } ): StreamingChunk { - return createStreamingChunk( - createSegmentFromBlockInfo(text, styles, blockInfo, isBlockDefining, isProcessingNewLine) - ) + const chunk = createChunk(text, offset, createBlockContext(blockInfo), options) + return createStreamingChunkWrapper(chunk) } -// Create a plain text paragraph segment. -export function createPlainTextChunk(text: string, isBlockDefining: boolean = false): StreamingChunk { - return createStreamingChunk({ - segment: text, - styles: [], - type: 'paragraph', - isBlockDefining, - isProcessingNewLine: text.includes('\n'), - }) +// Create a plain text paragraph chunk. +export function createPlainTextChunk( + text: string, + offset: number, + options?: { + opening?: OpenSpan[] + closing?: ClosedSpan[] + contained?: ClosedSpan[] + original?: string + } +): StreamingChunk { + return createStreamingChunkWrapper( + createChunk(text, offset, { type: 'paragraph' }, options) + ) } -// Create a code block segment. +// Create a code block chunk. export function createCodeBlockChunk( text: string, + offset: number, language: string = '', - isBlockDefining: boolean = false + options?: { + opening?: OpenSpan[] + closing?: ClosedSpan[] + contained?: ClosedSpan[] + original?: string + } ): StreamingChunk { - return createStreamingChunk({ - segment: text, - styles: [], - type: 'codeBlock', - isBlockDefining, - isProcessingNewLine: text.includes('\n'), - language, - }) + return createStreamingChunkWrapper( + createChunk(text, offset, { type: 'code_block', language }, options) + ) +} + +// Create a heading chunk. +export function createHeadingChunk( + text: string, + offset: number, + level: number, + options?: { + opening?: OpenSpan[] + closing?: ClosedSpan[] + contained?: ClosedSpan[] + original?: string + } +): StreamingChunk { + return createStreamingChunkWrapper( + createChunk(text, offset, { type: 'heading', level }, options) + ) +} + +// ============================================================================ +// SPAN BUILDERS +// ============================================================================ + +// Create an OpenSpan. +export function createOpenSpan(type: OpenSpan['type'], openOffset: number): OpenSpan { + return { type, openOffset } +} + +// Create a ClosedSpan for basic styles (bold, italic, code, strikethrough). +export function createClosedSpan( + type: 'bold' | 'italic' | 'code' | 'strikethrough', + offset: number, + length: number +): ClosedSpan { + return { type, offset, length } +} + +// Create a ClosedSpan for a link. +export function createLinkSpan( + offset: number, + length: number, + url: string +): ClosedSpan { + return { type: 'link', offset, length, url } +} + +// Create a ClosedSpan for an image. +export function createImageSpan( + offset: number, + length: number, + src: string, + alt?: string +): ClosedSpan { + return { type: 'image', offset, length, src, alt } } diff --git a/src/tree-sitter/segment-generator.ts b/src/tree-sitter/segment-generator.ts index bbbdeab..4bc4e46 100644 --- a/src/tree-sitter/segment-generator.ts +++ b/src/tree-sitter/segment-generator.ts @@ -1,24 +1,42 @@ import type { Parser } from 'web-tree-sitter' -import type { StreamingChunk, BlockState, BlockInfo, HEADER_MARKER_LEVELS, SUPPRESSED_SYNTAX_TYPES } from './types.js' +import type { + StreamingChunk, + BlockInfo, + SegmentGeneratorState, + OpenSpan, + ClosedSpan, + SpanType, + ParserConfig +} from './types.js' import { findActiveNodeAtPosition, findInlineNodeAtPosition, findBlockNode } from './tree-navigation.js' import { getBlockInfo, isNewBlock } from './block-detection.js' import { - detectActiveStyles, hasCompleteCodeSpanAt, hasCompleteBoldAt, hasCompleteItalicAt, hasCompleteStrikethroughAt, + hasCompleteLinkAt, + hasCompleteImageAt, + hasIncompleteLinkOpening, + hasIncompleteImageOpening, hasUnmatchedItalicMarker, - isInsideCodeBlock + isInsideCodeBlock, + detectActiveStyles } from './inline-detection.js' -import { getHeaderContent, getCodeBlockContent } from './content-extraction.js' +import { getHeaderContent, getCodeBlockContent, getInlineContent } from './content-extraction.js' import { - getInlineCodeSegments, - getBoldSegments, - getItalicSegments, - getStrikethroughSegments -} from './inline-extractors.js' -import { createStreamingChunk, createChunkFromBlockInfo, createPlainTextChunk, createCodeBlockChunk } from './segment-builder.js' + createChunkFromBlockInfo, + createPlainTextChunk, + createCodeBlockChunk, + createHeadingChunk, + byteOffsetToUtf16, + + createOpenSpan, + createClosedSpan, + createLinkSpan, + createImageSpan, + createBlockContext +} from './segment-builder.js' // Re-import the constant that we need locally const HEADER_MARKER_LEVELS_LOCAL: Record = { @@ -36,27 +54,192 @@ const SUPPRESSED_SYNTAX_TYPES_LOCAL = [ '|' // Table pipe delimiters ] -export type SegmentGeneratorState = { - pendingInlineContent: string - pendingInlineStartIndex: number - currentBlock: BlockState | null -} - export type SegmentGeneratorContext = { content: string currentTree: Parser.Tree inlineParser: Parser | null state: SegmentGeneratorState + config?: ParserConfig +} + +// Create initial segment generator state +export function createInitialState(): SegmentGeneratorState { + return { + totalUtf16Offset: 0, + lastEmittedOffset: 0, + openSpans: [], + currentBlock: null, + pendingInlineContent: '', + accumulatedContent: '' + } +} + +// Detect span type from tree-sitter node type +function detectSpanType(nodeType: string): SpanType | null { + switch (nodeType) { + case 'strong_emphasis': return 'bold' + case 'emphasis': return 'italic' + case 'code_span': return 'code' + case 'strikethrough': return 'strikethrough' + case 'inline_link': return 'link' + case 'image': return 'image' + default: return null + } +} + +// Extract span metadata (URL for links, src/alt for images) +function extractSpanMetadata(node: Parser.SyntaxNode): { url?: string; src?: string; alt?: string } { + if (node.type === 'inline_link') { + const destNode = node.descendantsOfType('link_destination')[0] + return { url: destNode?.text ?? '' } + } + if (node.type === 'image') { + const destNode = node.descendantsOfType('link_destination')[0] + const descNode = node.descendantsOfType('image_description')[0] + return { + src: destNode?.text ?? '', + alt: descNode?.text + } + } + return {} +} + +// Check if span is fully contained within chunk boundaries +function isSpanContained(spanStart: number, spanEnd: number, chunkStart: number, chunkEnd: number): boolean { + return spanStart >= chunkStart && spanEnd <= chunkEnd +} + +// Check if span opens in this chunk but closes later +function isSpanOpening(spanStart: number, spanEnd: number, chunkStart: number, chunkEnd: number): boolean { + return spanStart >= chunkStart && spanStart < chunkEnd && spanEnd > chunkEnd +} + +// Check if span opened earlier and closes in this chunk +function isSpanClosing(spanStart: number, spanEnd: number, chunkStart: number, chunkEnd: number): boolean { + return spanStart < chunkStart && spanEnd >= chunkStart && spanEnd <= chunkEnd +} + +// Create a closed span from node metadata +function createClosedSpanFromNode( + spanType: SpanType, + node: Parser.SyntaxNode, + offset: number, + length: number +): ClosedSpan | null { + const metadata = extractSpanMetadata(node) + + if (spanType === 'link' && metadata.url !== undefined) { + return createLinkSpan(offset, length, metadata.url) + } + if (spanType === 'image' && metadata.src !== undefined) { + return createImageSpan(offset, length, metadata.src, metadata.alt) + } + if (spanType === 'bold' || spanType === 'italic' || spanType === 'code' || spanType === 'strikethrough') { + return createClosedSpan(spanType, offset, length) + } + return null +} + +// Process a single style node and categorize it +function categorizeSpanNode( + node: Parser.SyntaxNode, + content: string, + chunkStartUtf16: number, + chunkEndUtf16: number, + openSpans: OpenSpan[] +): { + contained?: ClosedSpan + opening?: OpenSpan + closing?: ClosedSpan + closedOpenIndex?: number +} { + const spanType = detectSpanType(node.type) + if (!spanType) return {} + + const spanStartUtf16 = byteOffsetToUtf16(content, node.startIndex) + const spanEndUtf16 = byteOffsetToUtf16(content, node.endIndex) + const spanLength = spanEndUtf16 - spanStartUtf16 + + // Fully contained + if (isSpanContained(spanStartUtf16, spanEndUtf16, chunkStartUtf16, chunkEndUtf16)) { + const span = createClosedSpanFromNode(spanType, node, spanStartUtf16, spanLength) + return span ? { contained: span } : {} + } + + // Opens here, closes later + if (isSpanOpening(spanStartUtf16, spanEndUtf16, chunkStartUtf16, chunkEndUtf16)) { + return { opening: createOpenSpan(spanType, spanStartUtf16) } + } + + // Opened earlier, closes here + if (isSpanClosing(spanStartUtf16, spanEndUtf16, chunkStartUtf16, chunkEndUtf16)) { + const matchingIdx = openSpans.findIndex(s => s.type === spanType) + if (matchingIdx !== -1) { + const matchingOpen = openSpans[matchingIdx] + const totalLength = spanEndUtf16 - matchingOpen.openOffset + const span = createClosedSpanFromNode(spanType, node, matchingOpen.openOffset, totalLength) + return span ? { closing: span, closedOpenIndex: matchingIdx } : {} + } + } + + return {} +} + +// Process inline styles and categorize them as opening/closing/contained +function processInlineSpans( + inlineTree: Parser.Tree, + chunkStartUtf16: number, + chunkEndUtf16: number, + content: string, + state: SegmentGeneratorState +): { opening: OpenSpan[]; closing: ClosedSpan[]; contained: ClosedSpan[]; newOpenSpans: OpenSpan[] } { + const opening: OpenSpan[] = [] + const closing: ClosedSpan[] = [] + const contained: ClosedSpan[] = [] + const newOpenSpans = [...state.openSpans] + const indicesToRemove: number[] = [] + + const styleNodeTypes = ['code_span', 'strong_emphasis', 'emphasis', 'strikethrough', 'inline_link', 'image'] + + for (const nodeType of styleNodeTypes) { + const nodes = inlineTree.rootNode.descendantsOfType(nodeType) + + for (const node of nodes) { + const result = categorizeSpanNode(node, content, chunkStartUtf16, chunkEndUtf16, newOpenSpans) + + if (result.contained) { + contained.push(result.contained) + } + if (result.opening) { + opening.push(result.opening) + newOpenSpans.push(result.opening) + } + if (result.closing) { + closing.push(result.closing) + if (result.closedOpenIndex !== undefined) { + indicesToRemove.push(result.closedOpenIndex) + } + } + } + } + + // Remove closed spans from open list (in reverse order to preserve indices) + indicesToRemove.sort((a, b) => b - a) + for (const idx of indicesToRemove) { + newOpenSpans.splice(idx, 1) + } + + return { opening, closing, contained, newOpenSpans } } -// Generate segments for a range of content. -// This is the main segment generation function that handles all the complex logic. +// Generate chunks for a range of content using the new orthogonal chunks/spans model. +// Chunks represent text segments; spans represent inline styles that may cross chunk boundaries. export function generateSegments( fromIndex: number, toIndex: number, context: SegmentGeneratorContext ): { segments: StreamingChunk[]; state: SegmentGeneratorState } { - const { content, currentTree, inlineParser } = context + const { content, currentTree, inlineParser, config } = context let state = { ...context.state } if (!currentTree) { @@ -72,10 +255,15 @@ export function generateSegments( if (state.pendingInlineContent) { // Prepend pending content newContent = state.pendingInlineContent + newContent - actualFromIndex = state.pendingInlineStartIndex - state.pendingInlineContent = '' + actualFromIndex = state.pendingInlineStartIndex ?? fromIndex + state = { ...state, pendingInlineContent: '' } } + // Calculate UTF-16 offsets for this chunk + const chunkStartUtf16 = state.totalUtf16Offset + const chunkTextUtf16Length = newContent.length + const chunkEndUtf16 = chunkStartUtf16 + chunkTextUtf16Length + // Check if current content has unmatched inline delimiters const inlineNode = findInlineNodeAtPosition(currentTree.rootNode, actualFromIndex) if (inlineNode && inlineParser) { @@ -130,6 +318,26 @@ export function generateSegments( return { segments, state } } } + + // Check for incomplete link opening [ + if (newPortion.includes('[')) { + const hasCompleteLink = hasCompleteLinkAt(inlineTree.rootNode, newPortionStart, newPortionEnd) + if (!hasCompleteLink && hasIncompleteLinkOpening(newPortion, inlineParser)) { + state.pendingInlineContent = newContent + state.pendingInlineStartIndex = actualFromIndex + return { segments, state } + } + } + + // Check for incomplete image opening ![ + if (newPortion.includes('![')) { + const hasCompleteImage = hasCompleteImageAt(inlineTree.rootNode, newPortionStart, newPortionEnd) + if (!hasCompleteImage && hasIncompleteImageOpening(newPortion, inlineParser)) { + state.pendingInlineContent = newContent + state.pendingInlineStartIndex = actualFromIndex + return { segments, state } + } + } } // Skip empty content @@ -141,15 +349,23 @@ export function generateSegments( const nodeAtPosition = findActiveNodeAtPosition(currentTree.rootNode, actualFromIndex) if (!nodeAtPosition) { - // If no node found, treat as plain text - return { - segments: [createPlainTextChunk(newContent)], - state + // If no node found, treat as plain text with current offset + const chunk = createPlainTextChunk(newContent, chunkStartUtf16, { + original: config?.includeRawStreamedToken ? newContent : undefined + }) + state = { + ...state, + totalUtf16Offset: chunkEndUtf16, + lastEmittedOffset: chunkEndUtf16, + accumulatedContent: state.accumulatedContent + newContent } + return { segments: [chunk], state } } // Check if the node is a suppressed syntax type if (SUPPRESSED_SYNTAX_TYPES_LOCAL.indexOf(nodeAtPosition.type) !== -1) { + // Update offset but don't emit + state = { ...state, totalUtf16Offset: chunkEndUtf16 } return { segments, state } } @@ -158,6 +374,7 @@ export function generateSegments( while (currentForDelimiter) { if (currentForDelimiter.type === 'pipe_table_delimiter_row' || currentForDelimiter.type === 'pipe_table_delimiter_cell') { + state = { ...state, totalUtf16Offset: chunkEndUtf16 } return { segments, state } } currentForDelimiter = currentForDelimiter.parent @@ -166,12 +383,6 @@ export function generateSegments( // Determine the block type and properties const blockInfo = getBlockInfo(nodeAtPosition) - // Check if we're starting a new block - const isNewBlockFlag = isNewBlock(blockInfo, nodeAtPosition, state.currentBlock) - - // Detect styles in the current context - const styles = detectActiveStyles(nodeAtPosition, actualFromIndex, actualToIndex, currentTree, inlineParser) - // Process content based on block type let processedContent = newContent @@ -186,7 +397,7 @@ export function generateSegments( // Don't emit if it's only markers if (processedContent.length === 0 || processedContent.trim().length === 0) { - state = updateBlockState(state, isNewBlockFlag, blockInfo, nodeAtPosition, actualToIndex, styles, false) + state = { ...state, totalUtf16Offset: chunkEndUtf16 } return { segments, state } } } else if (blockInfo.type === 'codeBlock') { @@ -199,7 +410,7 @@ export function generateSegments( } if (processedContent.length === 0) { - state = updateBlockState(state, isNewBlockFlag, blockInfo, nodeAtPosition, actualToIndex, styles, false) + state = { ...state, totalUtf16Offset: chunkEndUtf16 } return { segments, state } } } else if (blockInfo.type === 'paragraph') { @@ -209,75 +420,88 @@ export function generateSegments( } // Handle code fence detection in paragraph content - const result = handleCodeFenceInParagraph(newContent, actualFromIndex, actualToIndex, content, segments) + const result = handleCodeFenceInParagraph( + newContent, actualFromIndex, actualToIndex, content, + segments, chunkStartUtf16, config + ) if (result.handled) { + state = { + ...state, + totalUtf16Offset: state.totalUtf16Offset + result.utf16Consumed, + lastEmittedOffset: state.totalUtf16Offset + result.utf16Consumed, + accumulatedContent: state.accumulatedContent + newContent + } return { segments: result.segments, state } } } - // Process inline styles for non-codeBlock types - if (blockInfo.type !== 'codeBlock') { - const inlineResult = processInlineStyles( - processedContent, nodeAtPosition, actualFromIndex, actualToIndex, - styles, blockInfo, isNewBlockFlag, state, currentTree, inlineParser! + // Process inline spans for non-codeBlock types + let opening: OpenSpan[] = [] + let closing: ClosedSpan[] = [] + let contained: ClosedSpan[] = [] + let strippedContent = processedContent + + if (blockInfo.type !== 'codeBlock' && inlineParser) { + const fullContent = state.accumulatedContent + processedContent + const inlineTree = inlineParser.parse(fullContent) + + // The chunk boundaries in the ACCUMULATED content space + const accumulatedOffset = state.accumulatedContent.length + const chunkStartInAccumulated = accumulatedOffset + const chunkEndInAccumulated = accumulatedOffset + processedContent.length + + const spanResult = processInlineSpans( + inlineTree, + chunkStartInAccumulated, // Use position in accumulated content + chunkEndInAccumulated, + fullContent, + state + ) + opening = spanResult.opening + closing = spanResult.closing + contained = spanResult.contained + state = { ...state, openSpans: spanResult.newOpenSpans } + + // Strip inline markers from the content + strippedContent = getInlineContent( + processedContent, + inlineParser.parse(processedContent), // Parse just the new content + 0, + processedContent.length ) - - if (inlineResult.handled) { - return { segments: inlineResult.segments, state: inlineResult.state } - } - } - - // Determine effective block defining status - let effectiveIsBlockDefining = isNewBlockFlag - if (!isNewBlockFlag && state.currentBlock && !state.currentBlock.hasEmittedContent && state.currentBlock.type === blockInfo.type) { - effectiveIsBlockDefining = true } - // Create and push the segment - segments.push(createChunkFromBlockInfo( - processedContent, - styles, + // Create the chunk with the new API + const chunk = createChunkFromBlockInfo( + strippedContent, + chunkStartUtf16, blockInfo, - effectiveIsBlockDefining, - newContent.includes('\n') - )) - - // Update block tracking - state = updateBlockState(state, isNewBlockFlag, blockInfo, nodeAtPosition, actualToIndex, styles, true) - - return { segments, state } -} - -// Update the block state after processing content -function updateBlockState( - state: SegmentGeneratorState, - isNewBlockFlag: boolean, - blockInfo: BlockInfo, - nodeAtPosition: Parser.SyntaxNode, - actualToIndex: number, - styles: string[], - hasEmittedContent: boolean -): SegmentGeneratorState { - const newState = { ...state } - - if (isNewBlockFlag) { - newState.currentBlock = { + { + opening, + closing, + contained, + original: config?.includeRawStreamedToken ? newContent : undefined + } + ) + segments.push(chunk) + + // Update state + state = { + ...state, + totalUtf16Offset: chunkStartUtf16 + processedContent.length, + lastEmittedOffset: chunkStartUtf16 + processedContent.length, + accumulatedContent: state.accumulatedContent + newContent, + currentBlock: { type: blockInfo.type, level: blockInfo.level, language: blockInfo.language, startIndex: nodeAtPosition.startIndex, lastSegmentEnd: actualToIndex, - styles: new Set(styles), - hasEmittedContent + hasEmittedContent: true } - } else if (newState.currentBlock) { - newState.currentBlock = { ...newState.currentBlock } - newState.currentBlock.lastSegmentEnd = actualToIndex - newState.currentBlock.hasEmittedContent = newState.currentBlock.hasEmittedContent || hasEmittedContent - styles.forEach(s => newState.currentBlock!.styles.add(s)) } - return newState + return { segments, state } } // Handle code fence detection when tree-sitter sees it as paragraph @@ -286,17 +510,20 @@ function handleCodeFenceInParagraph( actualFromIndex: number, actualToIndex: number, content: string, - existingSegments: StreamingChunk[] -): { handled: boolean; segments: StreamingChunk[] } { + existingSegments: StreamingChunk[], + currentUtf16Offset: number, + config?: ParserConfig +): { handled: boolean; segments: StreamingChunk[]; utf16Consumed: number } { const segments = [...existingSegments] + let utf16Consumed = 0 - // Check if new content contains a code fence opening - const codeFenceOpeningMatch = newContent.match(/```([a-zA-Z0-9]*)\n?/) - if (!codeFenceOpeningMatch) { - // Also check for content MIDDLE of a code block + // Find code fence opening (```) + const fenceStart = newContent.indexOf('```') + if (fenceStart === -1) { + // No fence in new content - check if we're inside an existing code block const contentBeforeThis = content.substring(0, actualFromIndex) - const allFences = contentBeforeThis.match(/```/g) || [] - const isInsideCodeBlockContext = allFences.length % 2 === 1 + const fenceCount = countOccurrences(contentBeforeThis, '```') + const isInsideCodeBlockContext = fenceCount % 2 === 1 if (isInsideCodeBlockContext) { const closingFenceIdx = newContent.indexOf('```') @@ -305,141 +532,121 @@ function handleCodeFenceInParagraph( // No closing fence - emit as code block content return { handled: true, - segments: [createCodeBlockChunk(newContent)] + segments: [createCodeBlockChunk(newContent, currentUtf16Offset, '', { + original: config?.includeRawStreamedToken ? newContent : undefined + })], + utf16Consumed: newContent.length } } else { // Has closing fence const codeContent = newContent.substring(0, closingFenceIdx) const afterFence = newContent.substring(closingFenceIdx + 3) + let offset = currentUtf16Offset if (codeContent.length > 0) { - segments.push(createCodeBlockChunk(codeContent)) + segments.push(createCodeBlockChunk(codeContent, offset, '', { + original: config?.includeRawStreamedToken ? codeContent : undefined + })) + offset += codeContent.length } - const textAfterFence = afterFence.replace(/^\n/, '') + // Skip the fence markers + offset += 3 + + const textAfterFence = stripLeadingNewline(afterFence) if (textAfterFence.length > 0) { - segments.push(createPlainTextChunk(textAfterFence, true)) + segments.push(createPlainTextChunk(textAfterFence, offset, { + original: config?.includeRawStreamedToken ? textAfterFence : undefined + })) } - return { handled: true, segments } + return { handled: true, segments, utf16Consumed: newContent.length } } } - return { handled: false, segments } + return { handled: false, segments, utf16Consumed: 0 } } - const fenceStart = newContent.indexOf(codeFenceOpeningMatch[0]) - const fenceLanguage = codeFenceOpeningMatch[1] || '' - const fenceMarker = codeFenceOpeningMatch[0] + // Extract language from fence line (```language) + const afterFenceMarker = newContent.substring(fenceStart + 3) + const newlineIdx = afterFenceMarker.indexOf('\n') + const fenceLanguage = newlineIdx === -1 + ? afterFenceMarker.trim() + : afterFenceMarker.substring(0, newlineIdx).trim() + const fenceMarkerLength = 3 + (newlineIdx === -1 ? afterFenceMarker.length : newlineIdx + 1) // Check for closing fence - const positionOfFence = actualFromIndex + fenceStart - const contentFromFence = content.substring(positionOfFence) - const closingFenceIdx = contentFromFence.substring(fenceMarker.length).indexOf('```') + const contentAfterOpening = newlineIdx === -1 + ? '' + : afterFenceMarker.substring(newlineIdx + 1) + const closingFenceIdx = contentAfterOpening.indexOf('```') // Content BEFORE the fence const contentBeforeFence = newContent.substring(0, fenceStart) + let offset = currentUtf16Offset if (closingFenceIdx === -1) { // No closing fence yet - emit content before fence and buffer the rest if (contentBeforeFence.trim().length > 0) { - segments.push(createPlainTextChunk(contentBeforeFence)) + segments.push(createPlainTextChunk(contentBeforeFence, offset, { + original: config?.includeRawStreamedToken ? contentBeforeFence : undefined + })) + utf16Consumed += contentBeforeFence.length } - // This would need state management - return partial result - // The caller should handle buffering - return { handled: true, segments } + return { handled: true, segments, utf16Consumed } } // Complete code block structure if (contentBeforeFence.trim().length > 0) { - segments.push(createPlainTextChunk(contentBeforeFence)) + segments.push(createPlainTextChunk(contentBeforeFence, offset, { + original: config?.includeRawStreamedToken ? contentBeforeFence : undefined + })) + offset += contentBeforeFence.length } - const contentAfterOpeningFence = newContent.substring(fenceStart + fenceMarker.length) - const closingFenceInContent = contentAfterOpeningFence.indexOf('```') - - let codeContent: string - if (closingFenceInContent === -1) { - codeContent = contentAfterOpeningFence - } else { - codeContent = contentAfterOpeningFence.substring(0, closingFenceInContent) - } + // Skip fence marker + offset += fenceMarkerLength - codeContent = codeContent.replace(/^\n/, '') + const codeContent = contentAfterOpening.substring(0, closingFenceIdx) if (codeContent.length > 0) { - segments.push(createCodeBlockChunk(codeContent, fenceLanguage, true)) + segments.push(createCodeBlockChunk(codeContent, offset, fenceLanguage, { + original: config?.includeRawStreamedToken ? codeContent : undefined + })) + offset += codeContent.length } - if (closingFenceInContent !== -1) { - const afterClosingFence = contentAfterOpeningFence.substring(closingFenceInContent + 3) - const textAfterFence = afterClosingFence.replace(/^\n/, '') - if (textAfterFence.trim().length > 0) { - segments.push(createPlainTextChunk(textAfterFence, true)) - } + // Skip closing fence + offset += 3 + + const afterClosingFence = contentAfterOpening.substring(closingFenceIdx + 3) + const textAfterFence = stripLeadingNewline(afterClosingFence) + if (textAfterFence.trim().length > 0) { + segments.push(createPlainTextChunk(textAfterFence, offset, { + original: config?.includeRawStreamedToken ? textAfterFence : undefined + })) } - return { handled: true, segments } + return { handled: true, segments, utf16Consumed: newContent.length } } -// Process inline styles and return segments if applicable -function processInlineStyles( - processedContent: string, - nodeAtPosition: Parser.SyntaxNode, - actualFromIndex: number, - actualToIndex: number, - styles: string[], - blockInfo: BlockInfo, - isNewBlockFlag: boolean, - state: SegmentGeneratorState, - currentTree: Parser.Tree, - inlineParser: Parser -): { handled: boolean; segments: StreamingChunk[]; state: SegmentGeneratorState } { - const styleHandlers: Array<{ - style: string - extractor: (content: string, node: Parser.SyntaxNode, startByte: number, endByte: number, baseStyles: string[], blockInfo: BlockInfo, currentTree: Parser.Tree, inlineParser: Parser) => StreamingChunk[] - }> = [ - { style: 'code', extractor: getInlineCodeSegments }, - { style: 'bold', extractor: getBoldSegments }, - { style: 'italic', extractor: getItalicSegments }, - { style: 'strikethrough', extractor: getStrikethroughSegments }, - ] - - for (const { style, extractor } of styleHandlers) { - if (styles.indexOf(style) !== -1) { - let splitSegments: StreamingChunk[] = [] - try { - splitSegments = extractor( - processedContent, nodeAtPosition, actualFromIndex, actualToIndex, - styles, blockInfo, currentTree, inlineParser - ) - } catch (e) { - console.warn(`[PARSER] Failed to extract ${style} segments, will use default processing:`, e) - continue - } - - if (splitSegments.length > 0) { - // Determine effective block defining - let effectiveIsBlockDefining = isNewBlockFlag - if (!isNewBlockFlag && state.currentBlock && !state.currentBlock.hasEmittedContent && state.currentBlock.type === blockInfo.type) { - effectiveIsBlockDefining = true - } - - // Apply block defining flag to first segment - splitSegments.forEach((seg, index) => { - if (index === 0 && seg.segment) { - seg.segment.isBlockDefining = effectiveIsBlockDefining - } - }) - - // Update state - const newState = updateBlockState(state, isNewBlockFlag, blockInfo, nodeAtPosition, actualToIndex, styles, true) - - return { handled: true, segments: splitSegments, state: newState } - } - } +// Count occurrences of a substring +function countOccurrences(str: string, substr: string): number { + let count = 0 + let pos = 0 + while ((pos = str.indexOf(substr, pos)) !== -1) { + count++ + pos += substr.length } + return count +} - return { handled: false, segments: [], state } +// Strip leading newline if present +function stripLeadingNewline(str: string): string { + if (str.startsWith('\n')) { + return str.substring(1) + } + return str } + diff --git a/src/tree-sitter/types.ts b/src/tree-sitter/types.ts index 067660d..18cd858 100644 --- a/src/tree-sitter/types.ts +++ b/src/tree-sitter/types.ts @@ -1,5 +1,130 @@ import type { Parser } from 'web-tree-sitter' +// ============================================================================ +// SPAN TYPES - Typed spans with metadata +// ============================================================================ + +// Typed span with metadata. Plain strings don't scale — +// links need URLs, images need src/alt, etc. +export type Span = + | { type: 'bold' } + | { type: 'italic' } + | { type: 'code' } + | { type: 'strikethrough' } + | { type: 'link'; url: string } + | { type: 'image'; src: string; alt?: string } + +// Extract the type string from Span union +export type SpanType = Span['type'] + +// An open span is a span that started but hasn't closed yet. +// The full span info (like URL for links) is only available on close. +export type OpenSpan = { + type: SpanType + // UTF-16 offset from stream start where this span opened + openOffset: number +} + +// A closed span includes the full span data plus position info. +export type ClosedSpan = Span & { + // UTF-16 offset from stream start + offset: number + // Length in UTF-16 code units + length: number +} + +// ============================================================================ +// BLOCK TYPES +// ============================================================================ + +// Block types the parser recognizes. +export type BlockType = + | 'paragraph' + | 'heading' + | 'code_block' + | 'list_item' + | 'table' + | 'table_row' + | 'table_cell' + | 'blockquote' + +// Block context for a chunk +export type BlockContext = { + type: BlockType + // For headings: 1-6 + level?: number + // For code blocks: language identifier + language?: string +} + +// ============================================================================ +// CHUNK TYPE - Core output unit +// ============================================================================ + +// A chunk is a unit of parsed output. Chunks and spans are +// completely orthogonal — spans can start/end mid-chunk, +// multiple spans can exist in one chunk, etc. +export type Chunk = { + // Plain text with formatting removed + text: string + + // UTF-16 offset from stream start + offset: number + + // Length in UTF-16 code units + length: number + + // Block context this chunk belongs to + block: BlockContext + + // Spans that opened in this chunk (will close in a future chunk) + opening: OpenSpan[] + + // Spans that closed in this chunk (opened in a past chunk) + closing: ClosedSpan[] + + // Spans fully contained within this chunk + contained: ClosedSpan[] + + // If set, this chunk corrects previous output starting from this + // UTF-16 offset. Consumer should discard everything from this + // offset onwards and replace with this chunk + subsequent chunks. + backtrackOffset?: number + + // Original markdown source (only if includeRawStreamedToken config is true). + // Useful as fallback when parser messes up or for unsupported formats. + original?: string +} + +// Stream status wrapper for chunks. +export type StreamingChunk = + | { status: 'STREAMING'; chunk: Chunk } + | { status: 'START_STREAM' } + | { status: 'END_STREAM' } + +// ============================================================================ +// PARSER CONFIGURATION +// ============================================================================ + +// Parser configuration options. +export type ParserConfig = { + // Maximum characters the consumer can backtrack. + // Default: undefined (unlimited backtracking). + // + // When corrections exceed this window: + // - Best effort: fix what's within window + // - Fall back to plain text if structure broken beyond repair + windowSize?: number + + // Include original markdown source in chunk output. + // Default: false (saves payload size). + includeRawStreamedToken?: boolean +} + +// ============================================================================ +// INTERNAL TYPES - Used by parser internals +// ============================================================================ + // Lookup map for tree-sitter ATX header marker node types to their heading levels. // Used for both level extraction and marker-only content detection. export const HEADER_MARKER_LEVELS: Record = { @@ -24,24 +149,6 @@ export const SUPPRESSED_SYNTAX_TYPES = [ '|' // Table pipe delimiters ] as const -// Represents a parsed segment of streaming markdown content -export type StreamingSegment = { - level?: number - language?: string - segment: string - styles: string[] - type: string - isBlockDefining: boolean - isProcessingNewLine: boolean - blockId?: number -} - -// A chunk of streaming data with status information -export type StreamingChunk = { - status: string - segment?: StreamingSegment -} - // Tracks the current block's state during parsing export type BlockState = { type: string @@ -49,7 +156,6 @@ export type BlockState = { language?: string startIndex: number lastSegmentEnd: number - styles: Set hasEmittedContent?: boolean } @@ -67,7 +173,7 @@ export type InlineExtractionContext = { node: Parser.SyntaxNode startByte: number endByte: number - baseStyles: string[] + baseSpans: OpenSpan[] blockInfo: BlockInfo inlineParser: Parser currentTree: Parser.Tree @@ -75,7 +181,7 @@ export type InlineExtractionContext = { // Configuration for a specific inline style type export type InlineStyleConfig = { - styleName: string + styleName: SpanType nodeType: string delimiterType: string minDelimiters: number @@ -107,4 +213,44 @@ export const INLINE_STYLE_CONFIGS: Record = { delimiterType: 'emphasis_delimiter', minDelimiters: 4, // ~~ on each side = 4 delimiter nodes }, + link: { + styleName: 'link', + nodeType: 'inline_link', + delimiterType: '', + minDelimiters: 0, + }, + image: { + styleName: 'image', + nodeType: 'image', + delimiterType: '', + minDelimiters: 0, + }, +} + +// ============================================================================ +// SEGMENT GENERATOR STATE +// ============================================================================ + +// State maintained by the segment generator across chunks +export type SegmentGeneratorState = { + // Total UTF-16 code units emitted so far (from stream start) + totalUtf16Offset: number + + // Last emitted UTF-16 offset (for backtrack detection) + lastEmittedOffset: number + + // Currently open spans that haven't closed yet + openSpans: OpenSpan[] + + // Current block being processed + currentBlock: BlockState | null + + // Pending inline content waiting for delimiter closure + pendingInlineContent: string + + // Start index for pending inline content + pendingInlineStartIndex?: number + + // Accumulated content for backtrack reference + accumulatedContent: string } From 263c6714ea22e36657fa7654fabe5d699dc763ea Mon Sep 17 00:00:00 2001 From: Shelby Carter Date: Fri, 6 Feb 2026 22:14:55 -0500 Subject: [PATCH 19/32] LIX-MDSP-5 # demo and readme updtes --- README.md | 250 ++++++++++------------- demo/svelte-demo/src/routes/+page.svelte | 4 +- 2 files changed, 112 insertions(+), 142 deletions(-) diff --git a/README.md b/README.md index 740eb9a..f63f88b 100644 --- a/README.md +++ b/README.md @@ -45,8 +45,8 @@ Can be used on a backend or frontend, there's no rendering logic involved. ### Basic Concepts -- **Singleton Pattern:** - Use `MarkdownStreamParser.getInstance(instanceId)` to ensure one parser per logical stream/session. +- **Singleton Pattern (async):** + Use `await MarkdownStreamParser.getInstance(instanceId)` to ensure one parser per logical stream/session. The first call loads the tree-sitter WASM grammars, so `getInstance()` returns a `Promise`. - **Parsing Lifecycle:** - `startParsing()`: Begin parsing and set up subscriptions. @@ -54,8 +54,7 @@ Can be used on a backend or frontend, there's no rendering logic involved. - `stopParsing()`: Flush buffers, reset state, and notify listeners of stream end. - **Subscribing to Output:** - Use `subscribeToTokenParse(listener)` to receive parsed segments as soon as they are available. Returns an unsubscribe function. - The unsubscribe function takes no arguments. + Use `subscribeToTokenParse(listener)` to receive parsed chunks as soon as they are available. The listener receives a `StreamingChunk` and an `unsubscribe` function. Returns an unsubscribe function. ## How to Use @@ -64,7 +63,7 @@ There are several ways to use the parser. It is quite modular. You can initializ ## Subscribing to the Parser -Before you can parse the stream, you must subscribe to the parser. If you do not subscribe in advance, the parser will likely stop and terminate before you receive the first segment. +Before you can parse the stream, you must subscribe to the parser. If you do not subscribe in advance, the parser will likely stop and terminate before you receive the first chunk. First import the parser and initialize it with an `instance-id`. (you can have as many parallel parsers as you want, just make sure to use different `instance-id`s) @@ -72,18 +71,19 @@ First import the parser and initialize it with an `instance-id`. (you can have a import { MarkdownStreamParser } from '@lixpi/markdown-stream-parser' // Get a parser instance (singleton per ID) -const parser = MarkdownStreamParser.getInstance('session-1') +// First call loads WASM grammars — subsequent calls for the same ID return instantly +const parser = await MarkdownStreamParser.getInstance('session-1') ``` #### Approach 1: The Simplest ```typescript // Subscribe to parsed output -parser.subscribeToTokenParse((parsedSegment, unsubscribe) => { - console.log(parsedSegment) // Happy little parsed segment +parser.subscribeToTokenParse((streamingChunk, unsubscribe) => { + console.log(streamingChunk) // Happy little parsed chunk // Clean up when the stream ends - if (parsedSegment.status === 'END_STREAM') { + if (streamingChunk.status === 'END_STREAM') { unsubscribe() MarkdownStreamParser.removeInstance('session-1') } @@ -93,16 +93,16 @@ parser.subscribeToTokenParse((parsedSegment, unsubscribe) => { #### Approach 2: Customizable ```typescript -// Subscribe to the parser service -const parserUnsubscribe = parser.subscribeToTokenParse(parsedSegment => { - console.log(parsedSegment) // Happy little parsed segment +// Subscribe to the parser +const parserUnsubscribe = parser.subscribeToTokenParse((streamingChunk) => { + console.log(streamingChunk) // Happy little parsed chunk }) // When the stream has ended stop the parser to avoid issues and memory leaks. // You can decide when to terminate the parser. // For example, using your own logic or rely on the `parser.parsing` flag. if (!parser.parsing) { - parserUnsubscribe() // Unsubscribe from the parser service + parserUnsubscribe() // Unsubscribe from the parser MarkdownStreamParser.removeInstance('session-1') // Dispose of the parser instance } ``` @@ -118,7 +118,7 @@ Again, this can be done in the same file or in a different part of your applicat import { MarkdownStreamParser } from '@lixpi/markdown-stream-parser' // Get a parser instance (singleton per ID) -const parser = MarkdownStreamParser.getInstance('session-1') +const parser = await MarkdownStreamParser.getInstance('session-1') // Start the parser parser.startParsing() @@ -177,26 +177,28 @@ The output is a series of `StreamingChunk` objects. Each chunk contains the text { status: 'END_STREAM' } ``` -### Key Concepts in the New API +### Key Concepts +- **`text`**: Plain text with markdown formatting syntax removed - **`offset`**: UTF-16 code unit offset from the start of the stream - **`length`**: UTF-16 code unit length of the text -- **`block`**: Block-level context (`paragraph`, `heading`, `code_block`, `list_item`, `table`, etc.) +- **`block`**: Block-level context (`paragraph`, `heading`, `code_block`, `list_item`, `table`, `blockquote`, etc.) - **`opening`**: Spans that start in this chunk but don't close (span continues to next chunks) - **`closing`**: Spans that close in this chunk (were opened in earlier chunks) - **`contained`**: Spans fully contained within this chunk +- **`backtrackOffset`**: If present, the parser corrected previous output — discard everything from this offset onwards ## Consumer Span State Management -The new API uses an **orthogonal model** where chunks and spans are completely independent. Spans can cross chunk boundaries. Consumers must track open spans to properly render styled content. +The API uses an **orthogonal model** where chunks and spans are completely independent. Spans can cross chunk boundaries. Consumers must track open spans to properly render styled content. ### How to Track Span State ```typescript import { MarkdownStreamParser, OpenSpan, ClosedSpan, Chunk } from '@lixpi/markdown-stream-parser' -const parser = MarkdownStreamParser.getInstance('session-1') +const parser = await MarkdownStreamParser.getInstance('session-1') // Track currently open spans let openSpans: OpenSpan[] = [] @@ -251,17 +253,22 @@ parser.subscribeToTokenParse((streamingChunk, unsubscribe) => { ### Span Types ```typescript -type SpanType = 'bold' | 'italic' | 'code' | 'strikethrough' | 'link' | 'image' +// The base Span is a discriminated union — links carry URLs, images carry src/alt +type Span = + | { type: 'bold' } + | { type: 'italic' } + | { type: 'code' } + | { type: 'strikethrough' } + | { type: 'link'; url: string } + | { type: 'image'; src: string; alt?: string } + +type SpanType = Span['type'] // 'bold' | 'italic' | 'code' | 'strikethrough' | 'link' | 'image' // Opening span: we know where it starts, but it's not closed yet type OpenSpan = { type: SpanType; openOffset: number } -// Closed/contained span: complete with offset and length +// Closed/contained span: full span data + position info type ClosedSpan = Span & { offset: number; length: number } - -// Link and image spans include additional metadata -type LinkSpan = { type: 'link'; url: string; offset: number; length: number } -type ImageSpan = { type: 'image'; src: string; alt?: string; offset: number; length: number } ``` ### Handling Backtracking @@ -286,21 +293,30 @@ if (chunk.backtrackOffset !== undefined) { ### Configuration Options ```typescript -const parser = MarkdownStreamParser.getInstance('session-1', { - windowSize: 500, // Lookback window for backtrack detection (chars) - includeRawStreamedToken: true // Include original token in chunk.original +// Optional: configure WASM grammar paths before creating any instance +// (only needed if you host the .wasm files at a non-default location) +MarkdownStreamParser.configureWasmPath('/custom/path/tree-sitter-markdown.wasm') + +const parser = await MarkdownStreamParser.getInstance('session-1', { + windowSize: 500, // Lookback window for backtrack detection (UTF-16 code units) + includeRawStreamedToken: true // Include original markdown source in chunk.original }) // Or configure after creation parser.setConfig({ windowSize: 1000 }) ``` +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `windowSize` | `number` | `undefined` (unlimited) | Maximum backtrack distance in UTF-16 code units. Limits how far back the parser can correct previous output. | +| `includeRawStreamedToken` | `boolean` | `false` | When `true`, each chunk includes the original markdown source in `chunk.original`. Useful as a fallback for unsupported formatting. | + ## Is that it? What am I supposed to do with that? -Good question. You can use this stream to render styled content in your application in real time. Having a `segment type` and `inline styles` is enough to style it however you want. +Good question. You can use this stream to render styled content in your application in real time. Having a `block type` and `span information` is enough to style it however you want. -It will **always remain `render-agnostic`** - whatever you use to render your styled text is entirely up to you. +It will **always remain `render-agnostic`** — whatever you use to render your styled text is entirely up to you. ## Features @@ -314,8 +330,8 @@ It will **always remain `render-agnostic`** - whatever you use to render your st - [x] Inline Strikethrough (`~~text~~`) - [x] Inline Code (`` `code` ``) - [x] Code Blocks (```` ```code-block``` ````) with language detection -- [x] Links (`[text](url)`) - with URL extraction -- [x] Images (`![alt](url)`) - with src and alt extraction +- [x] Links (`[text](url)`) — with URL extraction *(no test coverage yet)* +- [x] Images (`![alt](url)`) — with src and alt extraction *(no test coverage yet)* - [ ] Blockquotes (`> quote`) [Issue #2](https://github.com/Lixpi/markdown-stream-parser/issues/2) - [ ] //TODO: PRIORITY: Ordered Lists (`1. item`) [Issue #3](https://github.com/Lixpi/markdown-stream-parser/issues/3) - [ ] //TODO: PRIORITY: Unordered Lists (`- item`, `* item`, `+ item`) *BLOCKED BY:* [Issue #3](https://github.com/Lixpi/markdown-stream-parser/issues/3) @@ -333,7 +349,7 @@ It will **always remain `render-agnostic`** - whatever you use to render your st ## Running examples -To try out the parser with example streams, look inside the `llm-streams-examples` directory. This folder contains real LLM responses collected from various providers. Each response has two versions: +To try out the parser with example streams, look inside the `demo/llm-streams-examples` directory. This folder contains real LLM responses collected from various providers. Each response has two versions: - `*.json`: An array of items used for streaming - `*.txt`: The same stream combined into a single file @@ -348,16 +364,18 @@ Inside the repository root dir run: docker compose up -d ``` -2. Run the debug parser inside the container: +2. Run the tree-sitter debug parser inside the container: ```bash - docker exec -it lixpi-markdown-stream-parser-demo pnpm run debug-parser --file= + docker exec -it lixpi-markdown-stream-parser-demo pnpm run debug-parser-tree-sitter --file= ``` Replace `` with the relative path to any `.json` file. Examples: - - For files in `llm-streams-examples`: `--file=demo/llm-streams-examples/claude-3.5-1-quantum-physics.json` - - For manually created files: `--file=demo/llm-stream-examples-manually-simulated/long-consecutive-sequence.json` + - `--file=demo/llm-streams-examples/claude-3.5-1-quantum-physics.json` + - `--file=demo/llm-stream-examples-manually-simulated/long-consecutive-sequence.json` + + > There's also `debug-parser` which runs the legacy state-machine parser for comparison. -3. **Creating custom test streams**: You can also create your own chunked streams from arbitrary text files using the `split-sample-into-chunks.ts` script: +3. **Creating custom test streams**: You can also create your own chunked streams from arbitrary text files using the `split-sample-into-chunks` script: ```bash docker exec -it lixpi-markdown-stream-parser-demo pnpm run split-sample-into-chunks -- --file= --chunkSize= --outputPath= ``` @@ -367,11 +385,11 @@ Inside the repository root dir run: docker exec -it lixpi-markdown-stream-parser-demo pnpm run split-sample-into-chunks -- --file=demo/llm-input-examples-raw-text/long-consecutive-sequence.txt --chunkSize=2 --outputPath=demo/llm-stream-examples-manually-simulated/long-consecutive-sequence.json ``` -This will execute the parser against the selected example stream and print parsed segments to the console. +This will execute the parser against the selected example stream and print parsed chunks to the console. ## Running tests -The project includes comprehensive test coverage with 187 tests across all core functionality. To run the tests: +The project includes comprehensive test coverage with 219+ tests across all core functionality. To run the tests: 1. Start the Docker container: ```bash @@ -388,13 +406,6 @@ The project includes comprehensive test coverage with 187 tests across all core docker exec -it lixpi-markdown-stream-parser-demo pnpm test ``` -4. Run tests with coverage reporting: - ```bash - docker exec -it lixpi-markdown-stream-parser-demo pnpm test:coverage - ``` - -**Note:** 3 tests are intentionally designed to fail to prove the existence of the known bug with long consecutive character sequences. All other tests should pass. - --- @@ -411,62 +422,73 @@ flowchart LR B --> C[Accumulate Content] C --> D[Tree-sitter Parse] D --> E[AST Traversal] - E --> F[Emit Segments] + E --> F[Emit Chunks] F --> G[Subscribers] ``` ### Module Architecture -The tree-sitter parsing logic is split into focused modules: +The tree-sitter parsing pipeline is organized into four layers. Each layer has a single direction of dependency — top layers call into lower layers, never the reverse. ```mermaid %%{init: {'theme': 'base', 'themeVariables': { 'primaryColor': '#F6C7B3', 'primaryTextColor': '#5a3a2a', 'primaryBorderColor': '#d4956a', 'secondaryColor': '#C3DEDD', 'secondaryTextColor': '#1a3a47', 'secondaryBorderColor': '#4a8a9d', 'tertiaryColor': '#DCECE9', 'tertiaryTextColor': '#1a3a47', 'tertiaryBorderColor': '#82B2C0', 'lineColor': '#d4956a', 'textColor': '#5a3a2a'}}}%% graph TB - subgraph "Entry Point" - Parser[MarkdownStreamParser] + subgraph "Public API" + Parser[MarkdownStreamParser
Singleton · Pub/Sub] + end + + subgraph "Stream Processing" + Backend[TreeSitterStreamParser
Parser backend] + Buffer[TokensStreamBuffer
Token accumulation] end - subgraph "Tree-sitter Modules" - SG[segment-generator.ts] - BD[block-detection.ts] - ID[inline-detection.ts] - CE[content-extraction.ts] - TN[tree-navigation.ts] - SB[segment-builder.ts] - TY[types.ts] + subgraph "Orchestration" + SegGen[SegmentGenerator
Chunk generation] end - subgraph "External" + subgraph "Analysis" + BD[BlockDetection
Block type classification] + ID[InlineDetection
Span detection] + CE[ContentExtraction
Syntax stripping] + SB[SegmentBuilder
Chunk/Span construction] + end + + subgraph "Foundation" + TN[TreeNavigation
AST traversal utilities] + Types[Types
Chunk · Span · BlockContext] + end + + subgraph "External Grammars" TS[(web-tree-sitter)] - MD[(tree-sitter-markdown)] - MDI[(tree-sitter-markdown-inline)] + TS --> MD[(tree-sitter-markdown)] + TS --> MDI[(tree-sitter-markdown-inline)] end - Parser --> SG - SG --> BD - SG --> ID - SG --> CE - SG --> SB - SG --> TY + Parser --> Backend + Backend --> Buffer + Backend --> SegGen + Backend -.-> TS + SegGen --> BD + SegGen --> ID + SegGen --> CE + SegGen --> SB BD --> TN ID --> TN - SB --> TY - CE --> TS - BD --> TS - ID --> TS - TS --> MD - TS --> MDI ``` -| Module | Responsibility | -|--------|----------------| -| `segment-generator.ts` | Main orchestrator - generates chunks from content ranges with span detection | -| `block-detection.ts` | Determines block type (heading, paragraph, code_block, list_item, table) | -| `inline-detection.ts` | Detects inline spans (bold, italic, code, strikethrough, link, image) | -| `content-extraction.ts` | Strips markdown syntax and extracts clean content | -| `tree-navigation.ts` | AST traversal utilities | -| `segment-builder.ts` | Creates Chunk and Span objects with UTF-16 offsets | -| `types.ts` | Type definitions (Chunk, Span, BlockContext, etc.) | +> `types.ts` is a shared dependency imported by every module in the pipeline — arrows omitted for clarity. + +| Layer | Module | Responsibility | +|-------|--------|----------------| +| Stream Processing | `tree-sitter-markdown-stream-parser.ts` | Parser backend — manages tree-sitter lifecycle, drives the pipeline | +| Stream Processing | `tokens-stream-buffer.ts` | Accumulates incoming tokens into parseable content windows | +| Orchestration | `segment-generator.ts` | Central orchestrator — generates chunks with block context and span info | +| Analysis | `block-detection.ts` | Classifies block type: `heading`, `paragraph`, `code_block`, `list_item`, `table` | +| Analysis | `inline-detection.ts` | Detects inline spans: bold, italic, code, strikethrough, link, image | +| Analysis | `content-extraction.ts` | Strips markdown delimiters, extracts clean text content | +| Analysis | `segment-builder.ts` | Constructs `Chunk` and `Span` objects with UTF-16 offsets | +| Foundation | `tree-navigation.ts` | AST traversal — finds nodes at positions, walks inline trees | +| Foundation | `types.ts` | Shared type definitions: `Chunk`, `Span`, `BlockContext`, `SpanType` | ### Parser API Flow @@ -548,57 +570,16 @@ sequenceDiagram end ``` -### Parser State Transitions - -```mermaid -%%{init: {'theme': 'base', 'themeVariables': { 'primaryColor': '#F6C7B3', 'primaryTextColor': '#5a3a2a', 'primaryBorderColor': '#d4956a', 'secondaryColor': '#C3DEDD', 'lineColor': '#d4956a', 'textColor': '#5a3a2a'}}}%% -stateDiagram-v2 - [*] --> Idle: getInstance(config?) - - Idle --> Parsing: startParsing() - - state Parsing { - [*] --> AwaitingToken - - AwaitingToken --> ProcessingChunk: parseToken(chunk) - ProcessingChunk --> DetectingBlock: tree-sitter parse - ProcessingChunk --> BacktrackDetected: getChangedRanges() detects change - BacktrackDetected --> DetectingBlock: emit with backtrackOffset - DetectingBlock --> ProcessingHeading: atx_heading found - DetectingBlock --> ProcessingParagraph: paragraph found - DetectingBlock --> ProcessingCodeBlock: fenced_code_block found - DetectingBlock --> ProcessingList: list_item found - DetectingBlock --> ProcessingTable: pipe_table found - - ProcessingHeading --> DetectingSpans: check inline spans - ProcessingParagraph --> DetectingSpans: check inline spans - ProcessingList --> DetectingSpans: check inline spans - ProcessingTable --> DetectingSpans: check inline spans - - DetectingSpans --> BufferingIncomplete: unmatched delimiter - DetectingSpans --> ProcessSpans: spans detected - ProcessSpans --> CategorizeSpans: opening/closing/contained - BufferingIncomplete --> AwaitingToken: wait for more - - ProcessingCodeBlock --> EmitChunk: extract content - CategorizeSpans --> EmitChunk: build Chunk with spans - EmitChunk --> AwaitingToken: notify subscribers - } - - Parsing --> Flushing: stopParsing() - Flushing --> Idle: END_STREAM - Idle --> [*]: removeInstance() -``` ### How Content Gets Processed #### 1. Token Buffering -Incoming tokens are accumulated in a `TokensStreamBuffer`. This gives us enough context to parse meaningful chunks rather than character-by-character. +Incoming tokens are accumulated in a `TokensStreamBuffer`. The buffer waits for word boundaries (whitespace) before emitting, so the parser always has enough context to produce meaningful chunks rather than character-by-character. #### 2. AST-Based Parsing -The core parsing is done by `web-tree-sitter` with the `tree-sitter-markdown` grammar. When content comes in, we parse it and get an AST that tells us exactly what we're dealing with - headers, paragraphs, code blocks, lists, bold text, etc. +The core parsing is done by `web-tree-sitter` with the `tree-sitter-markdown` grammar. When content comes in, we parse it incrementally (editing the existing tree) and get an AST that tells us exactly what we're dealing with — headers, paragraphs, code blocks, lists, bold text, etc. Tree-sitter handles incomplete/malformed markdown gracefully. It uses error recovery and can still produce a usable tree even when the input is partial or slightly broken (which happens constantly with LLM streams). @@ -606,20 +587,9 @@ Tree-sitter handles incomplete/malformed markdown gracefully. It uses error reco A tricky problem with streaming is that inline markers can arrive split across chunks. For example, you might get `**hello` in one chunk and `**` in the next. -The parser buffers content when it detects an unmatched delimiter: - -```typescript -// Check for unmatched backtick -if (newPortion.includes('`')) { - const hasCompleteCodeSpan = hasCompleteCodeSpanAt(inlineTree.rootNode, ...); - if (!hasCompleteCodeSpan) { - state.pendingInlineContent = newContent; - return { segments, state }; // Buffer and wait for more - } -} -``` +The parser buffers content when it detects an unmatched delimiter. It checks whether the inline tree-sitter parser can see a complete structure (emphasis, code_span, etc.). If not, the content is held in `pendingInlineContent` and the parser waits for more tokens before emitting. -This applies to inline code, bold (`**`), italic (`*` or `_`), and strikethrough (`~~`). +This applies to inline code (`` ` ``), bold (`**`), italic (`*` or `_`), and strikethrough (`~~`). #### 4. Two-Parser Approach @@ -629,9 +599,9 @@ The two-parser approach (one for block structure, one for inline content) is how ### Pub/Sub and Singleton Patterns -The parser uses a **publish/subscribe** pattern - you subscribe to get parsed segments as they're ready. Parsing is decoupled from rendering, and multiple subscribers per parser instance are supported. +The parser uses a **publish/subscribe** pattern — you subscribe to get parsed chunks as they're ready. Parsing is decoupled from rendering, and multiple subscribers per parser instance are supported. -Each logical stream gets its own parser instance via `getInstance(instanceId)` (singleton pattern). This allows parallel processing of multiple streams without state conflicts. +Each logical stream gets its own parser instance via `await getInstance(instanceId)` (singleton pattern). This allows parallel processing of multiple streams without state conflicts. ```typescript const parser = await MarkdownStreamParser.getInstance('session-1') diff --git a/demo/svelte-demo/src/routes/+page.svelte b/demo/svelte-demo/src/routes/+page.svelte index a29029f..764815f 100644 --- a/demo/svelte-demo/src/routes/+page.svelte +++ b/demo/svelte-demo/src/routes/+page.svelte @@ -393,8 +393,8 @@ !!! Please keep that in mind...

- This demo is entirely `vibe-coded`, while - the parser is painstakingly created by a human being 👩‍💻 :) + This parser setup example is just an AI slop, its only goal is to visually showcase the parser. + pls refer to the readme file for better instruction on how to user parser API

From 6c7082e031b805e7c1eedcaf6f6b8804e6d353c9 Mon Sep 17 00:00:00 2001 From: Dmitry Bondarenko Date: Mon, 16 Feb 2026 22:32:32 +0600 Subject: [PATCH 20/32] LIX-MDSP-11: implement tree-sitter error recovery --- ...tree-sitter-markdown-stream-parser.test.ts | 90 +++++++++++++++++++ src/tree-sitter-markdown-stream-parser.ts | 52 +++++++++-- 2 files changed, 133 insertions(+), 9 deletions(-) diff --git a/src/tree-sitter-markdown-stream-parser.test.ts b/src/tree-sitter-markdown-stream-parser.test.ts index fff0b83..36ace7c 100644 --- a/src/tree-sitter-markdown-stream-parser.test.ts +++ b/src/tree-sitter-markdown-stream-parser.test.ts @@ -645,4 +645,94 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { expect(hasStrippedCode).toBe(true) }) }) + + describe('Error Recovery', () => { + it('should not emit backtrackOffset for clean streaming', async () => { + parser.parseToken('Hello ') + parser.parseToken('world\n') + parser.stopParsing() + + // No chunks should have backtrackOffset set + const backtrackChunks = parsedChunks.filter(c => c.backtrackOffset !== undefined) + expect(backtrackChunks.length).toBe(0) + }) + + it('should emit backtrackOffset when tree-sitter re-parses a region', async () => { + // Stream bold text split across chunks — tree-sitter will initially + // parse "**bold" as error/plain text, then correct when "**" arrives + parser.parseToken('Hello **bold') + parser.parseToken('** rest\n') + parser.stopParsing() + + // Check if any chunk has backtrackOffset set + const backtrackChunks = parsedChunks.filter(c => c.backtrackOffset !== undefined) + + // If tree-sitter detected a correction, we should see backtrackOffset + // and the corrected chunks should contain the bold span + if (backtrackChunks.length > 0) { + const backtrackChunk = backtrackChunks[0] + expect(backtrackChunk.backtrackOffset).toBeDefined() + expect(typeof backtrackChunk.backtrackOffset).toBe('number') + expect(backtrackChunk.backtrackOffset!).toBeGreaterThanOrEqual(0) + + // After the backtrack, the corrected chunks should have bold text + // Find all chunks emitted at or after the backtrack offset + const correctedChunks = parsedChunks.filter( + c => c.offset >= backtrackChunk.backtrackOffset! + ) + expect(correctedChunks.length).toBeGreaterThan(0) + } + }) + + it('should re-emit corrected segments from backtrack point', async () => { + // Stream italic that starts ambiguously + parser.parseToken('Text *italic') + parser.parseToken('* more\n') + parser.stopParsing() + + // Check the full reconstructed text contains expected content + const fullText = parsedChunks.map(c => c.text).join('') + expect(fullText).toContain('Text') + expect(fullText).toContain('more') + }) + + it('should respect windowSize configuration', async () => { + // Create a parser with windowSize constraint + const windowInstanceId = 'test-window-size' + const windowParser = await MarkdownStreamParser.getInstance(windowInstanceId, { + windowSize: 5, + }) + + const windowChunks: Chunk[] = [] + windowParser.subscribeToTokenParse((chunk) => { + if (chunk.status === 'STREAMING' && chunk.chunk) { + windowChunks.push(chunk.chunk) + } + }) + + windowParser.startParsing() + + // Stream content that might trigger backtracking + windowParser.parseToken('Hello **bold text here') + windowParser.parseToken('** end\n') + windowParser.stopParsing() + + // If backtracking occurred, the offset should be clamped + const backtrackChunks = windowChunks.filter(c => c.backtrackOffset !== undefined) + if (backtrackChunks.length > 0) { + const lastEmitted = Math.max(...windowChunks + .filter(c => c.backtrackOffset === undefined) + .map(c => c.offset + c.length)) + const backtrackChunk = backtrackChunks[0] + + // The backtrack distance should not exceed windowSize + if (lastEmitted > 0) { + const distance = lastEmitted - backtrackChunk.backtrackOffset! + expect(distance).toBeLessThanOrEqual(5) + } + } + + MarkdownStreamParser.removeInstance(windowInstanceId) + }) + }) }) diff --git a/src/tree-sitter-markdown-stream-parser.ts b/src/tree-sitter-markdown-stream-parser.ts index 1aaf893..f376864 100644 --- a/src/tree-sitter-markdown-stream-parser.ts +++ b/src/tree-sitter-markdown-stream-parser.ts @@ -2,6 +2,7 @@ import { Parser, Language } from 'web-tree-sitter' import TokensStreamBuffer from './tokens-stream-buffer.js' import type { StreamingChunk, BlockState, ParserConfig, SegmentGeneratorState, Chunk } from './tree-sitter/types.js' import { generateSegments, createInitialState } from './tree-sitter/segment-generator.js' +import { utf16ToByteOffset } from './tree-sitter/segment-builder.js' // Re-export types for external consumers export type { @@ -338,7 +339,48 @@ export class MarkdownStreamParser { } } - // Generate segments using the refactored module + if (backtrackOffset !== undefined) { + // Error recovery: re-generate segments from the backtrack point + // Convert UTF-16 backtrack offset to byte offset for generateSegments + const backtrackByteOffset = utf16ToByteOffset(this.content, backtrackOffset) + + // Reset generator state to the backtrack point + const resetState: SegmentGeneratorState = { + totalUtf16Offset: backtrackOffset, + lastEmittedOffset: backtrackOffset, + openSpans: [], + currentBlock: null, + pendingInlineContent: '', + accumulatedContent: this.content.substring(0, backtrackByteOffset), + } + + // Re-generate all segments from backtrack point through end of content + const result = generateSegments(backtrackByteOffset, this.content.length, { + content: this.content, + currentTree: this.currentTree, + inlineParser: this.inlineParser, + state: resetState, + config: this.config, + }) + + // Update state + this.generatorState = result.state + + // Set backtrackOffset on the first re-generated chunk + if (result.segments.length > 0) { + const firstSeg = result.segments[0] + if (firstSeg.status === 'STREAMING' && firstSeg.chunk) { + firstSeg.chunk.backtrackOffset = backtrackOffset + } + } + + // Store all segments for debugging + this.allSegments.push(...result.segments) + + return result.segments + } + + // Normal path: no backtracking, generate segments for new content only const result = generateSegments(oldLength, this.content.length, { content: this.content, currentTree: this.currentTree, @@ -350,14 +392,6 @@ export class MarkdownStreamParser { // Update state this.generatorState = result.state - // Add backtrackOffset to first chunk if needed - if (backtrackOffset !== undefined && result.segments.length > 0) { - const firstSeg = result.segments[0] - if (firstSeg.status === 'STREAMING' && firstSeg.chunk) { - firstSeg.chunk.backtrackOffset = backtrackOffset - } - } - // Store all segments for debugging this.allSegments.push(...result.segments) From dd7416d065e989a2b239ed02ea354545bb8367c8 Mon Sep 17 00:00:00 2001 From: Dmitry Bondarenko Date: Tue, 24 Feb 2026 22:21:42 +0600 Subject: [PATCH 21/32] Testing backtrack on frontend --- .../test-error-recovery.json | 8 + .../test-error-recovery.txt | 3 + demo/svelte-demo/src/routes/+page.svelte | 199 +++++++++++++----- .../static/llm-examples-manifest.json | 5 + 4 files changed, 168 insertions(+), 47 deletions(-) create mode 100644 demo/llm-streams-examples/test-error-recovery.json create mode 100644 demo/llm-streams-examples/test-error-recovery.txt diff --git a/demo/llm-streams-examples/test-error-recovery.json b/demo/llm-streams-examples/test-error-recovery.json new file mode 100644 index 0000000..1a98327 --- /dev/null +++ b/demo/llm-streams-examples/test-error-recovery.json @@ -0,0 +1,8 @@ +[ + "Hello ", + "**bold", + "** rest\n", + "Normal text.\n", + "Then *italic", + "* stuff\n" +] \ No newline at end of file diff --git a/demo/llm-streams-examples/test-error-recovery.txt b/demo/llm-streams-examples/test-error-recovery.txt new file mode 100644 index 0000000..2bff37c --- /dev/null +++ b/demo/llm-streams-examples/test-error-recovery.txt @@ -0,0 +1,3 @@ +Hello **bold** rest +Normal text. +Then *italic* stuff diff --git a/demo/svelte-demo/src/routes/+page.svelte b/demo/svelte-demo/src/routes/+page.svelte index 764815f..974a859 100644 --- a/demo/svelte-demo/src/routes/+page.svelte +++ b/demo/svelte-demo/src/routes/+page.svelte @@ -97,7 +97,7 @@ function updateOpenSpans(chunk: Chunk) { // Remove closed spans for (const closedSpan of chunk.closing) { - openSpans = openSpans.filter(s => s.type !== closedSpan.type); + openSpans = openSpans.filter((s) => s.type !== closedSpan.type); } // Add new opening spans @@ -135,6 +135,43 @@ } else if (parsed.status === "START_STREAM") { parsedSegments = [...parsedSegments, parsed]; } else if (parsed.status === "STREAMING") { + const chunk = parsed.chunk; + + console.log("📦 CHUNK received", { + text: JSON.stringify(chunk.text), + offset: chunk.offset, + length: chunk.length, + block: chunk.block.type, + backtrackOffset: chunk.backtrackOffset, + }); + + if (chunk.backtrackOffset !== undefined) { + console.warn("⚠️ BACKTRACK detected!", { + backtrackOffset: chunk.backtrackOffset, + chunkText: chunk.text, + chunkOffset: chunk.offset, + discarding: parsedSegments + .filter( + (seg) => + seg.status === "STREAMING" && + seg.chunk.offset + seg.chunk.length > + chunk.backtrackOffset!, + ) + .map((seg) => + seg.status === "STREAMING" ? seg.chunk.text : null, + ), + }); + + parsedSegments = parsedSegments.filter((seg) => { + if (seg.status !== "STREAMING") return true; + return ( + seg.chunk.offset + seg.chunk.length <= chunk.backtrackOffset! + ); + }); + + openSpans = []; + } + parsedSegments = [...parsedSegments, parsed]; updateOpenSpans(parsed.chunk); @@ -288,13 +325,14 @@ if (blockType !== lastBlockType) { isNewBlock = true; - } else if (blockType === 'heading' && blockLevel !== lastBlockLevel) { + } else if (blockType === "heading" && blockLevel !== lastBlockLevel) { isNewBlock = true; - } else if (blockType === 'list_item' && lastOffset >= 0) { + } else if (blockType === "list_item" && lastOffset >= 0) { // New list item if there's a significant gap in offset (indicates newline/new item) // Or if the text starts after a newline marker const gap = chunk.offset - lastOffset; - if (gap > 50) { // Heuristic: large gap suggests new list item + if (gap > 50) { + // Heuristic: large gap suggests new list item isNewBlock = true; } } @@ -358,24 +396,24 @@ function getSpanClasses(styles: SpanType[]): string { const classes: string[] = []; - if (styles.includes('bold') && styles.includes('italic')) { - classes.push('font-bold', 'italic'); - } else if (styles.includes('bold')) { - classes.push('font-bold'); - } else if (styles.includes('italic')) { - classes.push('italic'); + if (styles.includes("bold") && styles.includes("italic")) { + classes.push("font-bold", "italic"); + } else if (styles.includes("bold")) { + classes.push("font-bold"); + } else if (styles.includes("italic")) { + classes.push("italic"); } - if (styles.includes('strikethrough')) { - classes.push('line-through'); + if (styles.includes("strikethrough")) { + classes.push("line-through"); } - return classes.join(' '); + return classes.join(" "); } // Check if style includes code function hasCodeStyle(styles: SpanType[]): boolean { - return styles.includes('code'); + return styles.includes("code"); } @@ -393,8 +431,12 @@ !!! Please keep that in mind...

- This parser setup example is just an AI slop, its only goal is to visually showcase the parser. - pls refer to the readme file for better instruction on how to user parser API + This parser setup example is just an AI slop, its only goal is to + visually showcase the parser. + pls refer to the readme file for better instruction on how to user + parser API

@@ -476,16 +518,22 @@ {@const blockType = block[0]?.block.type} {@const blockLevel = block[0]?.block.level} {@const blockLanguage = block[0]?.block.language} - {@const hasTableCells = blockType === 'table_cell' || blockType === 'table_row'} + {@const hasTableCells = + blockType === "table_cell" || blockType === "table_row"}
- {#if blockType === 'heading'} + {#if blockType === "heading"} {#if blockLevel === 1}

{#each block as chunk} - {@const styles = [...chunk.contained.map(s => s.type), ...chunk.opening.map(s => s.type)]} + {@const styles = [ + ...chunk.contained.map((s) => s.type), + ...chunk.opening.map((s) => s.type), + ]} {#if hasCodeStyle(styles)} - {chunk.text} + {chunk.text} {:else} {chunk.text} {/if} @@ -494,9 +542,14 @@ {:else if blockLevel === 2}

{#each block as chunk} - {@const styles = [...chunk.contained.map(s => s.type), ...chunk.opening.map(s => s.type)]} + {@const styles = [ + ...chunk.contained.map((s) => s.type), + ...chunk.opening.map((s) => s.type), + ]} {#if hasCodeStyle(styles)} - {chunk.text} + {chunk.text} {:else} {chunk.text} {/if} @@ -505,9 +558,14 @@ {:else if blockLevel === 3}

{#each block as chunk} - {@const styles = [...chunk.contained.map(s => s.type), ...chunk.opening.map(s => s.type)]} + {@const styles = [ + ...chunk.contained.map((s) => s.type), + ...chunk.opening.map((s) => s.type), + ]} {#if hasCodeStyle(styles)} - {chunk.text} + {chunk.text} {:else} {chunk.text} {/if} @@ -516,9 +574,14 @@ {:else if blockLevel === 4}

{#each block as chunk} - {@const styles = [...chunk.contained.map(s => s.type), ...chunk.opening.map(s => s.type)]} + {@const styles = [ + ...chunk.contained.map((s) => s.type), + ...chunk.opening.map((s) => s.type), + ]} {#if hasCodeStyle(styles)} - {chunk.text} + {chunk.text} {:else} {chunk.text} {/if} @@ -527,9 +590,14 @@ {:else if blockLevel === 5}

{#each block as chunk} - {@const styles = [...chunk.contained.map(s => s.type), ...chunk.opening.map(s => s.type)]} + {@const styles = [ + ...chunk.contained.map((s) => s.type), + ...chunk.opening.map((s) => s.type), + ]} {#if hasCodeStyle(styles)} - {chunk.text} + {chunk.text} {:else} {chunk.text} {/if} @@ -538,9 +606,14 @@ {:else if blockLevel === 6}
{#each block as chunk} - {@const styles = [...chunk.contained.map(s => s.type), ...chunk.opening.map(s => s.type)]} + {@const styles = [ + ...chunk.contained.map((s) => s.type), + ...chunk.opening.map((s) => s.type), + ]} {#if hasCodeStyle(styles)} - {chunk.text} + {chunk.text} {:else} {chunk.text} {/if} @@ -549,49 +622,76 @@ {:else} {#each block as chunk} - {@const styles = [...chunk.contained.map(s => s.type), ...chunk.opening.map(s => s.type)]} + {@const styles = [ + ...chunk.contained.map((s) => s.type), + ...chunk.opening.map((s) => s.type), + ]} {#if hasCodeStyle(styles)} - {chunk.text} + {chunk.text} {:else} {chunk.text} {/if} {/each} {/if} - {:else if blockType === 'code_block'} - 
{#each block as chunk}{chunk.text}{/each}
+ {:else if blockType === "code_block"} + 
{#each block as chunk}{chunk.text}{/each}
{#if blockLanguage} {blockLanguage} {/if} - {:else if blockType === 'blockquote'} - + {:else if blockType === "blockquote"} + {#each block as chunk} - {@const styles = [...chunk.contained.map(s => s.type), ...chunk.opening.map(s => s.type)]} + {@const styles = [ + ...chunk.contained.map((s) => s.type), + ...chunk.opening.map((s) => s.type), + ]} {#if hasCodeStyle(styles)} - {chunk.text} + {chunk.text} {:else} {chunk.text} {/if} {/each} - {:else if blockType === 'list_item'} + {:else if blockType === "list_item"} • {#each block as chunk} - {@const styles = [...chunk.contained.map(s => s.type), ...chunk.opening.map(s => s.type)]} + {@const styles = [ + ...chunk.contained.map((s) => s.type), + ...chunk.opening.map((s) => s.type), + ]} {#if hasCodeStyle(styles)} - {chunk.text} + {chunk.text} {:else} {chunk.text} {/if} {/each} - {:else if blockType === 'table_cell' || blockType === 'table_row'} + {:else if blockType === "table_cell" || blockType === "table_row"} {#each block as chunk} - {@const styles = [...chunk.contained.map(s => s.type), ...chunk.opening.map(s => s.type)]} - + {@const styles = [ + ...chunk.contained.map((s) => s.type), + ...chunk.opening.map((s) => s.type), + ]} + {#if hasCodeStyle(styles)} - {chunk.text} + {chunk.text} {:else} {chunk.text} {/if} @@ -601,9 +701,14 @@ {#each block as chunk} - {@const styles = [...chunk.contained.map(s => s.type), ...chunk.opening.map(s => s.type)]} + {@const styles = [ + ...chunk.contained.map((s) => s.type), + ...chunk.opening.map((s) => s.type), + ]} {#if hasCodeStyle(styles)} - {chunk.text} + {chunk.text} {:else} {chunk.text} {/if} diff --git a/demo/svelte-demo/static/llm-examples-manifest.json b/demo/svelte-demo/static/llm-examples-manifest.json index 84e5e9d..39669e1 100644 --- a/demo/svelte-demo/static/llm-examples-manifest.json +++ b/demo/svelte-demo/static/llm-examples-manifest.json @@ -74,6 +74,11 @@ "json": "/llm-streams-examples/gpt-4.o-immortal people.json", "txt": "/llm-streams-examples/gpt-4.o-immortal people.txt" }, + { + "base": "test-error-recovery", + "json": "/llm-streams-examples/test-error-recovery.json", + "txt": "/llm-streams-examples/test-error-recovery.txt" + }, { "base": "test-strikethrough", "json": "/llm-streams-examples/test-strikethrough.json", From 8b4415187a01d799abf976ef48f594609d37b0b8 Mon Sep 17 00:00:00 2001 From: Dmitry Bondarenko Date: Sun, 15 Mar 2026 21:43:30 +0600 Subject: [PATCH 22/32] Remove Unnecessary byteToUtf16 Conversions --- .../test-error-recovery.json | 16 +++++--- .../test-error-recovery.txt | 17 ++++++-- demo/svelte-demo/src/routes/+page.svelte | 9 +---- src/tree-sitter-markdown-stream-parser.ts | 31 ++++----------- src/tree-sitter/segment-builder.ts | 39 ------------------- src/tree-sitter/segment-generator.ts | 6 +-- 6 files changed, 36 insertions(+), 82 deletions(-) diff --git a/demo/llm-streams-examples/test-error-recovery.json b/demo/llm-streams-examples/test-error-recovery.json index 1a98327..c77e834 100644 --- a/demo/llm-streams-examples/test-error-recovery.json +++ b/demo/llm-streams-examples/test-error-recovery.json @@ -1,8 +1,12 @@ [ - "Hello ", - "**bold", - "** rest\n", - "Normal text.\n", - "Then *italic", - "* stuff\n" + "## Table Test\n\n", + "| Col A | Col B |\n", + "| --- | --- |\n", + "| cell1 | cell2 |\n\n", + "## Code Fence Test\n\n", + "Some paragraph text here.\n", + "```python\n", + "x = 1\n", + "```\n\n", + "After code.\n" ] \ No newline at end of file diff --git a/demo/llm-streams-examples/test-error-recovery.txt b/demo/llm-streams-examples/test-error-recovery.txt index 2bff37c..c7d255b 100644 --- a/demo/llm-streams-examples/test-error-recovery.txt +++ b/demo/llm-streams-examples/test-error-recovery.txt @@ -1,3 +1,14 @@ -Hello **bold** rest -Normal text. -Then *italic* stuff +## Table Test + +| Col A | Col B | +| --- | --- | +| cell1 | cell2 | + +## Code Fence Test + +Some paragraph text here. +```python +x = 1 +``` + +After code. diff --git a/demo/svelte-demo/src/routes/+page.svelte b/demo/svelte-demo/src/routes/+page.svelte index 974a859..66702d4 100644 --- a/demo/svelte-demo/src/routes/+page.svelte +++ b/demo/svelte-demo/src/routes/+page.svelte @@ -137,13 +137,8 @@ } else if (parsed.status === "STREAMING") { const chunk = parsed.chunk; - console.log("📦 CHUNK received", { - text: JSON.stringify(chunk.text), - offset: chunk.offset, - length: chunk.length, - block: chunk.block.type, - backtrackOffset: chunk.backtrackOffset, - }); + + if (chunk.backtrackOffset !== undefined) { console.warn("⚠️ BACKTRACK detected!", { diff --git a/src/tree-sitter-markdown-stream-parser.ts b/src/tree-sitter-markdown-stream-parser.ts index f376864..ab7cfca 100644 --- a/src/tree-sitter-markdown-stream-parser.ts +++ b/src/tree-sitter-markdown-stream-parser.ts @@ -2,7 +2,7 @@ import { Parser, Language } from 'web-tree-sitter' import TokensStreamBuffer from './tokens-stream-buffer.js' import type { StreamingChunk, BlockState, ParserConfig, SegmentGeneratorState, Chunk } from './tree-sitter/types.js' import { generateSegments, createInitialState } from './tree-sitter/segment-generator.js' -import { utf16ToByteOffset } from './tree-sitter/segment-builder.js' + // Re-export types for external consumers export type { @@ -318,20 +318,18 @@ export class MarkdownStreamParser { const changedRanges = this.previousTree.getChangedRanges(this.currentTree) for (const range of changedRanges) { - // Convert byte offset to UTF-16 offset for the backtrack position + // range.startIndex is already a UTF-16 character offset in web-tree-sitter JS bindings // If the change starts before what we've emitted, we need to backtrack - const changeStartUtf16 = this.byteToUtf16(range.startIndex) + const changeStartUtf16 = range.startIndex if (changeStartUtf16 < this.generatorState.lastEmittedOffset) { + console.log('🔴 BACKTRACK TRIGGERED: changeStart', changeStartUtf16, '< lastEmitted', this.generatorState.lastEmittedOffset) // Check windowSize constraint const backtrackDistance = this.generatorState.lastEmittedOffset - changeStartUtf16 if (this.config.windowSize === undefined || backtrackDistance <= this.config.windowSize) { - // Backtrack is within window backtrackOffset = Math.min(backtrackOffset ?? Infinity, changeStartUtf16) } else { - // Backtrack exceeds window - best effort - // Set backtrack to the edge of the window const windowStart = this.generatorState.lastEmittedOffset - this.config.windowSize backtrackOffset = Math.min(backtrackOffset ?? Infinity, windowStart) } @@ -341,8 +339,7 @@ export class MarkdownStreamParser { if (backtrackOffset !== undefined) { // Error recovery: re-generate segments from the backtrack point - // Convert UTF-16 backtrack offset to byte offset for generateSegments - const backtrackByteOffset = utf16ToByteOffset(this.content, backtrackOffset) + // backtrackOffset is already a UTF-16 character offset // Reset generator state to the backtrack point const resetState: SegmentGeneratorState = { @@ -351,11 +348,11 @@ export class MarkdownStreamParser { openSpans: [], currentBlock: null, pendingInlineContent: '', - accumulatedContent: this.content.substring(0, backtrackByteOffset), + accumulatedContent: this.content.substring(0, backtrackOffset), } // Re-generate all segments from backtrack point through end of content - const result = generateSegments(backtrackByteOffset, this.content.length, { + const result = generateSegments(backtrackOffset, this.content.length, { content: this.content, currentTree: this.currentTree, inlineParser: this.inlineParser, @@ -398,21 +395,7 @@ export class MarkdownStreamParser { return result.segments } - // Convert byte offset to UTF-16 code unit offset. - private byteToUtf16(byteOffset: number): number { - const encoder = new TextEncoder() - let utf16Offset = 0 - let currentByteOffset = 0 - - for (const char of this.content) { - if (currentByteOffset >= byteOffset) break - const charBytes = encoder.encode(char).length - currentByteOffset += charBytes - utf16Offset += char.length - } - return utf16Offset - } // Get the current accumulated content. getCurrentContent(): string { diff --git a/src/tree-sitter/segment-builder.ts b/src/tree-sitter/segment-builder.ts index cbf4ae1..9662940 100644 --- a/src/tree-sitter/segment-builder.ts +++ b/src/tree-sitter/segment-builder.ts @@ -9,45 +9,6 @@ import type { ParserConfig } from './types.js' -// ============================================================================ -// UTF-16 OFFSET UTILITIES -// ============================================================================ - -// Convert byte offset to UTF-16 code unit offset. -// Tree-sitter gives us byte positions, but JavaScript strings use UTF-16. -export function byteOffsetToUtf16(text: string, byteOffset: number): number { - const encoder = new TextEncoder() - let utf16Offset = 0 - let currentByteOffset = 0 - - for (const char of text) { - if (currentByteOffset >= byteOffset) break - const charBytes = encoder.encode(char).length - currentByteOffset += charBytes - // Each JS string char is 1 UTF-16 code unit, except surrogates (2) - utf16Offset += char.length // .length gives UTF-16 code units - } - - return utf16Offset -} - -// Convert UTF-16 offset to byte offset. -// Needed when we have UTF-16 positions and need tree-sitter byte positions. -export function utf16ToByteOffset(text: string, utf16Offset: number): number { - const encoder = new TextEncoder() - let currentUtf16 = 0 - let byteOffset = 0 - - for (const char of text) { - if (currentUtf16 >= utf16Offset) break - const charBytes = encoder.encode(char).length - byteOffset += charBytes - currentUtf16 += char.length - } - - return byteOffset -} - // ============================================================================ // BLOCK CONTEXT HELPERS // ============================================================================ diff --git a/src/tree-sitter/segment-generator.ts b/src/tree-sitter/segment-generator.ts index 4bc4e46..4ffa4e6 100644 --- a/src/tree-sitter/segment-generator.ts +++ b/src/tree-sitter/segment-generator.ts @@ -29,7 +29,6 @@ import { createPlainTextChunk, createCodeBlockChunk, createHeadingChunk, - byteOffsetToUtf16, createOpenSpan, createClosedSpan, @@ -156,8 +155,9 @@ function categorizeSpanNode( const spanType = detectSpanType(node.type) if (!spanType) return {} - const spanStartUtf16 = byteOffsetToUtf16(content, node.startIndex) - const spanEndUtf16 = byteOffsetToUtf16(content, node.endIndex) + // node.startIndex/endIndex are already UTF-16 character offsets in web-tree-sitter JS bindings + const spanStartUtf16 = node.startIndex + const spanEndUtf16 = node.endIndex const spanLength = spanEndUtf16 - spanStartUtf16 // Fully contained From da20e82521d68777502bc0adf5bd93e30ed31503 Mon Sep 17 00:00:00 2001 From: Dmitry Bondarenko Date: Fri, 20 Mar 2026 22:01:59 +0600 Subject: [PATCH 23/32] Fixed broken backtrack re-generation, rewrote error recovery tests --- ...tree-sitter-markdown-stream-parser.test.ts | 122 +++++++++++------- src/tree-sitter-markdown-stream-parser.ts | 83 +++++++++--- 2 files changed, 145 insertions(+), 60 deletions(-) diff --git a/src/tree-sitter-markdown-stream-parser.test.ts b/src/tree-sitter-markdown-stream-parser.test.ts index 36ace7c..941d8f9 100644 --- a/src/tree-sitter-markdown-stream-parser.test.ts +++ b/src/tree-sitter-markdown-stream-parser.test.ts @@ -657,47 +657,80 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { expect(backtrackChunks.length).toBe(0) }) - it('should emit backtrackOffset when tree-sitter re-parses a region', async () => { - // Stream bold text split across chunks — tree-sitter will initially - // parse "**bold" as error/plain text, then correct when "**" arrives - parser.parseToken('Hello **bold') - parser.parseToken('** rest\n') - parser.stopParsing() + it('should emit backtrackOffset when table header is reclassified', async () => { + // When '| Col A | Col B |' arrives alone, tree-sitter parses it as paragraph. + // When '| --- | --- |' arrives next, tree-sitter reclassifies the first line + // as pipe_table_header — a genuine block-level structural change. + const tableId = 'test-table-backtrack' + const tableParser = await MarkdownStreamParser.getInstance(tableId) + const tableChunks: Chunk[] = [] + + tableParser.subscribeToTokenParse((chunk) => { + if (chunk.status === 'STREAMING' && chunk.chunk) { + tableChunks.push(chunk.chunk) + } + }) - // Check if any chunk has backtrackOffset set - const backtrackChunks = parsedChunks.filter(c => c.backtrackOffset !== undefined) + tableParser.startParsing() + tableParser.parseToken('| Col A | Col B |\n') + tableParser.parseToken('| --- | --- |\n') + tableParser.parseToken('| cell1 | cell2 |\n') + tableParser.stopParsing() - // If tree-sitter detected a correction, we should see backtrackOffset - // and the corrected chunks should contain the bold span - if (backtrackChunks.length > 0) { - const backtrackChunk = backtrackChunks[0] - expect(backtrackChunk.backtrackOffset).toBeDefined() - expect(typeof backtrackChunk.backtrackOffset).toBe('number') - expect(backtrackChunk.backtrackOffset!).toBeGreaterThanOrEqual(0) - - // After the backtrack, the corrected chunks should have bold text - // Find all chunks emitted at or after the backtrack offset - const correctedChunks = parsedChunks.filter( - c => c.offset >= backtrackChunk.backtrackOffset! - ) - expect(correctedChunks.length).toBeGreaterThan(0) - } + const backtrackChunks = tableChunks.filter(c => c.backtrackOffset !== undefined) + expect(backtrackChunks.length).toBeGreaterThan(0) + + const firstBacktrack = backtrackChunks[0] + expect(firstBacktrack.backtrackOffset).toBeDefined() + expect(typeof firstBacktrack.backtrackOffset).toBe('number') + expect(firstBacktrack.backtrackOffset!).toBeGreaterThanOrEqual(0) + + MarkdownStreamParser.removeInstance(tableId) }) it('should re-emit corrected segments from backtrack point', async () => { - // Stream italic that starts ambiguously - parser.parseToken('Text *italic') - parser.parseToken('* more\n') - parser.stopParsing() + const tableId = 'test-table-reemit' + const tableParser = await MarkdownStreamParser.getInstance(tableId) + const tableChunks: Chunk[] = [] - // Check the full reconstructed text contains expected content - const fullText = parsedChunks.map(c => c.text).join('') - expect(fullText).toContain('Text') - expect(fullText).toContain('more') + tableParser.subscribeToTokenParse((chunk) => { + if (chunk.status === 'STREAMING' && chunk.chunk) { + tableChunks.push(chunk.chunk) + } + }) + + tableParser.startParsing() + tableParser.parseToken('| Name | Age |\n') + tableParser.parseToken('| --- | --- |\n') + tableParser.parseToken('| Alice | 30 |\n') + tableParser.stopParsing() + + // After backtracking, re-generated chunks should be emitted + const backtrackChunks = tableChunks.filter(c => c.backtrackOffset !== undefined) + expect(backtrackChunks.length).toBeGreaterThan(0) + + // Reconstruct text using only the LATEST chunks (simulating a consumer + // that discards old content when backtrackOffset is seen) + let activeChunks = [...tableChunks] + for (const btChunk of backtrackChunks) { + const btOffset = btChunk.backtrackOffset! + const btIdx = activeChunks.indexOf(btChunk) + // Discard everything from btOffset onwards, keep only chunks before + activeChunks = [ + ...activeChunks.filter((c, idx) => idx < btIdx && c.offset + c.length <= btOffset), + ...activeChunks.slice(btIdx) + ] + } + + const fullText = activeChunks.map(c => c.text).join('') + expect(fullText).toContain('Name') + expect(fullText).toContain('Age') + expect(fullText).toContain('Alice') + + MarkdownStreamParser.removeInstance(tableId) }) it('should respect windowSize configuration', async () => { - // Create a parser with windowSize constraint const windowInstanceId = 'test-window-size' const windowParser = await MarkdownStreamParser.getInstance(windowInstanceId, { windowSize: 5, @@ -711,23 +744,22 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { }) windowParser.startParsing() - - // Stream content that might trigger backtracking - windowParser.parseToken('Hello **bold text here') - windowParser.parseToken('** end\n') + // Table reclassification triggers backtracking + windowParser.parseToken('| Header1 | Header2 |\n') + windowParser.parseToken('| --- | --- |\n') windowParser.stopParsing() - // If backtracking occurred, the offset should be clamped const backtrackChunks = windowChunks.filter(c => c.backtrackOffset !== undefined) if (backtrackChunks.length > 0) { - const lastEmitted = Math.max(...windowChunks - .filter(c => c.backtrackOffset === undefined) - .map(c => c.offset + c.length)) - const backtrackChunk = backtrackChunks[0] - - // The backtrack distance should not exceed windowSize - if (lastEmitted > 0) { - const distance = lastEmitted - backtrackChunk.backtrackOffset! + // Find the furthest emit point before the backtrack chunk + const btChunk = backtrackChunks[0] + const btIdx = windowChunks.indexOf(btChunk) + const priorChunks = windowChunks.slice(0, btIdx).filter(c => c.backtrackOffset === undefined) + + if (priorChunks.length > 0) { + const lastEmitted = Math.max(...priorChunks.map(c => c.offset + c.length)) + const distance = lastEmitted - btChunk.backtrackOffset! + // The backtrack distance should not exceed windowSize expect(distance).toBeLessThanOrEqual(5) } } diff --git a/src/tree-sitter-markdown-stream-parser.ts b/src/tree-sitter-markdown-stream-parser.ts index ab7cfca..b6caf97 100644 --- a/src/tree-sitter-markdown-stream-parser.ts +++ b/src/tree-sitter-markdown-stream-parser.ts @@ -323,7 +323,6 @@ export class MarkdownStreamParser { const changeStartUtf16 = range.startIndex if (changeStartUtf16 < this.generatorState.lastEmittedOffset) { - console.log('🔴 BACKTRACK TRIGGERED: changeStart', changeStartUtf16, '< lastEmitted', this.generatorState.lastEmittedOffset) // Check windowSize constraint const backtrackDistance = this.generatorState.lastEmittedOffset - changeStartUtf16 @@ -342,7 +341,7 @@ export class MarkdownStreamParser { // backtrackOffset is already a UTF-16 character offset // Reset generator state to the backtrack point - const resetState: SegmentGeneratorState = { + let state: SegmentGeneratorState = { totalUtf16Offset: backtrackOffset, lastEmittedOffset: backtrackOffset, openSpans: [], @@ -351,30 +350,43 @@ export class MarkdownStreamParser { accumulatedContent: this.content.substring(0, backtrackOffset), } - // Re-generate all segments from backtrack point through end of content - const result = generateSegments(backtrackOffset, this.content.length, { - content: this.content, - currentTree: this.currentTree, - inlineParser: this.inlineParser, - state: resetState, - config: this.config, - }) + // Re-generate all segments from backtrack point through end of content. + // generateSegments only processes one node per call, so we must loop + // through word-sized sub-ranges, matching how TokensStreamBuffer drives + // the parser in the normal path. + const allBacktrackSegments: StreamingChunk[] = [] + const contentToReprocess = this.content.substring(backtrackOffset) + const wordRanges = this.splitIntoWordRanges(contentToReprocess) + + for (const range of wordRanges) { + const fromIdx = backtrackOffset + range.start + const toIdx = backtrackOffset + range.end + const result = generateSegments(fromIdx, toIdx, { + content: this.content, + currentTree: this.currentTree, + inlineParser: this.inlineParser, + state, + config: this.config, + }) + state = result.state + allBacktrackSegments.push(...result.segments) + } // Update state - this.generatorState = result.state + this.generatorState = state // Set backtrackOffset on the first re-generated chunk - if (result.segments.length > 0) { - const firstSeg = result.segments[0] + if (allBacktrackSegments.length > 0) { + const firstSeg = allBacktrackSegments[0] if (firstSeg.status === 'STREAMING' && firstSeg.chunk) { firstSeg.chunk.backtrackOffset = backtrackOffset } } // Store all segments for debugging - this.allSegments.push(...result.segments) + this.allSegments.push(...allBacktrackSegments) - return result.segments + return allBacktrackSegments } // Normal path: no backtracking, generate segments for new content only @@ -396,6 +408,47 @@ export class MarkdownStreamParser { } + // Split content into word-sized ranges matching TokensStreamBuffer's logic. + // Each range is { start, end } relative to the input string. + private splitIntoWordRanges(text: string): Array<{ start: number; end: number }> { + const ranges: Array<{ start: number; end: number }> = [] + let i = 0 + + while (i < text.length) { + const segmentStart = i + + // Skip leading whitespace + while (i < text.length && this.isWhitespace(text[i])) { + i++ + } + + // If only whitespace remains, include it as final range + if (i >= text.length) { + if (i > segmentStart) { + ranges.push({ start: segmentStart, end: i }) + } + break + } + + // Consume non-whitespace (the word) + while (i < text.length && !this.isWhitespace(text[i])) { + i++ + } + + // Consume trailing whitespace + while (i < text.length && this.isWhitespace(text[i])) { + i++ + } + + ranges.push({ start: segmentStart, end: i }) + } + + return ranges + } + + private isWhitespace(char: string): boolean { + return char === ' ' || char === '\t' || char === '\n' || char === '\r' + } // Get the current accumulated content. getCurrentContent(): string { From 8cf3307d509912cb2ea4705fddd036a919b0a019 Mon Sep 17 00:00:00 2001 From: Dmitry Bondarenko Date: Sun, 21 Jun 2026 12:41:54 +0600 Subject: [PATCH 24/32] Tree-sitter error recovery, tsup to tsdown migration --- Dockerfile | 5 +- demo/svelte-demo/src/routes/+page.svelte | 2 +- demo/utils/char-streamer-tree-sitter.ts | 14 +- demo/utils/char-streamer.ts | 78 ++- error-recovery-plan.md | 72 +++ ...r-recovery-scaling-and-window-semantics.md | 154 ++++++ package.json | 15 +- src/markdown-stream-parser.test.ts | 174 ++----- src/markdown-stream-parser.ts | 112 +---- ...tree-sitter-markdown-stream-parser.test.ts | 346 ++++++++++++- src/tree-sitter-markdown-stream-parser.ts | 167 +++++-- src/tree-sitter/block-detection.ts | 12 +- src/tree-sitter/content-extraction.ts | 8 +- src/tree-sitter/inline-detection.ts | 26 +- src/tree-sitter/inline-extractors.ts | 26 +- src/tree-sitter/segment-builder.ts | 41 ++ src/tree-sitter/segment-generator.ts | 453 ++++++++++-------- src/tree-sitter/tree-navigation.ts | 12 +- src/tree-sitter/types.ts | 31 +- tsdown.config.ts | 12 + tsup.config.ts | 26 - 21 files changed, 1166 insertions(+), 620 deletions(-) create mode 100644 error-recovery-plan.md create mode 100644 error-recovery-scaling-and-window-semantics.md create mode 100644 tsdown.config.ts delete mode 100644 tsup.config.ts diff --git a/Dockerfile b/Dockerfile index 2517599..8b848b3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,8 +9,9 @@ FROM node:${NODE_VERSION}-alpine # cargo is needed to install tree-sitter-cli from source because npm install fails due to network/SSL issues with GitHub releases in this environment RUN apk add --update --no-cache curl python3 make g++ cargo -# Install pnpm globally -RUN npm install -g pnpm +# Match the packageManager version declared in package.json. Newer pnpm versions +# no longer read the demo's pnpm.onlyBuiltDependencies setting. +RUN npm install -g pnpm@9.15.0 # Install tree-sitter-cli from source via cargo (bypassing GitHub releases download issue) # Pin version to 0.25.0 to avoid dependency on libloading 0.9.0 which requires newer Rust than available in node:23-alpine diff --git a/demo/svelte-demo/src/routes/+page.svelte b/demo/svelte-demo/src/routes/+page.svelte index 66702d4..6dbcbcf 100644 --- a/demo/svelte-demo/src/routes/+page.svelte +++ b/demo/svelte-demo/src/routes/+page.svelte @@ -7,7 +7,7 @@ type OpenSpan, type ClosedSpan, type SpanType, - } from "../../../../src/tree-sitter-markdown-stream-parser.js"; + } from "../../../../src/markdown-stream-parser.js"; type ExampleFile = { base: string; json: string; txt: string }; diff --git a/demo/utils/char-streamer-tree-sitter.ts b/demo/utils/char-streamer-tree-sitter.ts index 02eb5e2..b608e76 100644 --- a/demo/utils/char-streamer-tree-sitter.ts +++ b/demo/utils/char-streamer-tree-sitter.ts @@ -1,9 +1,5 @@ import fs from 'fs' -import Parser from 'tree-sitter' -import Markdown from '@tree-sitter-grammars/tree-sitter-markdown'; -import { MarkdownStreamParser, type StreamingChunk } from '../../src/tree-sitter-markdown-stream-parser.ts' - -import { log, info, infoStr, warn, err } from './debug-tools.ts' +import { MarkdownStreamParser, type StreamingChunk } from '../../src/markdown-stream-parser.ts' // Parse CLI arguments const args = process.argv.slice(2); @@ -31,9 +27,6 @@ if (!filePath) { const sourceFile = `/usr/src/service/demo/llm-streams-examples/${filePath}`; -// Get parser instance with unique ID -const markdownStreamParser = MarkdownStreamParser.getInstance(filePath); - type JSONChunk = string | object; async function* streamJSONinChunks( @@ -53,6 +46,9 @@ async function* streamJSONinChunks( } (async () => { + MarkdownStreamParser.configureWasmPath('/usr/src/service/demo/svelte-demo/static/tree-sitter-markdown.wasm'); + const markdownStreamParser = await MarkdownStreamParser.getInstance(filePath); + console.log('\n'); console.log(`Loading file: ${sourceFile}`); console.log(`Delay between chunks: ${DELAY}ms`); @@ -76,7 +72,7 @@ async function* streamJSONinChunks( console.log('=== Stream Started ===\n'); } else if (chunk.status === 'END_STREAM') { console.log('\n=== Stream Ended ==='); - } else if (chunk.status === 'STREAMING' && chunk.segment) { + } else if (chunk.status === 'STREAMING' && chunk.chunk) { console.log(`Segment:`, JSON.stringify(chunk, null, 2)); } }); diff --git a/demo/utils/char-streamer.ts b/demo/utils/char-streamer.ts index a242afe..8ad6a3d 100644 --- a/demo/utils/char-streamer.ts +++ b/demo/utils/char-streamer.ts @@ -1,75 +1,65 @@ import fs from 'fs' +import { MarkdownStreamParser, type StreamingChunk } from '../../src/markdown-stream-parser.ts' -// import { log, info, infoStr, warn, err } from './debug-tools.ts' - -import { MarkdownStreamParser } from '../../src/markdown-stream-parser.ts' - -// Parse CLI arguments -const args = process.argv.slice(2); -let DELAY = 0; -let filePath = ''; +const args = process.argv.slice(2) +let DELAY = 0 +let filePath = '' for (const arg of args) { if (arg.startsWith('--interval=')) { - const val = parseInt(arg.split('=')[1], 10); - if (!isNaN(val)) DELAY = val; + const val = parseInt(arg.split('=')[1], 10) + if (!isNaN(val)) DELAY = val } if (arg.startsWith('--file=')) { - filePath = arg.split('=')[1]; + filePath = arg.split('=')[1] } } if (!filePath) { - throw new Error('Missing required argument: --file='); + throw new Error('Missing required argument: --file=') } -const sourceFile = `/usr/src/service/${filePath}`; +const sourceFile = `/usr/src/service/${filePath}` -const markdownStreamParser = MarkdownStreamParser.getInstance(filePath) - -type JSONChunk = string | object; // Adjust as needed for your JSON structure +type JSONChunk = string | object async function* streamJSONinChunks(jsonArray: JSONChunk[]): AsyncGenerator { - const delay = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); + const delay = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)) - // Iterate over each object in the json array for (const item of jsonArray) { if (item !== '') { - yield item; - await delay(DELAY); + yield item + await delay(DELAY) } } } - ;(async () => { - console.log('\n') - - const jsonContent: string = fs.readFileSync(sourceFile, { encoding: 'utf-8' }); - const parsedJson: JSONChunk[] = JSON.parse(jsonContent); - const textStream = streamJSONinChunks(parsedJson); - - markdownStreamParser.startParsing() // Parser has to be started before the stream is created - - for await (const chunk of textStream) { - markdownStreamParser.parseToken(chunk); - } + MarkdownStreamParser.configureWasmPath('/usr/src/service/demo/svelte-demo/static/tree-sitter-markdown.wasm') + const markdownStreamParser = await MarkdownStreamParser.getInstance(filePath) - markdownStreamParser.stopParsing() // At the end of the stream, it flushes any remaining content - -})() + const unsubscribe = markdownStreamParser.subscribeToTokenParse((parsed: StreamingChunk, unsubscribe) => { + console.log('parsed', parsed) + if (parsed.status === 'END_STREAM') { + unsubscribe() + } + }) -type UnsubscribeFn = () => void; + try { + const jsonContent: string = fs.readFileSync(sourceFile, { encoding: 'utf-8' }) + const parsedJson: JSONChunk[] = JSON.parse(jsonContent) -markdownStreamParser.subscribeToTokenParse( - (parsedSegment: any, unsubscribe: UnsubscribeFn) => { - console.log('parsedSegment', parsedSegment) // Happy little parsed segment + markdownStreamParser.startParsing() - // At the end of the stream, unsubscribe from the parser service - if (parsedSegment.status === 'END_STREAM') { - unsubscribe() - MarkdownStreamParser.removeInstance(filePath) + for await (const chunk of streamJSONinChunks(parsedJson)) { + const chunkStr = typeof chunk === 'string' ? chunk : JSON.stringify(chunk) + markdownStreamParser.parseToken(chunkStr) } + + markdownStreamParser.stopParsing() + } finally { + unsubscribe() + MarkdownStreamParser.removeInstance(filePath) } -) +})() diff --git a/error-recovery-plan.md b/error-recovery-plan.md new file mode 100644 index 0000000..cfbbbfc --- /dev/null +++ b/error-recovery-plan.md @@ -0,0 +1,72 @@ +# Tree-Sitter Recovery Implementation Plan + +## Summary + +- [x] Migrate the public package API to the tree-sitter parser; README already documents the async tree-sitter API. +- [x] Make all public offsets rendered-output UTF-16 offsets: `chunk.offset`, `chunk.length`, `OpenSpan.openOffset`, `ClosedSpan.offset`, `ClosedSpan.length`, and `backtrackOffset`. +- [x] Use tree-sitter recovery signals fully: changed ranges plus explicit `ERROR`/missing nodes from block and inline trees. +- [x] Preserve the current Docker test baseline and add focused recovery/build coverage. + +## Package Entrypoint + +- [x] Replace `src/markdown-stream-parser.ts` with a public re-export or wrapper around `src/tree-sitter-markdown-stream-parser.ts`. +- [x] Update legacy entrypoint tests to the async tree-sitter API. +- [x] Keep the build entry at `src/markdown-stream-parser.ts` so package exports stay stable. +- [x] Ensure runtime tree-sitter dependencies are in `dependencies`, not only `devDependencies`. +- [x] Add a package build smoke test that imports from `build/markdown-stream-parser.js`. + +## Offset Model + +- [x] Refactor generator state to track source offsets and rendered offsets separately. +- [x] Advance public rendered offsets only by emitted text, not stripped markdown markers or suppressed syntax. +- [x] Convert `OpenSpan` and `ClosedSpan` positions/lengths to rendered offsets. +- [x] Keep `chunk.original` as raw source when `includeRawStreamedToken` is enabled. +- [x] Apply `windowSize` to rendered offsets, because consumers discard rendered output. + +## Recovery Flow + +- [x] Find earliest affected source offset from `getChangedRanges()`. +- [x] Include explicit tree-sitter `ERROR` and missing nodes in affected-range detection. +- [x] Store generator checkpoints at stable emitted boundaries: source offset, rendered offset, open spans, pending inline state, accumulated content, and block state. +- [x] Rebuild recovery output from the nearest stable checkpoint before the affected source offset. +- [x] Emit `backtrackOffset` on the first correction chunk using rendered-offset coordinates. +- [x] Emit a zero-length correction chunk when recovery deletes stale output without replacement. +- [x] Prevent stale or duplicated chunks after repeated recovery events. + +## Tests + +- [x] Add exact rendered-offset tests for headings, inline styles, and code blocks after marker stripping. +- [x] Add table reclassification recovery after preceding heading/paragraph markdown. +- [x] Add code fence split-across-chunks coverage. +- [x] Add code fence reclassification recovery after emitted paragraph text. +- [x] Add unclosed code fence at stream end coverage. +- [x] Add code fence rendered-offset coverage after preceding markdown syntax. +- [ ] Add inline delimiter recovery with open/closing spans. +- [ ] Add recovery test for stale output deletion without replacement. +- [x] Add `windowSize` recovery test using rendered distance. +- [x] Add `includeRawStreamedToken` recovery test. +- [x] Strengthen consumer simulation tests to apply `backtrackOffset` to a rendered string buffer. + +## Integration Cleanup + +- [x] Update demo/debug utilities that import `tree-sitter-markdown-stream-parser` directly when package entrypoint migration is complete. +- [x] Update any old `{ status: "STREAMING", segment }` assumptions to the tree-sitter `{ status: "STREAMING", chunk }` shape. +- [x] Fix TypeScript build issues around `web-tree-sitter` types and nullable parse results. + +## Verification + +- [x] Run full tests in a one-off Docker container mounted to this workspace. +- [x] Run `pnpm run build` or the Docker equivalent. +- [x] Document any remaining skipped tests or known limitations. + +## Remaining Follow-Up + +- [x] Add explicit code fence recovery coverage: + - [x] Split fence across chunks strips markers and emits `code_block`. + - [x] Reclassification after emitted paragraph text applies corrected output. + - [x] Unclosed fence at stream end emits code content without stuck buffering. + - [x] Fence after stripped markdown syntax uses rendered offsets. + - [x] Remove `handleCodeFenceInParagraph` after tests prove tree-sitter parsing covers these cases. +- [ ] Add inline delimiter recovery coverage that asserts open/closing spans through replay. +- [ ] Add a direct stale-output deletion recovery case that exercises the zero-length correction chunk path. +- [ ] Existing skipped feature tests remain skipped for blockquotes and fuller table support; they are outside this recovery pass. diff --git a/error-recovery-scaling-and-window-semantics.md b/error-recovery-scaling-and-window-semantics.md new file mode 100644 index 0000000..bb3e181 --- /dev/null +++ b/error-recovery-scaling-and-window-semantics.md @@ -0,0 +1,154 @@ +# Error Recovery: Scaling and Window Semantics + +This note describes three engineering improvements needed to keep Markdown error recovery correct and efficient for long streams: + +1. Checkpoint pruning and indexing +2. Eliminating full-tree scans +3. Defining strict `windowSize` behavior + +## Checkpoint pruning and indexing + +A checkpoint stores enough parser state to restart segment generation from a previous source position. + +The current implementation adds checkpoints frequently and copies the complete checkpoint array whenever it adds one: + +```ts +const checkpoints = [...state.checkpoints, checkpoint] +``` + +For `n` emitted segments, repeated array copying approaches O(n²). Checkpoint lookup also scans the array linearly. + +### Recommended design + +- Create checkpoints only at stable boundaries: + - End of paragraph + - End of heading + - Complete list item + - Code fence boundary + - A configurable source-character interval +- Retain only checkpoints inside the supported recovery window. +- Retain one older baseline checkpoint when unlimited recovery is required. +- Index checkpoints by source and rendered offsets. +- Use binary search to select the latest valid checkpoint. +- Keep the checkpoint collection in mutable parser-internal storage instead of copying it into every generator state. + +```text +source: 0───100───200───300───400 +checkpoints: C1 C2 C3 C4 + ^ + changed region +``` + +Recovery should select `C2` using an indexed lookup and replay from that checkpoint. + +This changes checkpoint lookup from O(n) to O(log n), avoids repeated history copying, and prevents unbounded checkpoint growth. + +## Eliminating full-tree scans + +After each streamed token, the current parser recursively scans the complete syntax tree to find the earliest error. + +For a growing document, the cumulative work can become quadratic: + +```text +chunk 1: scan 100 nodes +chunk 2: scan 200 nodes +chunk 3: scan 300 nodes +... +``` + +Tree-sitter already reports changed ranges. Error inspection should normally be limited to: + +- Changed ranges +- Their containing block nodes +- A small surrounding recovery region +- Previously tracked unresolved errors that overlap the new changes + +The processing flow should be: + +```text +append token + ↓ +Tree-sitter returns changed ranges + ↓ +inspect affected blocks or subtrees + ↓ +recover only if previously emitted output was affected +``` + +For example, when a delimiter row reclassifies a preceding line as a table header, the parser should inspect the affected table subtree instead of rescanning unrelated headings and paragraphs. + +Unresolved errors can be tracked explicitly: + +```ts +type PendingError = { + sourceStart: number + sourceEnd: number + containingBlockStart: number +} +``` + +When new input arrives, the parser revisits only pending errors that overlap or are structurally related to the changed ranges. + +## Strict `windowSize` behavior + +`windowSize` represents how far back a consumer can revise already-rendered output. + +```text +rendered output length: 1,000 +windowSize: 100 +earliest legal backtrack: 900 +``` + +If tree-sitter discovers that output beginning at offset 700 is incorrect, the parser cannot start recovery at 900 and claim that the correction is complete. The structural change began before the recoverable window. + +### Required invariant + +The selected checkpoint must never occur after the earliest affected source position: + +```text +selected checkpoint source offset <= earliest affected source offset +``` + +Without this invariant, replay can start after the damaged region and produce output that appears valid but is internally inconsistent. + +### Possible overflow policies + +#### Strict failure + +Emit an explicit event when the required recovery exceeds the consumer's supported window: + +```ts +{ + status: 'RECOVERY_LIMIT_EXCEEDED', + requiredOffset: 700, + earliestAllowedOffset: 900 +} +``` + +The consumer can then restart parsing or request a complete replacement. + +#### Full snapshot replacement + +Emit the complete corrected document or affected block instead of attempting a partial backtrack. + +#### Block-level recovery + +Treat `windowSize` as a target while permitting recovery to extend to the beginning of the affected Markdown block. This is practical for Markdown, but the behavior must be part of the public contract. + +### Recommended contract + +1. Determine the earliest affected source position. +2. Select the latest checkpoint at or before that position. +3. Translate that checkpoint to its rendered offset. +4. If the rendered offset violates `windowSize`, emit an explicit recovery-limit event. +5. Optionally apply a configured fallback, such as full snapshot replacement. +6. Never silently select a later checkpoint to satisfy the window. + +## Suggested implementation order + +1. Define and test the `windowSize` overflow contract. +2. Move checkpoint storage out of copied generator state. +3. Add checkpoint pruning and binary-search lookup. +4. Restrict error detection to changed subtrees and tracked pending errors. +5. Add long-stream benchmarks and recovery correctness tests. + diff --git a/package.json b/package.json index fe9e8ed..9aa98bf 100644 --- a/package.json +++ b/package.json @@ -9,9 +9,9 @@ "types": "build/markdown-stream-parser.d.ts", "exports": { ".": { + "types": "./build/markdown-stream-parser.d.ts", "import": "./build/markdown-stream-parser.js", - "require": "./build/markdown-stream-parser.cjs", - "types": "./build/markdown-stream-parser.d.ts" + "require": "./build/markdown-stream-parser.cjs" } }, "files": [ @@ -23,7 +23,7 @@ "README.md" ], "scripts": { - "build": "tsup src/markdown-stream-parser.ts --dts --format esm,cjs --minify --out-dir build --no-sourcemap", + "build": "tsdown", "debug-parser-tree-sitter": "node ./demo/utils/char-streamer-tree-sitter.ts", "debug-parser": "node ./demo/utils/char-streamer.ts", "split-sample-into-chunks": "node ./demo/utils/split-sample-into-chunks.ts", @@ -34,14 +34,15 @@ "devDependencies": { "chalk": "*", "typescript": "*", - "tsup": "^8.0.0", + "tsdown": "^0.22.3", "semver": "^7.5.4", - "web-tree-sitter": "*", "@tree-sitter-grammars/tree-sitter-markdown": "*", "vitest": "^2.0.0", "@vitest/ui": "^2.0.0" }, - "dependencies": {}, + "dependencies": { + "web-tree-sitter": "*" + }, "repository": { "type": "git", "url": "https://github.com/lixpi/markdown-stream-parser" @@ -58,4 +59,4 @@ "publishConfig": { "access": "public" } -} \ No newline at end of file +} diff --git a/src/markdown-stream-parser.test.ts b/src/markdown-stream-parser.test.ts index b2c2ea2..a9c5c1d 100644 --- a/src/markdown-stream-parser.test.ts +++ b/src/markdown-stream-parser.test.ts @@ -1,147 +1,61 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { MarkdownStreamParser, type Chunk } from './markdown-stream-parser' +import path from 'path' +import { fileURLToPath } from 'url' -import { describe, it, expect, vi, beforeEach } from 'vitest' -import { MarkdownStreamParser } from './markdown-stream-parser' +const __filename = fileURLToPath(import.meta.url) +const __dirname = path.dirname(__filename) +const wasmDir = path.join(__dirname, '../demo/svelte-demo/static') -// Mock child components -vi.mock('./tokens-stream-buffer.ts', () => { - const TokensStreamBuffer = vi.fn() - TokensStreamBuffer.prototype.receiveChunk = vi.fn() - TokensStreamBuffer.prototype.flushBuffer = vi.fn() - TokensStreamBuffer.prototype.subscribeToSegmentCompletion = vi.fn((cb: (segment: string) => void) => { - TokensStreamBuffer.prototype.triggerSegmentCompletion = (segment: string) => cb(segment) - return vi.fn() // return unsubscribe function - }) - return { default: TokensStreamBuffer } -}) - -vi.mock('./state-machine/markdown-state-machine.ts', () => { - const MarkdownStreamParserStateMachine = vi.fn() - MarkdownStreamParserStateMachine.prototype.parseSegment = vi.fn() - MarkdownStreamParserStateMachine.prototype.resetParser = vi.fn() - MarkdownStreamParserStateMachine.prototype.subscribeToParsedSegment = vi.fn((cb: (segment: any) => void) => { - MarkdownStreamParserStateMachine.prototype.triggerParsedSegment = (segment: any) => cb(segment) - return vi.fn() // return unsubscribe function - }) - return { default: MarkdownStreamParserStateMachine } -}) - -describe('MarkdownStreamParser', () => { +describe('MarkdownStreamParser public entrypoint', () => { + const instanceId = 'test-public-entrypoint' let parser: MarkdownStreamParser - const instanceId = 'test-instance' - - beforeEach(() => { - // Clear all instances and mocks before each test - vi.clearAllMocks() - MarkdownStreamParser.removeInstance(instanceId) - parser = MarkdownStreamParser.getInstance(instanceId) - }) + let parsedChunks: Chunk[] - it('should be a singleton for a given instanceId', () => { - const parser2 = MarkdownStreamParser.getInstance(instanceId) - expect(parser).toBe(parser2) - }) - - it('should create different instances for different instanceIds', () => { - const parser2 = MarkdownStreamParser.getInstance('another-instance') - expect(parser).not.toBe(parser2) - MarkdownStreamParser.removeInstance('another-instance') + beforeEach(async () => { + parsedChunks = [] + MarkdownStreamParser.configureWasmPath(path.join(wasmDir, 'tree-sitter-markdown.wasm')) + parser = await MarkdownStreamParser.getInstance(instanceId) + parser.subscribeToTokenParse((chunk) => { + if (chunk.status === 'STREAMING') { + parsedChunks.push(chunk.chunk) + } + }) }) - it('should remove an instance', () => { + afterEach(() => { + parser.stopParsing() MarkdownStreamParser.removeInstance(instanceId) - const parser2 = MarkdownStreamParser.getInstance(instanceId) - expect(parser).not.toBe(parser2) }) - describe('Parsing Lifecycle', () => { - it('should not be parsing initially', () => { - expect(parser.parsing).toBe(false) - }) - - it('should start parsing and subscribe to dependencies', () => { - const listener = vi.fn() - parser.subscribeToTokenParse(listener) - parser.startParsing() - - expect(parser.parsing).toBe(true) - expect(parser.tokensStreamProcessor.subscribeToSegmentCompletion).toHaveBeenCalled() - expect(parser.markdownStreamParser.subscribeToParsedSegment).toHaveBeenCalled() - expect(listener).toHaveBeenCalledWith({ status: 'START_STREAM' }, expect.any(Function)) - }) - - it('should not start parsing if already parsing', () => { - parser.startParsing() - const listener = vi.fn() - parser.subscribeToTokenParse(listener) - parser.startParsing() // second call - expect(listener).not.toHaveBeenCalledWith({ status: 'START_STREAM' }, expect.any(Function)) - }) - - it('should stop parsing, flush, reset, and unsubscribe', () => { - const listener = vi.fn() - parser.subscribeToTokenParse(listener) - parser.startParsing() - parser.stopParsing() - - expect(parser.parsing).toBe(false) - expect(parser.tokensStreamProcessor.flushBuffer).toHaveBeenCalled() - expect(parser.markdownStreamParser.resetParser).toHaveBeenCalled() - expect(listener).toHaveBeenCalledWith({ status: 'END_STREAM' }, expect.any(Function)) - }) - - it('should throw error if parseToken is called before startParsing', () => { - const error = parser.parseToken('chunk') - expect(error).toBeInstanceOf(Error) - expect(error?.message).toBe('Parser is not started.') - }) + it('returns the same async parser instance for a given instanceId', async () => { + const parser2 = await MarkdownStreamParser.getInstance(instanceId) + expect(parser2).toBe(parser) }) - describe('Data Flow', () => { - it('should pass chunks to TokensStreamBuffer', () => { - parser.startParsing() - parser.parseToken('some chunk') - expect(parser.tokensStreamProcessor.receiveChunk).toHaveBeenCalledWith('some chunk') - }) - - it('should pass segments from buffer to state machine', () => { - parser.startParsing() - // @ts-ignore - triggerSegmentCompletion is a mock-specific helper - parser.tokensStreamProcessor.triggerSegmentCompletion('segment ') - expect(parser.markdownStreamParser.parseSegment).toHaveBeenCalledWith('segment ') - }) - - it('should notify listeners with parsed segments from state machine', () => { - const listener = vi.fn() - parser.subscribeToTokenParse(listener) - parser.startParsing() - - const parsedSegment = { type: 'paragraph', content: 'hello' } - // @ts-ignore - triggerParsedSegment is a mock-specific helper - parser.markdownStreamParser.triggerParsedSegment(parsedSegment) - - expect(listener).toHaveBeenCalledWith({ status: 'STREAMING', segment: parsedSegment }, expect.any(Function)) - }) + it('creates different parser instances for different instanceIds', async () => { + const other = await MarkdownStreamParser.getInstance('test-public-entrypoint-other') + expect(other).not.toBe(parser) + MarkdownStreamParser.removeInstance('test-public-entrypoint-other') }) - describe('Subscription', () => { - it('should subscribe and notify listeners', () => { - const listener = vi.fn() - parser.subscribeToTokenParse(listener) - parser.startParsing() - parser.stopParsing() - - expect(listener).toHaveBeenCalledWith({ status: 'START_STREAM' }, expect.any(Function)) - expect(listener).toHaveBeenCalledWith({ status: 'END_STREAM' }, expect.any(Function)) - }) - - it('should allow unsubscribing', () => { - const listener = vi.fn() - const unsubscribe = parser.subscribeToTokenParse(listener) + it('parses through the tree-sitter chunk API', () => { + parser.startParsing() + parser.parseToken('## ') + parser.parseToken('Hello **world**\n') + parser.stopParsing() - unsubscribe() + const text = parsedChunks.map(chunk => chunk.text).join('') + expect(text).toContain('Hello world') + expect(text).not.toContain('##') + expect(text).not.toContain('**') + expect(parsedChunks.some(chunk => chunk.block.type === 'heading')).toBe(true) + expect(parsedChunks.some(chunk => chunk.contained.some(span => span.type === 'bold'))).toBe(true) + }) - parser.startParsing() - expect(listener).not.toHaveBeenCalled() - }) + it('returns an error if parseToken is called before startParsing', () => { + const error = parser.parseToken('chunk') + expect(error).toBeInstanceOf(Error) + expect(error?.message).toBe('Parser is not started. Call startParsing() first.') }) }) diff --git a/src/markdown-stream-parser.ts b/src/markdown-stream-parser.ts index 6d497be..851c9db 100644 --- a/src/markdown-stream-parser.ts +++ b/src/markdown-stream-parser.ts @@ -1,99 +1,13 @@ -'use strict' - -import TokensStreamBuffer from './tokens-stream-buffer.ts' -import MarkdownStreamParserStateMachine from './state-machine/markdown-state-machine.ts' - -export class MarkdownStreamParser { - static instances = new Map() - tokensStreamProcessor: TokensStreamBuffer - markdownStreamParser: MarkdownStreamParserStateMachine - unsubscribeFromProcessor: () => void - unsubscribeFromStateMachine: () => void - parsing: boolean - tokenParseListeners: Array<(token: any) => void> - - - static getInstance(instanceId: string): MarkdownStreamParser { - if (!MarkdownStreamParser.instances.has(instanceId)) { - MarkdownStreamParser.instances.set(instanceId, new MarkdownStreamParser()) // Save the instance, ensure it is available statically - } - - return MarkdownStreamParser.instances.get(instanceId) - } - - static removeInstance(instanceId: string): void { - if (MarkdownStreamParser.instances.has(instanceId)) { - MarkdownStreamParser.instances.delete(instanceId) - } - } - - constructor() { - this.tokensStreamProcessor = new TokensStreamBuffer() - this.markdownStreamParser = new MarkdownStreamParserStateMachine() - this.unsubscribeFromProcessor = () => { } - this.unsubscribeFromStateMachine = () => { } - - this.parsing = false - this.tokenParseListeners = [] - } - - // Allow to subscribe to token parse, returns an unsubscribe function - subscribeToTokenParse(listener: (token: any, unsubscribe: () => void) => void): () => void { - const wrappedListener = (data: any) => listener(data, unsubscribe) - const unsubscribe = () => { - this.tokenParseListeners = this.tokenParseListeners.filter(l => l !== wrappedListener) - } - - this.tokenParseListeners.push(wrappedListener) - - return unsubscribe // Allow to unsubscribe from token parse - } - - // Internal method to notify all token complete listeners - notifyTokenParse(token: any): void { - this.tokenParseListeners.forEach(listener => listener(token)) - } - - // Start parsing by subscribing to the TokensStreamBuffer's word completion event - startParsing(): void { - if (this.parsing) { - return // Do not start parsing if it's already started - } - - this.notifyTokenParse({ status: 'START_STREAM' }) - - // Subscribe to receive the completed segment from TokensStreamBuffer - this.unsubscribeFromProcessor = this.tokensStreamProcessor.subscribeToSegmentCompletion((word: string) => { - // console.log('class.MarkdownStreamParser::subscribeToSegmentCompletion::word:', {word}) - this.markdownStreamParser.parseSegment(word) // Send the word to the state machine for parsing - }) - - // Subscribe to receive the parsed segment from TextStreamStateMachine - this.unsubscribeFromStateMachine = this.markdownStreamParser.subscribeToParsedSegment((parsedSegment: any) => { - this.notifyTokenParse({ status: 'STREAMING', segment: parsedSegment }) // Relay the parsed segment event - }) - - this.parsing = true - } - - // Parse individual chunk token - parseToken(chunk: string): Error | void { - if (!this.parsing) { - const error = new Error('Parser is not started.') - return error - } - - this.tokensStreamProcessor.receiveChunk(chunk) - } - - // Stop parsing by unsubscribing from the TokensStreamBuffer - stopParsing(): void { - this.tokensStreamProcessor.flushBuffer() // Flush any remaining content - this.markdownStreamParser.resetParser() // Reset the state machine - this.unsubscribeFromProcessor() // Unsubscribe from the processor - this.unsubscribeFromStateMachine() // Unsubscribe from the state machine - this.parsing = false // Mark as not parsing - this.notifyTokenParse({ status: 'END_STREAM' }) // Notify that the stream has ended - } -} - +export { MarkdownStreamParser } from './tree-sitter-markdown-stream-parser.js' + +export type { + Span, + SpanType, + OpenSpan, + ClosedSpan, + BlockType, + BlockContext, + Chunk, + StreamingChunk, + ParserConfig, +} from './tree-sitter-markdown-stream-parser.js' diff --git a/src/tree-sitter-markdown-stream-parser.test.ts b/src/tree-sitter-markdown-stream-parser.test.ts index 941d8f9..db6f5c5 100644 --- a/src/tree-sitter-markdown-stream-parser.test.ts +++ b/src/tree-sitter-markdown-stream-parser.test.ts @@ -20,6 +20,23 @@ function getSpanTypes(chunk: Chunk): SpanType[] { return allSpans.map(span => span.type) } +function getClosedSpans(chunks: Chunk[]): ClosedSpan[] { + return chunks.flatMap(c => [...c.contained, ...c.closing]) +} + +function applyBacktracks(chunks: Chunk[]): Chunk[] { + let activeChunks: Chunk[] = [] + + for (const chunk of chunks) { + if (chunk.backtrackOffset !== undefined) { + activeChunks = activeChunks.filter(c => c.offset + c.length <= chunk.backtrackOffset!) + } + activeChunks.push(chunk) + } + + return activeChunks +} + describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { let parser: MarkdownStreamParser let parsedChunks: Chunk[] = [] @@ -379,6 +396,88 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { // Should detect strikethrough style expect(strikethroughChunks.length).toBeGreaterThan(0) }) + + it('should parse inline code after stripped heading syntax', async () => { + parser.parseToken('## Use `npm install` now\n') + parser.stopParsing() + + const activeChunks = applyBacktracks(parsedChunks) + const fullText = activeChunks.map(c => c.text).join('') + const codeSpan = getClosedSpans(activeChunks).find(s => s.type === 'code') + + expect(fullText).toBe('Use npm install now') + expect(codeSpan).toBeDefined() + expect(codeSpan?.offset).toBe('Use '.length) + expect(codeSpan?.length).toBe('npm install'.length) + }) + + it('should parse inline code after stripped list syntax', async () => { + parser.parseToken('- Run `npm install` now\n') + parser.stopParsing() + + const activeChunks = applyBacktracks(parsedChunks) + const fullText = activeChunks.map(c => c.text).join('') + const codeSpan = getClosedSpans(activeChunks).find(s => s.type === 'code') + + expect(fullText).toBe('Run npm install now\n') + expect(codeSpan).toBeDefined() + expect(codeSpan?.offset).toBe('Run '.length) + expect(codeSpan?.length).toBe('npm install'.length) + }) + + it('should parse inline code after bold syntax using rendered offsets', async () => { + parser.parseToken('Use **bold** then `code` now\n') + parser.stopParsing() + + const activeChunks = applyBacktracks(parsedChunks) + const fullText = activeChunks.map(c => c.text).join('') + const boldSpan = getClosedSpans(activeChunks).find(s => s.type === 'bold') + const codeSpan = getClosedSpans(activeChunks).find(s => s.type === 'code') + + expect(fullText).toBe('Use bold then code now\n') + expect(boldSpan?.offset).toBe('Use '.length) + expect(boldSpan?.length).toBe('bold'.length) + expect(codeSpan?.offset).toBe('Use bold then '.length) + expect(codeSpan?.length).toBe('code'.length) + }) + }) + + describe('Split Inline Code', () => { + it('should buffer split inline code delimiters across chunks', async () => { + parser.parseToken('Run `npm') + parser.parseToken(' install` now\n') + parser.stopParsing() + + const activeChunks = applyBacktracks(parsedChunks) + const fullText = activeChunks.map(c => c.text).join('') + const codeSpan = getClosedSpans(activeChunks).find(s => s.type === 'code') + + expect(fullText).toBe('Run npm install now\n') + expect(codeSpan?.offset).toBe('Run '.length) + expect(codeSpan?.length).toBe('npm install'.length) + }) + + it('should buffer split inline code after stripped heading syntax', async () => { + parser.parseToken('## Run `npm') + parser.parseToken(' install` now\n') + parser.stopParsing() + + const activeChunks = applyBacktracks(parsedChunks) + const fullText = activeChunks.map(c => c.text).join('') + const codeSpan = getClosedSpans(activeChunks).find(s => s.type === 'code') + + expect(fullText).toBe('Run npm install now') + expect(codeSpan?.offset).toBe('Run '.length) + expect(codeSpan?.length).toBe('npm install'.length) + }) + + it('should flush unmatched inline backtick content at stream end', async () => { + parser.parseToken('Run `npm install now\n') + parser.stopParsing() + + const fullText = parsedChunks.map(c => c.text).join('') + expect(fullText).toBe('Run `npm install now\n') + }) }) describe('Real LLM Stream Integration', () => { @@ -577,9 +676,54 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { expect(parsedChunks[0].length).toBeGreaterThan(0) } }) + + it('should use rendered offsets after stripping heading markers', async () => { + parser.parseToken('## ') + parser.parseToken('Title\n') + parser.parseToken('Next\n') + parser.stopParsing() + + const rendered = parsedChunks.map(c => c.text).join('') + expect(rendered).toBe('TitleNext\n') + + const title = parsedChunks.find(c => c.text.includes('Title')) + const next = parsedChunks.find(c => c.text.includes('Next')) + expect(title?.offset).toBe(0) + expect(title?.length).toBe('Title'.length) + expect(next?.offset).toBe('Title'.length) + }) + + it('should use rendered offsets and lengths for inline spans', async () => { + parser.parseToken('Hello **world**\n') + parser.stopParsing() + + const rendered = parsedChunks.map(c => c.text).join('') + expect(rendered).toBe('Hello world\n') + + const boldSpan = parsedChunks.flatMap(c => c.contained).find(s => s.type === 'bold') + expect(boldSpan).toBeDefined() + expect(boldSpan?.offset).toBe('Hello '.length) + expect(boldSpan?.length).toBe('world'.length) + }) + + it('should use rendered offsets after stripping code fences', async () => { + parser.parseToken('```js\n') + parser.parseToken('code\n') + parser.parseToken('```\n') + parser.parseToken('After\n') + parser.stopParsing() + + const rendered = parsedChunks.map(c => c.text).join('') + expect(rendered).toBe('code\nAfter\n') + + const code = parsedChunks.find(c => c.block.type === 'code_block' && c.text.includes('code')) + const after = parsedChunks.find(c => c.text.includes('After')) + expect(code?.offset).toBe(0) + expect(after?.offset).toBe('code\n'.length) + }) }) describe('Table Inline Code', () => { - it.skip('should strip backticks from inline code inside tables', async () => { + it('should strip backticks from inline code inside tables', async () => { parser.parseToken('| Col | `code` |\n') parser.stopParsing() @@ -592,7 +736,7 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { expect(hasSpanType(cellChunks[0], 'code')).toBe(true) }) - it.skip('should detect table block types for complete tables', async () => { + it('should detect table block types for complete tables', async () => { parser.parseToken('| A | B |\n') parser.parseToken('|---|---|\n') parser.parseToken('| 1 | 2 |\n') @@ -618,7 +762,7 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { expect(pipeChunks.length).toBe(0) }) - it.skip('should suppress delimiter row content', async () => { + it('should suppress delimiter row content', async () => { parser.parseToken('| A |\n') parser.parseToken('|---|\n') parser.parseToken('| B |\n') @@ -630,7 +774,7 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { expect(delimiterChunks.length).toBe(0) }) - it.skip('should handle inline code in full table structure', async () => { + it('should handle inline code in full table structure', async () => { parser.parseToken('| Header |\n') parser.parseToken('|--------|\n') parser.parseToken('| `code` |\n') @@ -766,5 +910,199 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { MarkdownStreamParser.removeInstance(windowInstanceId) }) + + it('should backtrack using rendered offsets after preceding markdown syntax', async () => { + const recoveryId = 'test-rendered-backtrack-offset' + const recoveryParser = await MarkdownStreamParser.getInstance(recoveryId) + const recoveryChunks: Chunk[] = [] + + recoveryParser.subscribeToTokenParse((chunk) => { + if (chunk.status === 'STREAMING') { + recoveryChunks.push(chunk.chunk) + } + }) + + recoveryParser.startParsing() + recoveryParser.parseToken('## ') + recoveryParser.parseToken('Before\n\n') + recoveryParser.parseToken('| Name | Age |\n') + recoveryParser.parseToken('| --- | --- |\n') + recoveryParser.parseToken('| Alice | 30 |\n') + recoveryParser.stopParsing() + + const backtrack = recoveryChunks.find(c => c.backtrackOffset !== undefined) + expect(backtrack).toBeDefined() + expect(backtrack?.backtrackOffset).toBe('Before'.length) + + const activeChunks = applyBacktracks(recoveryChunks) + const fullText = activeChunks.map(c => c.text).join('') + expect(fullText).toContain('Before') + expect(fullText).toContain('Name') + expect(fullText).toContain('Age') + expect(fullText).toContain('Alice') + + MarkdownStreamParser.removeInstance(recoveryId) + }) + + it('should keep raw source originals while recovery offsets stay rendered', async () => { + const rawId = 'test-recovery-raw-original' + const rawParser = await MarkdownStreamParser.getInstance(rawId, { + includeRawStreamedToken: true, + }) + const rawChunks: Chunk[] = [] + + rawParser.subscribeToTokenParse((chunk) => { + if (chunk.status === 'STREAMING') { + rawChunks.push(chunk.chunk) + } + }) + + rawParser.startParsing() + rawParser.parseToken('Intro\n\n') + rawParser.parseToken('| Name | Age |\n') + rawParser.parseToken('| --- | --- |\n') + rawParser.stopParsing() + + const backtrack = rawChunks.find(c => c.backtrackOffset !== undefined) + expect(backtrack).toBeDefined() + expect(backtrack?.backtrackOffset).toBe('Intro\n\n'.length) + expect(rawChunks.some(c => c.original?.includes('Name'))).toBe(true) + + MarkdownStreamParser.removeInstance(rawId) + }) + }) + + describe('Code Fence Recovery', () => { + it('should strip split code fences and emit code block content', async () => { + parser.parseToken('Text before\n') + parser.parseToken('```js\n') + parser.parseToken('const x = 1\n') + parser.parseToken('```\n') + parser.stopParsing() + + const allText = parsedChunks.map(c => c.text).join('') + const codeChunks = parsedChunks.filter(c => c.block.type === 'code_block') + + expect(allText).toContain('Text before') + expect(allText).toContain('const x = 1') + expect(allText).not.toContain('```') + expect(codeChunks.map(c => c.text).join('')).toContain('const x = 1') + expect(codeChunks.some(c => c.block.language === 'js')).toBe(true) + }) + + it('should recover when an emitted paragraph is reclassified as a code fence', async () => { + const codeFenceId = 'test-code-fence-reclassification' + const codeFenceParser = await MarkdownStreamParser.getInstance(codeFenceId) + const codeFenceChunks: Chunk[] = [] + + codeFenceParser.subscribeToTokenParse((chunk) => { + if (chunk.status === 'STREAMING') { + codeFenceChunks.push(chunk.chunk) + } + }) + + codeFenceParser.startParsing() + codeFenceParser.parseToken('Text before\n\n') + codeFenceParser.parseToken('```') + codeFenceParser.parseToken('js\n') + codeFenceParser.parseToken('const x = 1\n') + codeFenceParser.stopParsing() + + const activeChunks = applyBacktracks(codeFenceChunks) + const allText = activeChunks.map(c => c.text).join('') + const codeChunks = activeChunks.filter(c => c.block.type === 'code_block') + + expect(allText).toContain('Text before') + expect(allText).toContain('const x = 1') + expect(allText).not.toContain('```') + expect(allText).not.toContain('js\nconst x') + expect(codeChunks.map(c => c.text).join('')).toContain('const x = 1') + expect(codeChunks.some(c => c.block.language === 'js')).toBe(true) + + MarkdownStreamParser.removeInstance(codeFenceId) + }) + + it('should emit unclosed fence content as code at stream end', async () => { + parser.parseToken('```python\n') + parser.parseToken('print("hi")\n') + parser.stopParsing() + + const allText = parsedChunks.map(c => c.text).join('') + const codeChunks = parsedChunks.filter(c => c.block.type === 'code_block') + + expect(allText).toBe('print("hi")\n') + expect(allText).not.toContain('```') + expect(codeChunks.map(c => c.text).join('')).toBe('print("hi")\n') + expect(codeChunks.some(c => c.block.language === 'python')).toBe(true) + }) + + it('should use rendered offsets for code fences after stripped markdown syntax', async () => { + parser.parseToken('## ') + parser.parseToken('Heading\n\n') + parser.parseToken('```ts\n') + parser.parseToken('let a = 1\n') + parser.parseToken('```\n') + parser.stopParsing() + + const heading = parsedChunks.find(c => c.block.type === 'heading') + const codeChunks = parsedChunks.filter(c => c.block.type === 'code_block') + const firstCode = codeChunks[0] + const backtrack = parsedChunks.find(c => c.backtrackOffset !== undefined) + + expect(heading?.text).toBe('Heading') + expect(codeChunks.map(c => c.text).join('')).toBe('let a = 1\n') + expect(firstCode?.offset).toBe('Heading'.length) + if (backtrack) { + expect(backtrack.backtrackOffset).toBeGreaterThanOrEqual(0) + expect(backtrack.backtrackOffset).toBeLessThanOrEqual('Heading'.length) + } + }) + + it('should strip an opening fence split across chunks', async () => { + parser.parseToken('``') + parser.parseToken('`js\n') + parser.parseToken('const x = 1\n') + parser.stopParsing() + + const activeChunks = applyBacktracks(parsedChunks) + const allText = activeChunks.map(c => c.text).join('') + const codeChunks = activeChunks.filter(c => c.block.type === 'code_block') + + expect(allText).toBe('const x = 1\n') + expect(codeChunks.map(c => c.text).join('')).toBe('const x = 1\n') + expect(codeChunks.some(c => c.block.language === 'js')).toBe(true) + }) + + it('should keep rendered offsets when a fence follows stripped formatting', async () => { + const codeFenceId = 'test-code-fence-rendered-reclassification' + const codeFenceParser = await MarkdownStreamParser.getInstance(codeFenceId) + const codeFenceChunks: Chunk[] = [] + + codeFenceParser.subscribeToTokenParse((chunk) => { + if (chunk.status === 'STREAMING') { + codeFenceChunks.push(chunk.chunk) + } + }) + + codeFenceParser.startParsing() + codeFenceParser.parseToken('- Before\n\n') + codeFenceParser.parseToken('```') + codeFenceParser.parseToken('js\n') + codeFenceParser.parseToken('const x = 1\n') + codeFenceParser.stopParsing() + + const backtrack = codeFenceChunks.find(c => c.backtrackOffset !== undefined) + const activeChunks = applyBacktracks(codeFenceChunks) + const allText = activeChunks.map(c => c.text).join('') + + expect(allText).toContain('Before') + expect(allText).toContain('const x = 1') + expect(allText).not.toContain('```') + if (backtrack) { + expect(backtrack.backtrackOffset).toBe('Before'.length) + } + + MarkdownStreamParser.removeInstance(codeFenceId) + }) }) }) diff --git a/src/tree-sitter-markdown-stream-parser.ts b/src/tree-sitter-markdown-stream-parser.ts index b6caf97..b95713b 100644 --- a/src/tree-sitter-markdown-stream-parser.ts +++ b/src/tree-sitter-markdown-stream-parser.ts @@ -1,7 +1,7 @@ -import { Parser, Language } from 'web-tree-sitter' +import { Parser, Language, type Tree, type Node } from 'web-tree-sitter' import TokensStreamBuffer from './tokens-stream-buffer.js' -import type { StreamingChunk, BlockState, ParserConfig, SegmentGeneratorState, Chunk } from './tree-sitter/types.js' -import { generateSegments, createInitialState } from './tree-sitter/segment-generator.js' +import type { StreamingChunk, ParserConfig, SegmentGeneratorState } from './tree-sitter/types.js' +import { generateSegments, createInitialState, stateFromCheckpoint } from './tree-sitter/segment-generator.js' // Re-export types for external consumers @@ -31,16 +31,16 @@ export class MarkdownStreamParser { private static instances = new Map() private static parserInitialized = false private static parserInitPromise: Promise | null = null - private static markdownLanguage: Parser.Language | null = null - private static markdownInlineLanguage: Parser.Language | null = null + private static markdownLanguage: Language | null = null + private static markdownInlineLanguage: Language | null = null private static wasmPath: string | null = null private static wasmInlinePath: string | null = null // Parser instances private parser: Parser | null = null private inlineParser: Parser | null = null - private currentTree: Parser.Tree | null = null - private previousTree: Parser.Tree | null = null + private currentTree: Tree | null = null + private previousTree: Tree | null = null // Configuration private config: ParserConfig = {} @@ -264,6 +264,34 @@ export class MarkdownStreamParser { this.tokensStreamProcessor.flushBuffer() + if (this.generatorState.pendingInlineContent) { + const text = this.generatorState.pendingInlineContent + const chunk: Chunk = { + text, + offset: this.generatorState.totalUtf16Offset, + length: text.length, + block: { type: 'paragraph' }, + opening: [], + closing: [], + contained: [], + original: this.config.includeRawStreamedToken ? text : undefined, + } + + this.generatorState = { + ...this.generatorState, + totalUtf16Offset: this.generatorState.totalUtf16Offset + text.length, + lastEmittedOffset: this.generatorState.totalUtf16Offset + text.length, + lastEmittedSourceOffset: this.generatorState.sourceOffset, + pendingInlineContent: '', + pendingInlineStartIndex: undefined, + accumulatedContent: this.generatorState.accumulatedContent + text, + } + + const streamingChunk: StreamingChunk = { status: 'STREAMING', chunk } + this.allSegments.push(streamingChunk) + this.notifyTokenParse(streamingChunk) + } + if (this.unsubscribeFromProcessor) { this.unsubscribeFromProcessor() this.unsubscribeFromProcessor = null @@ -310,60 +338,53 @@ export class MarkdownStreamParser { } // Parse the updated content - this.currentTree = this.parser.parse(this.content, this.currentTree || undefined) + const parsedTree = this.parser.parse(this.content, this.currentTree || undefined) + if (!parsedTree) { + return [] + } + this.currentTree = parsedTree + const currentTree = this.currentTree - // Detect backtracking by checking changed ranges - let backtrackOffset: number | undefined + // Detect backtracking by checking changed source ranges. + let affectedSourceOffset: number | undefined if (this.previousTree && this.currentTree) { const changedRanges = this.previousTree.getChangedRanges(this.currentTree) for (const range of changedRanges) { // range.startIndex is already a UTF-16 character offset in web-tree-sitter JS bindings - // If the change starts before what we've emitted, we need to backtrack + // If the change starts before source that produced emitted output, we need to backtrack const changeStartUtf16 = range.startIndex - if (changeStartUtf16 < this.generatorState.lastEmittedOffset) { - // Check windowSize constraint - const backtrackDistance = this.generatorState.lastEmittedOffset - changeStartUtf16 - - if (this.config.windowSize === undefined || backtrackDistance <= this.config.windowSize) { - backtrackOffset = Math.min(backtrackOffset ?? Infinity, changeStartUtf16) - } else { - const windowStart = this.generatorState.lastEmittedOffset - this.config.windowSize - backtrackOffset = Math.min(backtrackOffset ?? Infinity, windowStart) - } + if (changeStartUtf16 < this.generatorState.lastEmittedSourceOffset) { + affectedSourceOffset = Math.min(affectedSourceOffset ?? Infinity, changeStartUtf16) } } } - if (backtrackOffset !== undefined) { - // Error recovery: re-generate segments from the backtrack point - // backtrackOffset is already a UTF-16 character offset + const errorSourceOffset = this.findEarliestErrorOffset() + if (errorSourceOffset !== undefined && errorSourceOffset < this.generatorState.lastEmittedSourceOffset) { + affectedSourceOffset = Math.min(affectedSourceOffset ?? Infinity, errorSourceOffset) + } - // Reset generator state to the backtrack point - let state: SegmentGeneratorState = { - totalUtf16Offset: backtrackOffset, - lastEmittedOffset: backtrackOffset, - openSpans: [], - currentBlock: null, - pendingInlineContent: '', - accumulatedContent: this.content.substring(0, backtrackOffset), - } + if (affectedSourceOffset !== undefined) { + const checkpoint = this.findRecoveryCheckpoint(affectedSourceOffset) + let state: SegmentGeneratorState = stateFromCheckpoint(checkpoint) + let backtrackOffset = checkpoint.renderedOffset // Re-generate all segments from backtrack point through end of content. // generateSegments only processes one node per call, so we must loop // through word-sized sub-ranges, matching how TokensStreamBuffer drives // the parser in the normal path. const allBacktrackSegments: StreamingChunk[] = [] - const contentToReprocess = this.content.substring(backtrackOffset) + const contentToReprocess = this.content.substring(checkpoint.sourceOffset) const wordRanges = this.splitIntoWordRanges(contentToReprocess) for (const range of wordRanges) { - const fromIdx = backtrackOffset + range.start - const toIdx = backtrackOffset + range.end + const fromIdx = checkpoint.sourceOffset + range.start + const toIdx = checkpoint.sourceOffset + range.end const result = generateSegments(fromIdx, toIdx, { content: this.content, - currentTree: this.currentTree, + currentTree, inlineParser: this.inlineParser, state, config: this.config, @@ -381,6 +402,20 @@ export class MarkdownStreamParser { if (firstSeg.status === 'STREAMING' && firstSeg.chunk) { firstSeg.chunk.backtrackOffset = backtrackOffset } + } else if (backtrackOffset < this.generatorState.lastEmittedOffset) { + allBacktrackSegments.push({ + status: 'STREAMING', + chunk: { + text: '', + offset: backtrackOffset, + length: 0, + block: { type: 'paragraph' }, + opening: [], + closing: [], + contained: [], + backtrackOffset, + } + }) } // Store all segments for debugging @@ -392,7 +427,7 @@ export class MarkdownStreamParser { // Normal path: no backtracking, generate segments for new content only const result = generateSegments(oldLength, this.content.length, { content: this.content, - currentTree: this.currentTree, + currentTree, inlineParser: this.inlineParser, state: this.generatorState, config: this.config, @@ -407,6 +442,62 @@ export class MarkdownStreamParser { return result.segments } + private findRecoveryCheckpoint(sourceOffset: number): SegmentGeneratorState['checkpoints'][number] { + const baseCheckpoint: SegmentGeneratorState['checkpoints'][number] = { + sourceOffset: 0, + renderedOffset: 0, + lastEmittedSourceOffset: 0, + lastEmittedOffset: 0, + openSpans: [], + currentBlock: null, + pendingInlineContent: '', + accumulatedContent: '', + } + + const checkpoints = this.generatorState.checkpoints.length > 0 + ? this.generatorState.checkpoints + : [baseCheckpoint] + + let checkpoint = baseCheckpoint + for (const candidate of checkpoints) { + if (candidate.sourceOffset <= sourceOffset && candidate.sourceOffset >= checkpoint.sourceOffset) { + checkpoint = candidate + } + } + + if (this.config.windowSize !== undefined) { + const windowStart = Math.max(0, this.generatorState.lastEmittedOffset - this.config.windowSize) + if (checkpoint.renderedOffset < windowStart) { + for (const candidate of checkpoints) { + if (candidate.renderedOffset >= windowStart) { + checkpoint = candidate + break + } + } + } + } + + return checkpoint + } + + private findEarliestErrorOffset(): number | undefined { + if (!this.currentTree) return undefined + + let earliest: number | undefined + const visit = (node: Node) => { + if (node.hasError || node.isError || node.isMissing) { + earliest = Math.min(earliest ?? Infinity, node.startIndex) + } + + for (const child of node.children) { + visit(child) + } + } + + visit(this.currentTree.rootNode) + return earliest + } + // Split content into word-sized ranges matching TokensStreamBuffer's logic. // Each range is { start, end } relative to the input string. diff --git a/src/tree-sitter/block-detection.ts b/src/tree-sitter/block-detection.ts index 9b25f5d..440f6c2 100644 --- a/src/tree-sitter/block-detection.ts +++ b/src/tree-sitter/block-detection.ts @@ -1,11 +1,11 @@ -import type { Parser } from 'web-tree-sitter' +import type { Node } from 'web-tree-sitter' import { HEADER_MARKER_LEVELS, type BlockInfo, type BlockState } from './types.js' import { findBlockNode } from './tree-navigation.js' // Get the block type and properties from a tree-sitter node. // Walks up the tree to find the enclosing block structure. -export function getBlockInfo(node: Parser.SyntaxNode): BlockInfo { - let current: Parser.SyntaxNode | null = node +export function getBlockInfo(node: Node): BlockInfo { + let current: Node | null = node let foundParagraph = false let foundTableCell = false let isInHeader = false @@ -70,7 +70,7 @@ export function getBlockInfo(node: Parser.SyntaxNode): BlockInfo { } // Check if the given node represents a new block compared to the current block state. -export function isNewBlock(blockInfo: BlockInfo, node: Parser.SyntaxNode, currentBlock: BlockState | null): boolean { +export function isNewBlock(blockInfo: BlockInfo, node: Node, currentBlock: BlockState | null): boolean { const blockNode = findBlockNode(node) if (!blockNode) return false @@ -86,7 +86,7 @@ export function isNewBlock(blockInfo: BlockInfo, node: Parser.SyntaxNode, curren // Extract the heading level from an atx_heading node. // Uses tree-sitter node type lookup with fallback to character counting. -export function getHeadingLevel(node: Parser.SyntaxNode): number { +export function getHeadingLevel(node: Node): number { // Use tree-sitter node type lookup instead of regex for (const child of node.children) { const level = HEADER_MARKER_LEVELS[child.type] @@ -109,7 +109,7 @@ export function getHeadingLevel(node: Parser.SyntaxNode): number { } // Extract the language identifier from a fenced_code_block node. -export function getCodeBlockLanguage(node: Parser.SyntaxNode): string { +export function getCodeBlockLanguage(node: Node): string { // For fenced_code_block, look for info_string child if (node.type === 'fenced_code_block') { for (let i = 0; i < node.childCount; i++) { diff --git a/src/tree-sitter/content-extraction.ts b/src/tree-sitter/content-extraction.ts index 6e26606..e92f73c 100644 --- a/src/tree-sitter/content-extraction.ts +++ b/src/tree-sitter/content-extraction.ts @@ -1,10 +1,10 @@ -import type { Parser } from 'web-tree-sitter' +import type { Node, Tree } from 'web-tree-sitter' // Extract header content from a chunk, excluding marker nodes (# symbols). // Requires tree-sitter node for accurate extraction. export function getHeaderContent( content: string, - node: Parser.SyntaxNode | undefined, + node: Node | undefined, startByte: number | undefined, endByte: number | undefined ): string { @@ -44,7 +44,7 @@ export function getHeaderContent( // Requires tree-sitter node for accurate extraction. export function getCodeBlockContent( content: string, - node: Parser.SyntaxNode | undefined, + node: Node | undefined, startByte: number | undefined, endByte: number | undefined ): string { @@ -81,7 +81,7 @@ export function getCodeBlockContent( // Uses the inline parser tree to identify and skip delimiter nodes. export function getInlineContent( content: string, - inlineTree: Parser.Tree, + inlineTree: Tree, startOffset: number, endOffset: number ): string { diff --git a/src/tree-sitter/inline-detection.ts b/src/tree-sitter/inline-detection.ts index 5317624..7ba33c8 100644 --- a/src/tree-sitter/inline-detection.ts +++ b/src/tree-sitter/inline-detection.ts @@ -1,8 +1,8 @@ -import type { Parser } from 'web-tree-sitter' +import type { Parser, Tree, Node } from 'web-tree-sitter' import { findActiveNodeAtPosition, findInlineNodeAtPosition } from './tree-navigation.js' // Check if there's a complete inline_link that overlaps with the given range. -export function hasCompleteLinkAt(inlineRoot: Parser.SyntaxNode, startPos: number, endPos: number): boolean { +export function hasCompleteLinkAt(inlineRoot: Node, startPos: number, endPos: number): boolean { const linkNodes = inlineRoot.descendantsOfType('inline_link') for (const link of linkNodes) { // Check if this link overlaps with our range @@ -18,7 +18,7 @@ export function hasCompleteLinkAt(inlineRoot: Parser.SyntaxNode, startPos: numbe } // Check if there's a complete image that overlaps with the given range. -export function hasCompleteImageAt(inlineRoot: Parser.SyntaxNode, startPos: number, endPos: number): boolean { +export function hasCompleteImageAt(inlineRoot: Node, startPos: number, endPos: number): boolean { const imageNodes = inlineRoot.descendantsOfType('image') for (const img of imageNodes) { // Check if this image overlaps with our range @@ -112,7 +112,7 @@ export function hasIncompleteImageOpening(text: string, inlineParser: Parser | n } // Check if there's a complete code_span that overlaps with the given range. -export function hasCompleteCodeSpanAt(inlineRoot: Parser.SyntaxNode, startPos: number, endPos: number): boolean { +export function hasCompleteCodeSpanAt(inlineRoot: Node, startPos: number, endPos: number): boolean { const codeSpans = inlineRoot.descendantsOfType('code_span') for (const span of codeSpans) { // Check if this code_span overlaps with our range @@ -128,7 +128,7 @@ export function hasCompleteCodeSpanAt(inlineRoot: Parser.SyntaxNode, startPos: n } // Check if there's a complete strong_emphasis that overlaps with the given range. -export function hasCompleteBoldAt(inlineRoot: Parser.SyntaxNode, startPos: number, endPos: number): boolean { +export function hasCompleteBoldAt(inlineRoot: Node, startPos: number, endPos: number): boolean { const strongNodes = inlineRoot.descendantsOfType('strong_emphasis') for (const span of strongNodes) { // Check if this strong_emphasis overlaps with our range @@ -144,7 +144,7 @@ export function hasCompleteBoldAt(inlineRoot: Parser.SyntaxNode, startPos: numbe } // Check if there's a complete emphasis that overlaps with the given range. -export function hasCompleteItalicAt(inlineRoot: Parser.SyntaxNode, startPos: number, endPos: number): boolean { +export function hasCompleteItalicAt(inlineRoot: Node, startPos: number, endPos: number): boolean { const emphasisNodes = inlineRoot.descendantsOfType('emphasis') for (const span of emphasisNodes) { // Check if this emphasis overlaps with our range @@ -160,7 +160,7 @@ export function hasCompleteItalicAt(inlineRoot: Parser.SyntaxNode, startPos: num } // Check if there's a complete strikethrough that overlaps with the given range. -export function hasCompleteStrikethroughAt(inlineRoot: Parser.SyntaxNode, startPos: number, endPos: number): boolean { +export function hasCompleteStrikethroughAt(inlineRoot: Node, startPos: number, endPos: number): boolean { const strikethroughNodes = inlineRoot.descendantsOfType('strikethrough') for (const span of strikethroughNodes) { // Check if this strikethrough overlaps with our range @@ -243,12 +243,12 @@ export function hasUnmatchedItalicMarker(text: string, inlineParser: Parser | nu // Check if the position is inside a fenced_code_block or code_span (inline code). // Used to skip italic buffering inside code contexts where _ is common in variable names. export function isInsideCodeBlock( - node: Parser.SyntaxNode, + node: Node, position: number, - currentTree: Parser.Tree | null, + currentTree: Tree | null, inlineParser: Parser | null ): boolean { - let current: Parser.SyntaxNode | null = findActiveNodeAtPosition(node, position) + let current: Node | null = findActiveNodeAtPosition(node, position) while (current) { if (current.type === 'fenced_code_block' || current.type === 'code_fence_content') { @@ -281,14 +281,14 @@ export function isInsideCodeBlock( // Detect active inline styles at the given position range. // Checks both the inline tree and block tree for style nodes. export function detectActiveStyles( - node: Parser.SyntaxNode, + node: Node, startIdx: number, endIdx: number, - currentTree: Parser.Tree | null, + currentTree: Tree | null, inlineParser: Parser | null ): string[] { const styles: Set = new Set() - let current: Parser.SyntaxNode | null = node + let current: Node | null = node // First, find the inline node from the BLOCK tree (not the inline tree) // to get document-relative positions diff --git a/src/tree-sitter/inline-extractors.ts b/src/tree-sitter/inline-extractors.ts index f9f5026..ad55d55 100644 --- a/src/tree-sitter/inline-extractors.ts +++ b/src/tree-sitter/inline-extractors.ts @@ -1,4 +1,4 @@ -import type { Parser } from 'web-tree-sitter' +import type { Parser, Tree, Node } from 'web-tree-sitter' import type { StreamingChunk, BlockInfo, InlineStyleConfig, SpanType, ClosedSpan } from './types.js' import { findInlineNodeAtPosition } from './tree-navigation.js' import { createChunkFromBlockInfo, createClosedSpan, byteOffsetToUtf16 } from './segment-builder.js' @@ -17,7 +17,7 @@ function extractInlineStyleSegments( startByte: number, endByte: number, blockInfo: BlockInfo, - currentTree: Parser.Tree, + currentTree: Tree, inlineParser: Parser, currentUtf16Offset: number = 0, useDescendants: boolean = false @@ -45,12 +45,12 @@ function extractInlineStyleSegments( // Check overlap if (styleNode.startIndex < relativeEnd && styleNode.endIndex > relativeStart) { // Get delimiters - either children or descendants based on config - let delimiters: Parser.SyntaxNode[] + let delimiters: Node[] if (useDescendants) { delimiters = styleNode.descendantsOfType(config.delimiterType) - .sort((a: Parser.SyntaxNode, b: Parser.SyntaxNode) => a.startIndex - b.startIndex) + .sort((a: Node, b: Node) => a.startIndex - b.startIndex) } else { - delimiters = styleNode.children.filter((c: Parser.SyntaxNode) => c.type === config.delimiterType) + delimiters = styleNode.children.filter((c: Node) => c.type === config.delimiterType) } if (delimiters.length >= config.minDelimiters) { @@ -126,11 +126,11 @@ function extractInlineStyleSegments( // DEPRECATED: Use processInlineSpans() in segment-generator.ts instead. export function getInlineCodeSegments( content: string, - node: Parser.SyntaxNode, + node: Node, startByte: number, endByte: number, blockInfo: BlockInfo, - currentTree: Parser.Tree, + currentTree: Tree, inlineParser: Parser, currentUtf16Offset: number = 0 ): StreamingChunk[] { @@ -151,11 +151,11 @@ export function getInlineCodeSegments( // DEPRECATED: Use processInlineSpans() in segment-generator.ts instead. export function getBoldSegments( content: string, - node: Parser.SyntaxNode, + node: Node, startByte: number, endByte: number, blockInfo: BlockInfo, - currentTree: Parser.Tree, + currentTree: Tree, inlineParser: Parser, currentUtf16Offset: number = 0 ): StreamingChunk[] { @@ -176,11 +176,11 @@ export function getBoldSegments( // DEPRECATED: Use processInlineSpans() in segment-generator.ts instead. export function getItalicSegments( content: string, - node: Parser.SyntaxNode, + node: Node, startByte: number, endByte: number, blockInfo: BlockInfo, - currentTree: Parser.Tree, + currentTree: Tree, inlineParser: Parser, currentUtf16Offset: number = 0 ): StreamingChunk[] { @@ -201,11 +201,11 @@ export function getItalicSegments( // DEPRECATED: Use processInlineSpans() in segment-generator.ts instead. export function getStrikethroughSegments( content: string, - node: Parser.SyntaxNode, + node: Node, startByte: number, endByte: number, blockInfo: BlockInfo, - currentTree: Parser.Tree, + currentTree: Tree, inlineParser: Parser, currentUtf16Offset: number = 0 ): StreamingChunk[] { diff --git a/src/tree-sitter/segment-builder.ts b/src/tree-sitter/segment-builder.ts index 9662940..64e5af3 100644 --- a/src/tree-sitter/segment-builder.ts +++ b/src/tree-sitter/segment-builder.ts @@ -9,6 +9,45 @@ import type { ParserConfig } from './types.js' +// ============================================================================ +// UTF-16 OFFSET UTILITIES +// ============================================================================ + +// Convert byte offset to UTF-16 code unit offset. +// Tree-sitter gives us byte positions, but JavaScript strings use UTF-16. +export function byteOffsetToUtf16(text: string, byteOffset: number): number { + const encoder = new TextEncoder() + let utf16Offset = 0 + let currentByteOffset = 0 + + for (const char of text) { + if (currentByteOffset >= byteOffset) break + const charBytes = encoder.encode(char).length + currentByteOffset += charBytes + // Each JS string char is 1 UTF-16 code unit, except surrogates (2) + utf16Offset += char.length // .length gives UTF-16 code units + } + + return utf16Offset +} + +// Convert UTF-16 offset to byte offset. +// Needed when we have UTF-16 positions and need tree-sitter byte positions. +export function utf16ToByteOffset(text: string, utf16Offset: number): number { + const encoder = new TextEncoder() + let currentUtf16 = 0 + let byteOffset = 0 + + for (const char of text) { + if (currentUtf16 >= utf16Offset) break + const charBytes = encoder.encode(char).length + byteOffset += charBytes + currentUtf16 += char.length + } + + return byteOffset +} + // ============================================================================ // BLOCK CONTEXT HELPERS // ============================================================================ @@ -30,6 +69,8 @@ function mapBlockType(type: string): BlockType { case 'pipe_table_row': return 'table_row' case 'pipe_table_cell': + case 'table_header_cell': + case 'table_cell': return 'table_cell' case 'blockquote': return 'blockquote' diff --git a/src/tree-sitter/segment-generator.ts b/src/tree-sitter/segment-generator.ts index 4ffa4e6..93ff03c 100644 --- a/src/tree-sitter/segment-generator.ts +++ b/src/tree-sitter/segment-generator.ts @@ -1,4 +1,4 @@ -import type { Parser } from 'web-tree-sitter' +import type { Parser, Tree, Node } from 'web-tree-sitter' import type { StreamingChunk, BlockInfo, @@ -9,7 +9,7 @@ import type { ParserConfig } from './types.js' import { findActiveNodeAtPosition, findInlineNodeAtPosition, findBlockNode } from './tree-navigation.js' -import { getBlockInfo, isNewBlock } from './block-detection.js' +import { getBlockInfo } from './block-detection.js' import { hasCompleteCodeSpanAt, hasCompleteBoldAt, @@ -20,21 +20,17 @@ import { hasIncompleteLinkOpening, hasIncompleteImageOpening, hasUnmatchedItalicMarker, - isInsideCodeBlock, - detectActiveStyles + isInsideCodeBlock } from './inline-detection.js' import { getHeaderContent, getCodeBlockContent, getInlineContent } from './content-extraction.js' import { createChunkFromBlockInfo, createPlainTextChunk, - createCodeBlockChunk, - createHeadingChunk, createOpenSpan, createClosedSpan, createLinkSpan, - createImageSpan, - createBlockContext + createImageSpan } from './segment-builder.js' // Re-import the constant that we need locally @@ -55,10 +51,11 @@ const SUPPRESSED_SYNTAX_TYPES_LOCAL = [ export type SegmentGeneratorContext = { content: string - currentTree: Parser.Tree + currentTree: Tree inlineParser: Parser | null state: SegmentGeneratorState config?: ParserConfig + disableBlockBoundarySplit?: boolean } // Create initial segment generator state @@ -66,13 +63,80 @@ export function createInitialState(): SegmentGeneratorState { return { totalUtf16Offset: 0, lastEmittedOffset: 0, + sourceOffset: 0, + lastEmittedSourceOffset: 0, openSpans: [], currentBlock: null, pendingInlineContent: '', - accumulatedContent: '' + accumulatedContent: '', + checkpoints: [] } } +export function createCheckpoint(state: SegmentGeneratorState): SegmentGeneratorState['checkpoints'][number] { + return { + sourceOffset: state.sourceOffset, + renderedOffset: state.totalUtf16Offset, + lastEmittedSourceOffset: state.lastEmittedSourceOffset, + lastEmittedOffset: state.lastEmittedOffset, + openSpans: state.openSpans.map(span => ({ ...span })), + currentBlock: state.currentBlock ? { ...state.currentBlock } : null, + pendingInlineContent: state.pendingInlineContent, + pendingInlineStartIndex: state.pendingInlineStartIndex, + accumulatedContent: state.accumulatedContent, + } +} + +export function stateFromCheckpoint(checkpoint: SegmentGeneratorState['checkpoints'][number]): SegmentGeneratorState { + return { + totalUtf16Offset: checkpoint.renderedOffset, + lastEmittedOffset: checkpoint.lastEmittedOffset, + sourceOffset: checkpoint.sourceOffset, + lastEmittedSourceOffset: checkpoint.lastEmittedSourceOffset, + openSpans: checkpoint.openSpans.map(span => ({ ...span })), + currentBlock: checkpoint.currentBlock ? { ...checkpoint.currentBlock } : null, + pendingInlineContent: checkpoint.pendingInlineContent, + pendingInlineStartIndex: checkpoint.pendingInlineStartIndex, + accumulatedContent: checkpoint.accumulatedContent, + checkpoints: [checkpoint], + } +} + +function withCheckpoint(state: SegmentGeneratorState): SegmentGeneratorState { + const checkpoint = createCheckpoint(state) + const checkpoints = [...state.checkpoints, checkpoint] + return { + ...state, + checkpoints, + } +} + +function findFirstBlockBoundaryInRange( + node: Node, + fromIndex: number, + toIndex: number +): number | undefined { + let boundary: number | undefined + + const visit = (current: Node) => { + if (current.endIndex <= fromIndex || current.startIndex >= toIndex) { + return + } + + if (current.type === 'fenced_code_block' && current.startIndex > fromIndex && current.startIndex < toIndex) { + boundary = Math.min(boundary ?? Infinity, current.startIndex) + return + } + + for (const child of current.children) { + visit(child) + } + } + + visit(node) + return boundary +} + // Detect span type from tree-sitter node type function detectSpanType(nodeType: string): SpanType | null { switch (nodeType) { @@ -87,7 +151,7 @@ function detectSpanType(nodeType: string): SpanType | null { } // Extract span metadata (URL for links, src/alt for images) -function extractSpanMetadata(node: Parser.SyntaxNode): { url?: string; src?: string; alt?: string } { +function extractSpanMetadata(node: Node): { url?: string; src?: string; alt?: string } { if (node.type === 'inline_link') { const destNode = node.descendantsOfType('link_destination')[0] return { url: destNode?.text ?? '' } @@ -121,7 +185,7 @@ function isSpanClosing(spanStart: number, spanEnd: number, chunkStart: number, c // Create a closed span from node metadata function createClosedSpanFromNode( spanType: SpanType, - node: Parser.SyntaxNode, + node: Node, offset: number, length: number ): ClosedSpan | null { @@ -139,13 +203,68 @@ function createClosedSpanFromNode( return null } +function collectInlineDelimiterRanges(inlineTree: Tree): Array<{ start: number; end: number }> { + const root = inlineTree.rootNode + const ranges: Array<{ start: number; end: number }> = [] + + for (const delimiter of root.descendantsOfType('emphasis_delimiter')) { + ranges.push({ start: delimiter.startIndex, end: delimiter.endIndex }) + } + + for (const delimiter of root.descendantsOfType('code_span_delimiter')) { + ranges.push({ start: delimiter.startIndex, end: delimiter.endIndex }) + } + + for (const node of root.descendantsOfType('strikethrough')) { + const firstChild = node.child(0) + const lastChild = node.child(node.childCount - 1) + if (firstChild?.text === '~~') { + ranges.push({ start: firstChild.startIndex, end: firstChild.endIndex }) + } + if (lastChild?.text === '~~') { + ranges.push({ start: lastChild.startIndex, end: lastChild.endIndex }) + } + } + + ranges.sort((a, b) => a.start - b.start) + return ranges +} + +function rawToRenderedOffset(rawOffset: number, delimiterRanges: Array<{ start: number; end: number }>): number { + let renderedOffset = 0 + let cursor = 0 + + for (const range of delimiterRanges) { + if (range.start >= rawOffset) { + break + } + + if (cursor < range.start) { + renderedOffset += Math.max(0, Math.min(range.start, rawOffset) - cursor) + } + + cursor = Math.max(cursor, range.end) + if (cursor >= rawOffset) { + return renderedOffset + } + } + + if (cursor < rawOffset) { + renderedOffset += rawOffset - cursor + } + + return renderedOffset +} + // Process a single style node and categorize it function categorizeSpanNode( - node: Parser.SyntaxNode, + node: Node, content: string, - chunkStartUtf16: number, - chunkEndUtf16: number, - openSpans: OpenSpan[] + chunkStartRaw: number, + chunkEndRaw: number, + openSpans: OpenSpan[], + delimiterRanges: Array<{ start: number; end: number }>, + baseRenderedOffset: number ): { contained?: ClosedSpan opening?: OpenSpan @@ -156,23 +275,25 @@ function categorizeSpanNode( if (!spanType) return {} // node.startIndex/endIndex are already UTF-16 character offsets in web-tree-sitter JS bindings - const spanStartUtf16 = node.startIndex - const spanEndUtf16 = node.endIndex + const spanStartRaw = node.startIndex + const spanEndRaw = node.endIndex + const spanStartUtf16 = baseRenderedOffset + rawToRenderedOffset(spanStartRaw, delimiterRanges) + const spanEndUtf16 = baseRenderedOffset + rawToRenderedOffset(spanEndRaw, delimiterRanges) const spanLength = spanEndUtf16 - spanStartUtf16 // Fully contained - if (isSpanContained(spanStartUtf16, spanEndUtf16, chunkStartUtf16, chunkEndUtf16)) { + if (isSpanContained(spanStartRaw, spanEndRaw, chunkStartRaw, chunkEndRaw)) { const span = createClosedSpanFromNode(spanType, node, spanStartUtf16, spanLength) return span ? { contained: span } : {} } // Opens here, closes later - if (isSpanOpening(spanStartUtf16, spanEndUtf16, chunkStartUtf16, chunkEndUtf16)) { + if (isSpanOpening(spanStartRaw, spanEndRaw, chunkStartRaw, chunkEndRaw)) { return { opening: createOpenSpan(spanType, spanStartUtf16) } } // Opened earlier, closes here - if (isSpanClosing(spanStartUtf16, spanEndUtf16, chunkStartUtf16, chunkEndUtf16)) { + if (isSpanClosing(spanStartRaw, spanEndRaw, chunkStartRaw, chunkEndRaw)) { const matchingIdx = openSpans.findIndex(s => s.type === spanType) if (matchingIdx !== -1) { const matchingOpen = openSpans[matchingIdx] @@ -187,17 +308,19 @@ function categorizeSpanNode( // Process inline styles and categorize them as opening/closing/contained function processInlineSpans( - inlineTree: Parser.Tree, - chunkStartUtf16: number, - chunkEndUtf16: number, + inlineTree: Tree, + chunkStartRaw: number, + chunkEndRaw: number, content: string, - state: SegmentGeneratorState + state: SegmentGeneratorState, + baseRenderedOffset: number ): { opening: OpenSpan[]; closing: ClosedSpan[]; contained: ClosedSpan[]; newOpenSpans: OpenSpan[] } { const opening: OpenSpan[] = [] const closing: ClosedSpan[] = [] const contained: ClosedSpan[] = [] const newOpenSpans = [...state.openSpans] const indicesToRemove: number[] = [] + const delimiterRanges = collectInlineDelimiterRanges(inlineTree) const styleNodeTypes = ['code_span', 'strong_emphasis', 'emphasis', 'strikethrough', 'inline_link', 'image'] @@ -205,7 +328,7 @@ function processInlineSpans( const nodes = inlineTree.rootNode.descendantsOfType(nodeType) for (const node of nodes) { - const result = categorizeSpanNode(node, content, chunkStartUtf16, chunkEndUtf16, newOpenSpans) + const result = categorizeSpanNode(node, content, chunkStartRaw, chunkEndRaw, newOpenSpans, delimiterRanges, baseRenderedOffset) if (result.contained) { contained.push(result.contained) @@ -259,10 +382,8 @@ export function generateSegments( state = { ...state, pendingInlineContent: '' } } - // Calculate UTF-16 offsets for this chunk + // Public offsets are rendered-output UTF-16 offsets. Source offsets stay internal. const chunkStartUtf16 = state.totalUtf16Offset - const chunkTextUtf16Length = newContent.length - const chunkEndUtf16 = chunkStartUtf16 + chunkTextUtf16Length // Check if current content has unmatched inline delimiters const inlineNode = findInlineNodeAtPosition(currentTree.rootNode, actualFromIndex) @@ -281,6 +402,7 @@ export function generateSegments( if (!hasCompleteCodeSpan) { state.pendingInlineContent = newContent state.pendingInlineStartIndex = actualFromIndex + state.sourceOffset = actualToIndex return { segments, state } } } @@ -291,6 +413,7 @@ export function generateSegments( if (!hasCompleteBold) { state.pendingInlineContent = newContent state.pendingInlineStartIndex = actualFromIndex + state.sourceOffset = actualToIndex return { segments, state } } } @@ -304,6 +427,7 @@ export function generateSegments( if (!hasCompleteItalic) { state.pendingInlineContent = newContent state.pendingInlineStartIndex = actualFromIndex + state.sourceOffset = actualToIndex return { segments, state } } } @@ -315,6 +439,7 @@ export function generateSegments( if (!hasCompleteStrikethrough) { state.pendingInlineContent = newContent state.pendingInlineStartIndex = actualFromIndex + state.sourceOffset = actualToIndex return { segments, state } } } @@ -325,6 +450,7 @@ export function generateSegments( if (!hasCompleteLink && hasIncompleteLinkOpening(newPortion, inlineParser)) { state.pendingInlineContent = newContent state.pendingInlineStartIndex = actualFromIndex + state.sourceOffset = actualToIndex return { segments, state } } } @@ -335,6 +461,7 @@ export function generateSegments( if (!hasCompleteImage && hasIncompleteImageOpening(newPortion, inlineParser)) { state.pendingInlineContent = newContent state.pendingInlineStartIndex = actualFromIndex + state.sourceOffset = actualToIndex return { segments, state } } } @@ -345,6 +472,29 @@ export function generateSegments( return { segments, state } } + if (!context.disableBlockBoundarySplit) { + const boundary = findFirstBlockBoundaryInRange(currentTree.rootNode, actualFromIndex, actualToIndex) + if (boundary !== undefined) { + const prefix = generateSegments(actualFromIndex, boundary, { + ...context, + state, + disableBlockBoundarySplit: true, + }) + state = prefix.state + + const suffix = generateSegments(boundary, actualToIndex, { + ...context, + state, + disableBlockBoundarySplit: true, + }) + + return { + segments: [...prefix.segments, ...suffix.segments], + state: suffix.state, + } + } + } + // Find the deepest node containing the new content position const nodeAtPosition = findActiveNodeAtPosition(currentTree.rootNode, actualFromIndex) @@ -355,26 +505,38 @@ export function generateSegments( }) state = { ...state, - totalUtf16Offset: chunkEndUtf16, - lastEmittedOffset: chunkEndUtf16, + totalUtf16Offset: chunkStartUtf16 + newContent.length, + lastEmittedOffset: chunkStartUtf16 + newContent.length, + sourceOffset: actualToIndex, + lastEmittedSourceOffset: actualToIndex, accumulatedContent: state.accumulatedContent + newContent } + state = withCheckpoint(state) return { segments: [chunk], state } } // Check if the node is a suppressed syntax type if (SUPPRESSED_SYNTAX_TYPES_LOCAL.indexOf(nodeAtPosition.type) !== -1) { - // Update offset but don't emit - state = { ...state, totalUtf16Offset: chunkEndUtf16 } + state = { + ...state, + sourceOffset: actualToIndex, + accumulatedContent: state.accumulatedContent + newContent + } + state = withCheckpoint(state) return { segments, state } } // Check if we're inside a table delimiter row - let currentForDelimiter: Parser.SyntaxNode | null = nodeAtPosition + let currentForDelimiter: Node | null = nodeAtPosition while (currentForDelimiter) { if (currentForDelimiter.type === 'pipe_table_delimiter_row' || currentForDelimiter.type === 'pipe_table_delimiter_cell') { - state = { ...state, totalUtf16Offset: chunkEndUtf16 } + state = { + ...state, + sourceOffset: actualToIndex, + accumulatedContent: state.accumulatedContent + newContent + } + state = withCheckpoint(state) return { segments, state } } currentForDelimiter = currentForDelimiter.parent @@ -397,7 +559,12 @@ export function generateSegments( // Don't emit if it's only markers if (processedContent.length === 0 || processedContent.trim().length === 0) { - state = { ...state, totalUtf16Offset: chunkEndUtf16 } + state = { + ...state, + sourceOffset: actualToIndex, + accumulatedContent: state.accumulatedContent + newContent + } + state = withCheckpoint(state) return { segments, state } } } else if (blockInfo.type === 'codeBlock') { @@ -410,29 +577,26 @@ export function generateSegments( } if (processedContent.length === 0) { - state = { ...state, totalUtf16Offset: chunkEndUtf16 } + state = { + ...state, + sourceOffset: actualToIndex, + accumulatedContent: state.accumulatedContent + newContent + } + state = withCheckpoint(state) return { segments, state } } } else if (blockInfo.type === 'paragraph') { // Handle incomplete header markers if (nodeAtPosition.type in HEADER_MARKER_LEVELS_LOCAL) { - return { segments, state } - } - - // Handle code fence detection in paragraph content - const result = handleCodeFenceInParagraph( - newContent, actualFromIndex, actualToIndex, content, - segments, chunkStartUtf16, config - ) - if (result.handled) { state = { ...state, - totalUtf16Offset: state.totalUtf16Offset + result.utf16Consumed, - lastEmittedOffset: state.totalUtf16Offset + result.utf16Consumed, + sourceOffset: actualToIndex, accumulatedContent: state.accumulatedContent + newContent } - return { segments: result.segments, state } + state = withCheckpoint(state) + return { segments, state } } + } // Process inline spans for non-codeBlock types @@ -442,20 +606,23 @@ export function generateSegments( let strippedContent = processedContent if (blockInfo.type !== 'codeBlock' && inlineParser) { - const fullContent = state.accumulatedContent + processedContent - const inlineTree = inlineParser.parse(fullContent) - - // The chunk boundaries in the ACCUMULATED content space - const accumulatedOffset = state.accumulatedContent.length - const chunkStartInAccumulated = accumulatedOffset - const chunkEndInAccumulated = accumulatedOffset + processedContent.length + const hostInlineNode = findInlineNodeAtPosition(currentTree.rootNode, actualFromIndex) + const inlineContent = hostInlineNode?.text ?? processedContent + const inlineTree = inlineParser.parse(inlineContent) + const chunkStartInInline = hostInlineNode + ? Math.max(0, actualFromIndex - hostInlineNode.startIndex) + : 0 + const chunkEndInInline = hostInlineNode + ? Math.max(chunkStartInInline, actualToIndex - hostInlineNode.startIndex) + : processedContent.length const spanResult = processInlineSpans( inlineTree, - chunkStartInAccumulated, // Use position in accumulated content - chunkEndInAccumulated, - fullContent, - state + chunkStartInInline, + chunkEndInInline, + inlineContent, + state, + chunkStartUtf16 - rawToRenderedOffset(chunkStartInInline, collectInlineDelimiterRanges(inlineTree)) ) opening = spanResult.opening closing = spanResult.closing @@ -464,11 +631,15 @@ export function generateSegments( // Strip inline markers from the content strippedContent = getInlineContent( - processedContent, - inlineParser.parse(processedContent), // Parse just the new content - 0, - processedContent.length + hostInlineNode ? inlineContent.substring(chunkStartInInline, chunkEndInInline) : processedContent, + inlineTree, + chunkStartInInline, + chunkEndInInline ) + + if (hostInlineNode && blockInfo.type !== 'header' && actualToIndex > hostInlineNode.endIndex) { + strippedContent += content.substring(Math.max(actualFromIndex, hostInlineNode.endIndex), actualToIndex) + } } // Create the chunk with the new API @@ -488,8 +659,10 @@ export function generateSegments( // Update state state = { ...state, - totalUtf16Offset: chunkStartUtf16 + processedContent.length, - lastEmittedOffset: chunkStartUtf16 + processedContent.length, + totalUtf16Offset: chunkStartUtf16 + strippedContent.length, + lastEmittedOffset: chunkStartUtf16 + strippedContent.length, + sourceOffset: actualToIndex, + lastEmittedSourceOffset: actualToIndex, accumulatedContent: state.accumulatedContent + newContent, currentBlock: { type: blockInfo.type, @@ -500,153 +673,7 @@ export function generateSegments( hasEmittedContent: true } } + state = withCheckpoint(state) return { segments, state } } - -// Handle code fence detection when tree-sitter sees it as paragraph -function handleCodeFenceInParagraph( - newContent: string, - actualFromIndex: number, - actualToIndex: number, - content: string, - existingSegments: StreamingChunk[], - currentUtf16Offset: number, - config?: ParserConfig -): { handled: boolean; segments: StreamingChunk[]; utf16Consumed: number } { - const segments = [...existingSegments] - let utf16Consumed = 0 - - // Find code fence opening (```) - const fenceStart = newContent.indexOf('```') - if (fenceStart === -1) { - // No fence in new content - check if we're inside an existing code block - const contentBeforeThis = content.substring(0, actualFromIndex) - const fenceCount = countOccurrences(contentBeforeThis, '```') - const isInsideCodeBlockContext = fenceCount % 2 === 1 - - if (isInsideCodeBlockContext) { - const closingFenceIdx = newContent.indexOf('```') - - if (closingFenceIdx === -1) { - // No closing fence - emit as code block content - return { - handled: true, - segments: [createCodeBlockChunk(newContent, currentUtf16Offset, '', { - original: config?.includeRawStreamedToken ? newContent : undefined - })], - utf16Consumed: newContent.length - } - } else { - // Has closing fence - const codeContent = newContent.substring(0, closingFenceIdx) - const afterFence = newContent.substring(closingFenceIdx + 3) - let offset = currentUtf16Offset - - if (codeContent.length > 0) { - segments.push(createCodeBlockChunk(codeContent, offset, '', { - original: config?.includeRawStreamedToken ? codeContent : undefined - })) - offset += codeContent.length - } - - // Skip the fence markers - offset += 3 - - const textAfterFence = stripLeadingNewline(afterFence) - if (textAfterFence.length > 0) { - segments.push(createPlainTextChunk(textAfterFence, offset, { - original: config?.includeRawStreamedToken ? textAfterFence : undefined - })) - } - - return { handled: true, segments, utf16Consumed: newContent.length } - } - } - - return { handled: false, segments, utf16Consumed: 0 } - } - - // Extract language from fence line (```language) - const afterFenceMarker = newContent.substring(fenceStart + 3) - const newlineIdx = afterFenceMarker.indexOf('\n') - const fenceLanguage = newlineIdx === -1 - ? afterFenceMarker.trim() - : afterFenceMarker.substring(0, newlineIdx).trim() - const fenceMarkerLength = 3 + (newlineIdx === -1 ? afterFenceMarker.length : newlineIdx + 1) - - // Check for closing fence - const contentAfterOpening = newlineIdx === -1 - ? '' - : afterFenceMarker.substring(newlineIdx + 1) - const closingFenceIdx = contentAfterOpening.indexOf('```') - - // Content BEFORE the fence - const contentBeforeFence = newContent.substring(0, fenceStart) - let offset = currentUtf16Offset - - if (closingFenceIdx === -1) { - // No closing fence yet - emit content before fence and buffer the rest - if (contentBeforeFence.trim().length > 0) { - segments.push(createPlainTextChunk(contentBeforeFence, offset, { - original: config?.includeRawStreamedToken ? contentBeforeFence : undefined - })) - utf16Consumed += contentBeforeFence.length - } - - return { handled: true, segments, utf16Consumed } - } - - // Complete code block structure - if (contentBeforeFence.trim().length > 0) { - segments.push(createPlainTextChunk(contentBeforeFence, offset, { - original: config?.includeRawStreamedToken ? contentBeforeFence : undefined - })) - offset += contentBeforeFence.length - } - - // Skip fence marker - offset += fenceMarkerLength - - const codeContent = contentAfterOpening.substring(0, closingFenceIdx) - - if (codeContent.length > 0) { - segments.push(createCodeBlockChunk(codeContent, offset, fenceLanguage, { - original: config?.includeRawStreamedToken ? codeContent : undefined - })) - offset += codeContent.length - } - - // Skip closing fence - offset += 3 - - const afterClosingFence = contentAfterOpening.substring(closingFenceIdx + 3) - const textAfterFence = stripLeadingNewline(afterClosingFence) - if (textAfterFence.trim().length > 0) { - segments.push(createPlainTextChunk(textAfterFence, offset, { - original: config?.includeRawStreamedToken ? textAfterFence : undefined - })) - } - - return { handled: true, segments, utf16Consumed: newContent.length } -} - -// Count occurrences of a substring -function countOccurrences(str: string, substr: string): number { - let count = 0 - let pos = 0 - while ((pos = str.indexOf(substr, pos)) !== -1) { - count++ - pos += substr.length - } - return count -} - -// Strip leading newline if present -function stripLeadingNewline(str: string): string { - if (str.startsWith('\n')) { - return str.substring(1) - } - return str -} - diff --git a/src/tree-sitter/tree-navigation.ts b/src/tree-sitter/tree-navigation.ts index 2b78452..4ec06cc 100644 --- a/src/tree-sitter/tree-navigation.ts +++ b/src/tree-sitter/tree-navigation.ts @@ -1,10 +1,10 @@ -import type { Parser } from 'web-tree-sitter' +import type { Node } from 'web-tree-sitter' import { BLOCK_TYPES } from './types.js' // Find the deepest node in the BLOCK tree that contains the given position. // Uses exclusive end: position must be strictly less than endIndex. // This ensures we find nodes that START at position, not ones that END at position. -export function findActiveNodeAtPosition(node: Parser.SyntaxNode, position: number): Parser.SyntaxNode | null { +export function findActiveNodeAtPosition(node: Node, position: number): Node | null { if (position < node.startIndex || position >= node.endIndex) { return null } @@ -26,7 +26,7 @@ export function findActiveNodeAtPosition(node: Parser.SyntaxNode, position: numb // Find a node in the tree that contains the given position. // Uses inclusive end bounds. -export function findNodeInTree(node: Parser.SyntaxNode, position: number): Parser.SyntaxNode | null { +export function findNodeInTree(node: Node, position: number): Node | null { if (position < node.startIndex || position > node.endIndex) { return null } @@ -43,7 +43,7 @@ export function findNodeInTree(node: Parser.SyntaxNode, position: number): Parse // Find an inline node that contains the given position. // Returns the 'inline' or 'pipe_table_cell' node if found. -export function findInlineNodeAtPosition(node: Parser.SyntaxNode, position: number): Parser.SyntaxNode | null { +export function findInlineNodeAtPosition(node: Node, position: number): Node | null { // If this node is an inline node that contains the position, return it if (node.type === 'inline' && position >= node.startIndex && position < node.endIndex) { return node @@ -67,8 +67,8 @@ export function findInlineNodeAtPosition(node: Parser.SyntaxNode, position: numb // Find the block-level node that contains the given node. // Walks up the tree until a block type is found. -export function findBlockNode(node: Parser.SyntaxNode): Parser.SyntaxNode | null { - let current: Parser.SyntaxNode | null = node +export function findBlockNode(node: Node): Node | null { + let current: Node | null = node while (current) { if (BLOCK_TYPES.indexOf(current.type as typeof BLOCK_TYPES[number]) !== -1) { diff --git a/src/tree-sitter/types.ts b/src/tree-sitter/types.ts index 18cd858..65c97fa 100644 --- a/src/tree-sitter/types.ts +++ b/src/tree-sitter/types.ts @@ -1,4 +1,4 @@ -import type { Parser } from 'web-tree-sitter' +import type { Parser, Tree, Node } from 'web-tree-sitter' // ============================================================================ // SPAN TYPES - Typed spans with metadata @@ -170,13 +170,13 @@ export type BlockInfo = { // Context passed to inline style extractors export type InlineExtractionContext = { content: string - node: Parser.SyntaxNode + node: Node startByte: number endByte: number baseSpans: OpenSpan[] blockInfo: BlockInfo inlineParser: Parser - currentTree: Parser.Tree + currentTree: Tree } // Configuration for a specific inline style type @@ -233,12 +233,18 @@ export const INLINE_STYLE_CONFIGS: Record = { // State maintained by the segment generator across chunks export type SegmentGeneratorState = { - // Total UTF-16 code units emitted so far (from stream start) + // Total rendered UTF-16 code units emitted so far. totalUtf16Offset: number - // Last emitted UTF-16 offset (for backtrack detection) + // Last rendered UTF-16 offset emitted to consumers. lastEmittedOffset: number + // Raw markdown source UTF-16 offset processed so far. + sourceOffset: number + + // Last raw markdown source UTF-16 offset that produced public output. + lastEmittedSourceOffset: number + // Currently open spans that haven't closed yet openSpans: OpenSpan[] @@ -253,4 +259,19 @@ export type SegmentGeneratorState = { // Accumulated content for backtrack reference accumulatedContent: string + + // Stable replay points used to translate source recovery ranges to rendered offsets. + checkpoints: SegmentGeneratorCheckpoint[] +} + +export type SegmentGeneratorCheckpoint = { + sourceOffset: number + renderedOffset: number + lastEmittedSourceOffset: number + lastEmittedOffset: number + openSpans: OpenSpan[] + currentBlock: BlockState | null + pendingInlineContent: string + pendingInlineStartIndex?: number + accumulatedContent: string } diff --git a/tsdown.config.ts b/tsdown.config.ts new file mode 100644 index 0000000..e439441 --- /dev/null +++ b/tsdown.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from 'tsdown' + +export default defineConfig({ + entry: ['src/markdown-stream-parser.ts'], + dts: true, + format: ['esm', 'cjs'], + minify: true, + outDir: 'build', + clean: true, + sourcemap: false, + target: 'es2015', +}) diff --git a/tsup.config.ts b/tsup.config.ts deleted file mode 100644 index 90bb43c..0000000 --- a/tsup.config.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { defineConfig } from 'tsup'; - -export default defineConfig({ - entry: ['src/markdown-stream-parser.ts'], - dts: true, - format: ['esm', 'cjs'], - minify: true, - outDir: 'build', - clean: true, - sourcemap: false, // Disabled sourcemap for smaller package size - target: 'es2015', - esbuildOptions(options) { - options.plugins = [ - { - name: 'fix-ts-extensions', - setup(build) { - build.onResolve({ filter: /\.ts$/ }, args => { - if (args.path.startsWith('.')) { - return { path: args.path.replace(/\.ts$/, '.js'), external: false } - } - }); - } - } - ]; - } -}); From e294d916a8c149b085ac4b3e449a7cbd0b90c67e Mon Sep 17 00:00:00 2001 From: Dmitry Bondarenko Date: Sun, 21 Jun 2026 15:52:49 +0600 Subject: [PATCH 25/32] Corrects imports, cleans some obsolete mds --- README.md | 17 +- demo/svelte-demo/src/routes/+page.svelte | 2 +- error-recovery-plan.md | 72 -------- ...r-recovery-scaling-and-window-semantics.md | 154 ------------------ src/markdown-stream-parser.ts | 4 +- ...tree-sitter-markdown-stream-parser.test.ts | 2 +- src/tree-sitter-markdown-stream-parser.ts | 8 +- src/tree-sitter/block-detection.ts | 4 +- src/tree-sitter/inline-detection.ts | 2 +- src/tree-sitter/inline-extractors.ts | 6 +- src/tree-sitter/segment-builder.ts | 2 +- src/tree-sitter/segment-generator.ts | 12 +- src/tree-sitter/tree-navigation.ts | 2 +- 13 files changed, 34 insertions(+), 253 deletions(-) delete mode 100644 error-recovery-plan.md delete mode 100644 error-recovery-scaling-and-window-semantics.md diff --git a/README.md b/README.md index f63f88b..c2418c8 100644 --- a/README.md +++ b/README.md @@ -308,7 +308,7 @@ parser.setConfig({ windowSize: 1000 }) | Option | Type | Default | Description | |--------|------|---------|-------------| -| `windowSize` | `number` | `undefined` (unlimited) | Maximum backtrack distance in UTF-16 code units. Limits how far back the parser can correct previous output. | +| `windowSize` | `number` | `undefined` (unlimited) | Requested lookback window in rendered UTF-16 code units. Recovery beyond this window does not yet have a strict public overflow contract; see [Known issues and limitations](#known-issues-and-limitations). | | `includeRawStreamedToken` | `boolean` | `false` | When `true`, each chunk includes the original markdown source in `chunk.original`. Useful as a fallback for unsupported formatting. | @@ -389,7 +389,7 @@ This will execute the parser against the selected example stream and print parse ## Running tests -The project includes comprehensive test coverage with 219+ tests across all core functionality. To run the tests: +The project includes comprehensive test coverage with 235+ tests across all core functionality. To run the tests: 1. Start the Docker container: ```bash @@ -612,9 +612,13 @@ MarkdownStreamParser.removeInstance('session-1') // cleanup when done --- -## Known issues +## Known issues and limitations - **Delayed processing for extremely long sequences of characters without whitespace**: Due to how token buffering works, extremely long uninterrupted sequences (like a huge regex) can delay output until the sequence completes. In practice this is rarely noticeable with modern LLM speeds, but it can happen. +- **Recovery beyond `windowSize` is not strictly defined yet**: `windowSize` is measured against rendered UTF-16 output, but the parser does not currently emit a dedicated overflow event when the structurally correct recovery point is older than the configured window. Consumers that require guaranteed correction should leave `windowSize` undefined until an explicit overflow/fallback contract is implemented. +- **Inline-delimiter replay needs more coverage**: Split inline delimiters are buffered during normal streaming, but recovery tests do not yet fully assert opening and closing span state when already-emitted inline content is replayed. +- **Deletion-only recovery needs direct coverage**: The parser can emit a zero-length correction chunk when stale rendered output must be removed without replacement, but this path does not yet have a dedicated recovery test. +- **Some Markdown structures remain incomplete**: Blockquotes and full table behavior still have skipped feature tests and are tracked in the feature list above. --- @@ -626,8 +630,11 @@ MarkdownStreamParser.removeInstance('session-1') // cleanup when done - **Roadmap:** - Support for the missing markdown features listed earlier - - Performance optimizations - - Improved error recovery for malformed streams + - Define a strict `windowSize` overflow contract. Recovery must select a checkpoint at or before the earliest structurally affected source position and must never silently choose a later checkpoint merely to fit the window. The intended API should report that the recovery limit was exceeded or use an explicitly configured fallback such as a full snapshot replacement. + - Add recovery coverage for inline delimiter replay, including opening and closing spans, and for deletion-only corrections that emit a zero-length chunk. + - Improve checkpoint scaling: create checkpoints only at stable block boundaries or configured intervals, move checkpoint storage out of copied generator state, prune history outside the supported recovery range, retain an older baseline when unlimited recovery is enabled, and use indexed/binary-search lookup by source and rendered offsets. + - Limit error inspection to tree-sitter changed ranges, their containing blocks, a small surrounding recovery region, and explicitly tracked unresolved errors instead of recursively scanning the complete syntax tree after every token. + - Add long-stream benchmarks and recovery correctness tests to prevent quadratic checkpoint-copying and full-tree-scan regressions. --- diff --git a/demo/svelte-demo/src/routes/+page.svelte b/demo/svelte-demo/src/routes/+page.svelte index 6dbcbcf..978842d 100644 --- a/demo/svelte-demo/src/routes/+page.svelte +++ b/demo/svelte-demo/src/routes/+page.svelte @@ -7,7 +7,7 @@ type OpenSpan, type ClosedSpan, type SpanType, - } from "../../../../src/markdown-stream-parser.js"; + } from "../../../../src/markdown-stream-parser.ts"; type ExampleFile = { base: string; json: string; txt: string }; diff --git a/error-recovery-plan.md b/error-recovery-plan.md deleted file mode 100644 index cfbbbfc..0000000 --- a/error-recovery-plan.md +++ /dev/null @@ -1,72 +0,0 @@ -# Tree-Sitter Recovery Implementation Plan - -## Summary - -- [x] Migrate the public package API to the tree-sitter parser; README already documents the async tree-sitter API. -- [x] Make all public offsets rendered-output UTF-16 offsets: `chunk.offset`, `chunk.length`, `OpenSpan.openOffset`, `ClosedSpan.offset`, `ClosedSpan.length`, and `backtrackOffset`. -- [x] Use tree-sitter recovery signals fully: changed ranges plus explicit `ERROR`/missing nodes from block and inline trees. -- [x] Preserve the current Docker test baseline and add focused recovery/build coverage. - -## Package Entrypoint - -- [x] Replace `src/markdown-stream-parser.ts` with a public re-export or wrapper around `src/tree-sitter-markdown-stream-parser.ts`. -- [x] Update legacy entrypoint tests to the async tree-sitter API. -- [x] Keep the build entry at `src/markdown-stream-parser.ts` so package exports stay stable. -- [x] Ensure runtime tree-sitter dependencies are in `dependencies`, not only `devDependencies`. -- [x] Add a package build smoke test that imports from `build/markdown-stream-parser.js`. - -## Offset Model - -- [x] Refactor generator state to track source offsets and rendered offsets separately. -- [x] Advance public rendered offsets only by emitted text, not stripped markdown markers or suppressed syntax. -- [x] Convert `OpenSpan` and `ClosedSpan` positions/lengths to rendered offsets. -- [x] Keep `chunk.original` as raw source when `includeRawStreamedToken` is enabled. -- [x] Apply `windowSize` to rendered offsets, because consumers discard rendered output. - -## Recovery Flow - -- [x] Find earliest affected source offset from `getChangedRanges()`. -- [x] Include explicit tree-sitter `ERROR` and missing nodes in affected-range detection. -- [x] Store generator checkpoints at stable emitted boundaries: source offset, rendered offset, open spans, pending inline state, accumulated content, and block state. -- [x] Rebuild recovery output from the nearest stable checkpoint before the affected source offset. -- [x] Emit `backtrackOffset` on the first correction chunk using rendered-offset coordinates. -- [x] Emit a zero-length correction chunk when recovery deletes stale output without replacement. -- [x] Prevent stale or duplicated chunks after repeated recovery events. - -## Tests - -- [x] Add exact rendered-offset tests for headings, inline styles, and code blocks after marker stripping. -- [x] Add table reclassification recovery after preceding heading/paragraph markdown. -- [x] Add code fence split-across-chunks coverage. -- [x] Add code fence reclassification recovery after emitted paragraph text. -- [x] Add unclosed code fence at stream end coverage. -- [x] Add code fence rendered-offset coverage after preceding markdown syntax. -- [ ] Add inline delimiter recovery with open/closing spans. -- [ ] Add recovery test for stale output deletion without replacement. -- [x] Add `windowSize` recovery test using rendered distance. -- [x] Add `includeRawStreamedToken` recovery test. -- [x] Strengthen consumer simulation tests to apply `backtrackOffset` to a rendered string buffer. - -## Integration Cleanup - -- [x] Update demo/debug utilities that import `tree-sitter-markdown-stream-parser` directly when package entrypoint migration is complete. -- [x] Update any old `{ status: "STREAMING", segment }` assumptions to the tree-sitter `{ status: "STREAMING", chunk }` shape. -- [x] Fix TypeScript build issues around `web-tree-sitter` types and nullable parse results. - -## Verification - -- [x] Run full tests in a one-off Docker container mounted to this workspace. -- [x] Run `pnpm run build` or the Docker equivalent. -- [x] Document any remaining skipped tests or known limitations. - -## Remaining Follow-Up - -- [x] Add explicit code fence recovery coverage: - - [x] Split fence across chunks strips markers and emits `code_block`. - - [x] Reclassification after emitted paragraph text applies corrected output. - - [x] Unclosed fence at stream end emits code content without stuck buffering. - - [x] Fence after stripped markdown syntax uses rendered offsets. - - [x] Remove `handleCodeFenceInParagraph` after tests prove tree-sitter parsing covers these cases. -- [ ] Add inline delimiter recovery coverage that asserts open/closing spans through replay. -- [ ] Add a direct stale-output deletion recovery case that exercises the zero-length correction chunk path. -- [ ] Existing skipped feature tests remain skipped for blockquotes and fuller table support; they are outside this recovery pass. diff --git a/error-recovery-scaling-and-window-semantics.md b/error-recovery-scaling-and-window-semantics.md deleted file mode 100644 index bb3e181..0000000 --- a/error-recovery-scaling-and-window-semantics.md +++ /dev/null @@ -1,154 +0,0 @@ -# Error Recovery: Scaling and Window Semantics - -This note describes three engineering improvements needed to keep Markdown error recovery correct and efficient for long streams: - -1. Checkpoint pruning and indexing -2. Eliminating full-tree scans -3. Defining strict `windowSize` behavior - -## Checkpoint pruning and indexing - -A checkpoint stores enough parser state to restart segment generation from a previous source position. - -The current implementation adds checkpoints frequently and copies the complete checkpoint array whenever it adds one: - -```ts -const checkpoints = [...state.checkpoints, checkpoint] -``` - -For `n` emitted segments, repeated array copying approaches O(n²). Checkpoint lookup also scans the array linearly. - -### Recommended design - -- Create checkpoints only at stable boundaries: - - End of paragraph - - End of heading - - Complete list item - - Code fence boundary - - A configurable source-character interval -- Retain only checkpoints inside the supported recovery window. -- Retain one older baseline checkpoint when unlimited recovery is required. -- Index checkpoints by source and rendered offsets. -- Use binary search to select the latest valid checkpoint. -- Keep the checkpoint collection in mutable parser-internal storage instead of copying it into every generator state. - -```text -source: 0───100───200───300───400 -checkpoints: C1 C2 C3 C4 - ^ - changed region -``` - -Recovery should select `C2` using an indexed lookup and replay from that checkpoint. - -This changes checkpoint lookup from O(n) to O(log n), avoids repeated history copying, and prevents unbounded checkpoint growth. - -## Eliminating full-tree scans - -After each streamed token, the current parser recursively scans the complete syntax tree to find the earliest error. - -For a growing document, the cumulative work can become quadratic: - -```text -chunk 1: scan 100 nodes -chunk 2: scan 200 nodes -chunk 3: scan 300 nodes -... -``` - -Tree-sitter already reports changed ranges. Error inspection should normally be limited to: - -- Changed ranges -- Their containing block nodes -- A small surrounding recovery region -- Previously tracked unresolved errors that overlap the new changes - -The processing flow should be: - -```text -append token - ↓ -Tree-sitter returns changed ranges - ↓ -inspect affected blocks or subtrees - ↓ -recover only if previously emitted output was affected -``` - -For example, when a delimiter row reclassifies a preceding line as a table header, the parser should inspect the affected table subtree instead of rescanning unrelated headings and paragraphs. - -Unresolved errors can be tracked explicitly: - -```ts -type PendingError = { - sourceStart: number - sourceEnd: number - containingBlockStart: number -} -``` - -When new input arrives, the parser revisits only pending errors that overlap or are structurally related to the changed ranges. - -## Strict `windowSize` behavior - -`windowSize` represents how far back a consumer can revise already-rendered output. - -```text -rendered output length: 1,000 -windowSize: 100 -earliest legal backtrack: 900 -``` - -If tree-sitter discovers that output beginning at offset 700 is incorrect, the parser cannot start recovery at 900 and claim that the correction is complete. The structural change began before the recoverable window. - -### Required invariant - -The selected checkpoint must never occur after the earliest affected source position: - -```text -selected checkpoint source offset <= earliest affected source offset -``` - -Without this invariant, replay can start after the damaged region and produce output that appears valid but is internally inconsistent. - -### Possible overflow policies - -#### Strict failure - -Emit an explicit event when the required recovery exceeds the consumer's supported window: - -```ts -{ - status: 'RECOVERY_LIMIT_EXCEEDED', - requiredOffset: 700, - earliestAllowedOffset: 900 -} -``` - -The consumer can then restart parsing or request a complete replacement. - -#### Full snapshot replacement - -Emit the complete corrected document or affected block instead of attempting a partial backtrack. - -#### Block-level recovery - -Treat `windowSize` as a target while permitting recovery to extend to the beginning of the affected Markdown block. This is practical for Markdown, but the behavior must be part of the public contract. - -### Recommended contract - -1. Determine the earliest affected source position. -2. Select the latest checkpoint at or before that position. -3. Translate that checkpoint to its rendered offset. -4. If the rendered offset violates `windowSize`, emit an explicit recovery-limit event. -5. Optionally apply a configured fallback, such as full snapshot replacement. -6. Never silently select a later checkpoint to satisfy the window. - -## Suggested implementation order - -1. Define and test the `windowSize` overflow contract. -2. Move checkpoint storage out of copied generator state. -3. Add checkpoint pruning and binary-search lookup. -4. Restrict error detection to changed subtrees and tracked pending errors. -5. Add long-stream benchmarks and recovery correctness tests. - diff --git a/src/markdown-stream-parser.ts b/src/markdown-stream-parser.ts index 851c9db..cfa0e88 100644 --- a/src/markdown-stream-parser.ts +++ b/src/markdown-stream-parser.ts @@ -1,4 +1,4 @@ -export { MarkdownStreamParser } from './tree-sitter-markdown-stream-parser.js' +export { MarkdownStreamParser } from './tree-sitter-markdown-stream-parser.ts' export type { Span, @@ -10,4 +10,4 @@ export type { Chunk, StreamingChunk, ParserConfig, -} from './tree-sitter-markdown-stream-parser.js' +} from './tree-sitter-markdown-stream-parser.ts' diff --git a/src/tree-sitter-markdown-stream-parser.test.ts b/src/tree-sitter-markdown-stream-parser.test.ts index db6f5c5..379185c 100644 --- a/src/tree-sitter-markdown-stream-parser.test.ts +++ b/src/tree-sitter-markdown-stream-parser.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' import { MarkdownStreamParser } from './tree-sitter-markdown-stream-parser' -import type { Chunk, ClosedSpan, SpanType } from './tree-sitter/types.js' +import type { Chunk, ClosedSpan, SpanType } from './tree-sitter/types.ts' import path from 'path' import { fileURLToPath } from 'url' import fs from 'fs' diff --git a/src/tree-sitter-markdown-stream-parser.ts b/src/tree-sitter-markdown-stream-parser.ts index b95713b..6439ed1 100644 --- a/src/tree-sitter-markdown-stream-parser.ts +++ b/src/tree-sitter-markdown-stream-parser.ts @@ -1,7 +1,7 @@ import { Parser, Language, type Tree, type Node } from 'web-tree-sitter' -import TokensStreamBuffer from './tokens-stream-buffer.js' -import type { StreamingChunk, ParserConfig, SegmentGeneratorState } from './tree-sitter/types.js' -import { generateSegments, createInitialState, stateFromCheckpoint } from './tree-sitter/segment-generator.js' +import TokensStreamBuffer from './tokens-stream-buffer.ts' +import type { StreamingChunk, ParserConfig, SegmentGeneratorState } from './tree-sitter/types.ts' +import { generateSegments, createInitialState, stateFromCheckpoint } from './tree-sitter/segment-generator.ts' // Re-export types for external consumers @@ -15,7 +15,7 @@ export type { Chunk, StreamingChunk, ParserConfig -} from './tree-sitter/types.js' +} from './tree-sitter/types.ts' // Tree-sitter based streaming markdown parser. // diff --git a/src/tree-sitter/block-detection.ts b/src/tree-sitter/block-detection.ts index 440f6c2..a3f3deb 100644 --- a/src/tree-sitter/block-detection.ts +++ b/src/tree-sitter/block-detection.ts @@ -1,6 +1,6 @@ import type { Node } from 'web-tree-sitter' -import { HEADER_MARKER_LEVELS, type BlockInfo, type BlockState } from './types.js' -import { findBlockNode } from './tree-navigation.js' +import { HEADER_MARKER_LEVELS, type BlockInfo, type BlockState } from './types.ts' +import { findBlockNode } from './tree-navigation.ts' // Get the block type and properties from a tree-sitter node. // Walks up the tree to find the enclosing block structure. diff --git a/src/tree-sitter/inline-detection.ts b/src/tree-sitter/inline-detection.ts index 7ba33c8..1723e40 100644 --- a/src/tree-sitter/inline-detection.ts +++ b/src/tree-sitter/inline-detection.ts @@ -1,5 +1,5 @@ import type { Parser, Tree, Node } from 'web-tree-sitter' -import { findActiveNodeAtPosition, findInlineNodeAtPosition } from './tree-navigation.js' +import { findActiveNodeAtPosition, findInlineNodeAtPosition } from './tree-navigation.ts' // Check if there's a complete inline_link that overlaps with the given range. export function hasCompleteLinkAt(inlineRoot: Node, startPos: number, endPos: number): boolean { diff --git a/src/tree-sitter/inline-extractors.ts b/src/tree-sitter/inline-extractors.ts index ad55d55..36ed279 100644 --- a/src/tree-sitter/inline-extractors.ts +++ b/src/tree-sitter/inline-extractors.ts @@ -1,7 +1,7 @@ import type { Parser, Tree, Node } from 'web-tree-sitter' -import type { StreamingChunk, BlockInfo, InlineStyleConfig, SpanType, ClosedSpan } from './types.js' -import { findInlineNodeAtPosition } from './tree-navigation.js' -import { createChunkFromBlockInfo, createClosedSpan, byteOffsetToUtf16 } from './segment-builder.js' +import type { StreamingChunk, BlockInfo, InlineStyleConfig, SpanType, ClosedSpan } from './types.ts' +import { findInlineNodeAtPosition } from './tree-navigation.ts' +import { createChunkFromBlockInfo, createClosedSpan, byteOffsetToUtf16 } from './segment-builder.ts' // NOTE: These legacy extractors are kept for backward compatibility but are // no longer used by the main segment generator. The new API uses processInlineSpans() diff --git a/src/tree-sitter/segment-builder.ts b/src/tree-sitter/segment-builder.ts index 64e5af3..4359cb7 100644 --- a/src/tree-sitter/segment-builder.ts +++ b/src/tree-sitter/segment-builder.ts @@ -7,7 +7,7 @@ import type { OpenSpan, ClosedSpan, ParserConfig -} from './types.js' +} from './types.ts' // ============================================================================ // UTF-16 OFFSET UTILITIES diff --git a/src/tree-sitter/segment-generator.ts b/src/tree-sitter/segment-generator.ts index 93ff03c..1de14de 100644 --- a/src/tree-sitter/segment-generator.ts +++ b/src/tree-sitter/segment-generator.ts @@ -7,9 +7,9 @@ import type { ClosedSpan, SpanType, ParserConfig -} from './types.js' -import { findActiveNodeAtPosition, findInlineNodeAtPosition, findBlockNode } from './tree-navigation.js' -import { getBlockInfo } from './block-detection.js' +} from './types.ts' +import { findActiveNodeAtPosition, findInlineNodeAtPosition, findBlockNode } from './tree-navigation.ts' +import { getBlockInfo } from './block-detection.ts' import { hasCompleteCodeSpanAt, hasCompleteBoldAt, @@ -21,8 +21,8 @@ import { hasIncompleteImageOpening, hasUnmatchedItalicMarker, isInsideCodeBlock -} from './inline-detection.js' -import { getHeaderContent, getCodeBlockContent, getInlineContent } from './content-extraction.js' +} from './inline-detection.ts' +import { getHeaderContent, getCodeBlockContent, getInlineContent } from './content-extraction.ts' import { createChunkFromBlockInfo, createPlainTextChunk, @@ -31,7 +31,7 @@ import { createClosedSpan, createLinkSpan, createImageSpan -} from './segment-builder.js' +} from './segment-builder.ts' // Re-import the constant that we need locally const HEADER_MARKER_LEVELS_LOCAL: Record = { diff --git a/src/tree-sitter/tree-navigation.ts b/src/tree-sitter/tree-navigation.ts index 4ec06cc..be3bb7d 100644 --- a/src/tree-sitter/tree-navigation.ts +++ b/src/tree-sitter/tree-navigation.ts @@ -1,5 +1,5 @@ import type { Node } from 'web-tree-sitter' -import { BLOCK_TYPES } from './types.js' +import { BLOCK_TYPES } from './types.ts' // Find the deepest node in the BLOCK tree that contains the given position. // Uses exclusive end: position must be strictly less than endIndex. From 202f7a254b606cc5d06ebe60d013c4c201db5ed0 Mon Sep 17 00:00:00 2001 From: Dmitry Bondarenko Date: Mon, 22 Jun 2026 22:10:47 +0600 Subject: [PATCH 26/32] Updates documentation --- MAINTAINING-DOCUMENTATION.md | 207 ++++++++++ README.md | 751 ++++++++++++----------------------- 2 files changed, 469 insertions(+), 489 deletions(-) create mode 100644 MAINTAINING-DOCUMENTATION.md diff --git a/MAINTAINING-DOCUMENTATION.md b/MAINTAINING-DOCUMENTATION.md new file mode 100644 index 0000000..75d1f85 --- /dev/null +++ b/MAINTAINING-DOCUMENTATION.md @@ -0,0 +1,207 @@ +--- +title: Maintaining Documentation +description: How to keep Lixpi's developer documentation accurate, readable, Markdoc-compatible, and easy to navigate as the product and architecture change. +--- + +# Maintaining Documentation + +Lixpi documentation should be useful to a human developer first. It can help agents too, but it should not read like agent scaffolding, a checklist dump, or a frozen snapshot of the repo tree. + +Use this guide when creating, moving, deleting, or reorganizing documentation. + +## Start by Discovering the Live Shape + +Do not assume folders, page names, or architecture boundaries are permanent. Before changing documentation: + +1. Read the docs index at the root of the documentation tree. +2. Check the generated docs-site navigation or list the Markdown files. +3. Read the pages around the area you are changing. +4. Read nearby source-code READMEs for the implementation area. +5. Fact-check behavior against the live code before repeating or rewriting it. + +The docs index is a map, not a contract. If the product shape changes, update the map to match the new shape. Avoid adding tiny "read this folder first" files whose only job is routing; put real guidance in this guide, in the relevant domain page, or in the docs index. + +## Document the Live System, Not a Timeline + +Hard rule: product and developer docs describe how the system works. They are +not a history record, migration diary, before/after report, or commentary on +what changed. + +Never frame normal documentation with phrases like: + +- "Current Responsibilities" +- "Current State" +- "now" +- "previously" +- "used to" +- "no longer" +- "old behavior" +- "new behavior" +- "deprecated path" +- "legacy path" + +Write the contract directly: + +- Use "Responsibilities", not "Current Responsibilities". +- Use "Input Flow", not "Current Flow". +- Use "Schema", "Runtime Wiring", "Files", "Transaction Meta", and similar direct headings. +- Say what the code does, not what it replaced. + +Mention removed or replaced behavior only in an explicit archive, migration +plan, changelog, or compatibility section where that history is the subject. If +a symbol remains for compatibility, document the live compatibility contract: +"parses `aiUserInput` and removes it in `appendTransaction()`", not "this used +to be the composer." + +## Keep the Structure Flexible + +Organize by stable product or engineering concerns, not by whatever filenames happen to exist today. Good documentation domains usually answer one of these questions: + +- What is this part of the product? +- What data does it persist? +- How does the runtime path work? +- How does a user flow move through the system? +- How is it deployed or operated? +- What conventions must implementation code follow? + +When the architecture changes, the documentation shape should change with it. Moving a page is fine. Splitting a page is fine. Deleting a page is fine if the content was moved or is false. + +Before deleting or replacing docs, compare against the existing version and account for every important concept: + +- Keep still-true product behavior. +- Drop false behavior. +- Keep history out of normal docs unless the page is explicitly an archive, migration plan, changelog, or compatibility note. +- Preserve useful rationale, constraints, and gotchas. +- Remove stale route-finding breadcrumbs. + +## Keep the Docs Honest + +Every factual claim should be easy to defend from live code, infrastructure, tests, or linked external source. + +Prefer durable statements over brittle ones: + +- Say "application tables" instead of freezing a table count. +- Say "configured by the deployment" instead of hardcoding a task count unless the exact number is the point. +- Say "configured default" when a setting can change. +- Say "computed and logged" if the code does not publish or persist something. +- Say "future split needs worker subscription code" if the boundary exists but the implementation is not wired. + +Avoid broad absolute claims unless the code enforces them: + +- "all" +- "every" +- "never" +- "guarantees" +- "only source" +- "production-ready" +- "no code changes" + +If the claim is a benchmark, capacity estimate, market comparison, legal/compliance statement, or vendor capability, either cite an up-to-date source or make it clear that it is a hypothesis that needs validation. + +## Write Like a Developer + +Use direct, natural language. Prefer the plain sentence that explains the thing. + +Avoid bureaucratic filler: + +- "source of truth" when "covers" or "explains" works +- "owned by" when "covered in" works +- "delta" when "what is specific to this page" works +- "leverage" when "use" works +- "robust solution" without saying what failure it handles + +Documentation should sound like a senior engineer explaining the system to another engineer: precise, calm, and not puffed up. + +## Keep Markdoc Compatibility + +These docs are Markdown that must render through the static Markdoc site. + +Use this authoring shape: + +```markdown +--- +title: Page Title +description: One sentence about what this page covers. +--- + +# Page Title +``` + +Frontmatter is not mandatory for the renderer, but human-facing pages should have it. + +Use standard Markdown whenever possible: + +- Relative links to documentation pages should point at `.md` files. +- Links to source code outside the documentation tree should be normal relative repo links. +- Use fenced code blocks with a language tag. +- Use Mermaid only inside fenced `mermaid` blocks. +- Use Markdoc callouts for notes, warnings, important details, and tips. + +```markdoc +{% callout type="warning" %} +Explain the risk and what to do about it. +{% /callout %} +``` + +Avoid: + +- Raw framework components. +- JSX/Svelte syntax. +- Inline HTML that Markdoc may parse differently from GitHub. +- Unclosed `{% callout %}` tags. +- Mermaid diagrams that depend on unsupported runtime plugins. +- Anchor links guessed by hand. Prefer linking to the page when you cannot verify a heading fragment. + +The docs build can validate heading IDs and anchor fragments when a human explicitly asks for that check. Do not run it as a default agent step. + +## Moving or Renaming Pages + +Do not delete documentation files silently. If cleanup, reverting agent edits, moving content, or replacing docs would delete files, ask the user to confirm the exact file path(s) first. If the user does not confirm, keep the files and report them as cleanup candidates. + +When reorganizing documentation: + +1. Map old pages to their new homes before deleting anything. +2. Search for old paths and old page titles across the repo. +3. Update links in docs, source comments, package READMEs, and tests. +4. Use static link review unless the user explicitly asks for the docs build. +5. If a source-shape test asserts a documentation path, update the test with the new path. + +Do not leave references to deleted pages. Keep links defensible from static review unless a requested docs build validates the rendered site. + +## Updating the Docs Index + +The docs index should help readers choose a starting point. It does not need to list every file forever. + +Keep the index useful by: + +- Linking to the main entry points for each active domain. +- Describing what each domain is for. +- Letting the generated site sidebar provide the exhaustive file inventory. +- Removing links to pages that became archives, implementation memory, or stale planning notes. + +When a domain changes shape, update the index at the same time as the pages. Do not add a separate "using this directory" page just to tell agents to inspect a folder. + +## Verification + +Do not run the docs build after documentation changes unless the user explicitly asks for it. Use static review by default. + +When a docs build is explicitly requested, run it through the documented Docker-only workflow. Never run `pnpm docs:build` on the host. + +If documentation changes a tested source assertion, run the relevant test +through the allowed project test command only when the user explicitly asks for +tests in the current thread. For web UI tests, use Dockerized Vitest. Do not use +`svelte-check`, browsers, screenshots, or manual visual inspection as +substitutes for permitted tests. + +## Before Calling It Done + +Check these: + +- The docs describe the live code as the actual system. +- Normal docs do not use before/after framing, "current" headings, or old-vs-new commentary. +- Historical behavior appears only when the page is explicitly an archive, migration plan, changelog, or compatibility note. +- Links work in the generated site, not only on GitHub. +- Page names and headings are human-readable. +- The docs index still gives a good starting point. +- No tiny routing-only guide was added. +- No brittle counts, capacity promises, or exact file inventories were added unless they are intentionally part of the subject. diff --git a/README.md b/README.md index c2418c8..fad263c 100644 --- a/README.md +++ b/README.md @@ -1,259 +1,201 @@ -# @lixpi/markdown-stream-parser - -A library designed to incrementally parse Markdown text from a stream of tokens. - -It uses **tree-sitter** under the hood. Instead of regex pattern matching, we get a proper AST that tells us exactly what's a header, what's a code block, what's bold text, etc. Tree-sitter's error recovery also handles the imperfect markdown that LLMs tend to produce. +--- +title: Markdown Stream Parser +description: Incrementally parse streamed Markdown into render-agnostic text chunks, block context, inline spans, and recovery instructions. +--- -### ⚠️ ***This project is still in active development - there are bugs and missing features.*** +# Markdown Stream Parser -
+`@lixpi/markdown-stream-parser` incrementally parses Markdown from token streams. It emits plain rendered text with block context, inline span metadata, and correction offsets that a consumer can apply to its output buffer. -### DEMO: [markdown-stream-parser.lixpi.org](https://markdown-stream-parser.lixpi.org) +The parser uses the block and inline grammars from `tree-sitter-markdown`. It does not render HTML and does not depend on a UI framework. -
+The project is under active development. Review [Supported Markdown](#supported-markdown) and [Limitations](#limitations) before using it in a production rendering path. -![sample](https://github.com/user-attachments/assets/6e3525f7-9082-46e9-853b-90ee20447fe5) +- [Live demo](https://markdown-stream-parser.lixpi.org) +- [Repository](https://github.com/Lixpi/markdown-stream-parser) +![Markdown stream parser demo](https://github.com/user-attachments/assets/6e3525f7-9082-46e9-853b-90ee20447fe5) ## Installation -NPM: +Install the package with your package manager: + ```bash -pnpm i @lixpi/markdown-stream-parser -npm i @lixpi/markdown-stream-parser -yarn add @lixpi/markdown-stream-parser +pnpm add @lixpi/markdown-stream-parser ``` -Or just clone the repository and import it directly from the source. +```bash +npm install @lixpi/markdown-stream-parser +``` -### Importing +```bash +yarn add @lixpi/markdown-stream-parser +``` -The parser supports both ES6 module and CommonJS (Node.js) import styles. +ES modules and CommonJS entry points are declared by the package: -**ES6 import:** ```typescript import { MarkdownStreamParser } from '@lixpi/markdown-stream-parser' ``` -**CommonJS require:** -```typescript +```javascript const { MarkdownStreamParser } = require('@lixpi/markdown-stream-parser') ``` -Can be used on a backend or frontend, there's no rendering logic involved. +## WASM Assets +The parser needs three WASM files at runtime: -### Basic Concepts +- `tree-sitter.wasm` from `web-tree-sitter` +- `tree-sitter-markdown.wasm` +- `tree-sitter-markdown-inline.wasm` -- **Singleton Pattern (async):** - Use `await MarkdownStreamParser.getInstance(instanceId)` to ensure one parser per logical stream/session. The first call loads the tree-sitter WASM grammars, so `getInstance()` returns a `Promise`. +The package does not copy these assets into an application. Copy or serve them as part of your deployment before creating a parser instance. -- **Parsing Lifecycle:** - - `startParsing()`: Begin parsing and set up subscriptions. - - `parseToken(chunk: string)`: Feed incoming text chunks. - - `stopParsing()`: Flush buffers, reset state, and notify listeners of stream end. +In a browser, `web-tree-sitter` resolves its runtime as `/tree-sitter.wasm`. The Markdown grammars default to `/tree-sitter-markdown.wasm` and `/tree-sitter-markdown-inline.wasm`. -- **Subscribing to Output:** - Use `subscribeToTokenParse(listener)` to receive parsed chunks as soon as they are available. The listener receives a `StreamingChunk` and an `unsubscribe` function. Returns an unsubscribe function. +Call `configureWasmPath()` before the first call to `getInstance()` when the grammar files use different paths: +```typescript +MarkdownStreamParser.configureWasmPath( + '/parsers/tree-sitter-markdown.wasm', + '/parsers/tree-sitter-markdown-inline.wasm' +) +``` -## How to Use +If the second argument is omitted, the inline path is derived by replacing `.wasm` with `-inline.wasm` in the Markdown grammar path. -There are several ways to use the parser. It is quite modular. You can initialize it in one place and consume the parsed stream elsewhere, thanks to the singleton pattern. +In Node.js, the grammar defaults are `./wasm/tree-sitter-markdown.wasm` and `./wasm/tree-sitter-markdown-inline.wasm`. Pass filesystem paths to `configureWasmPath()` when the assets are stored elsewhere. -## Subscribing to the Parser +## Quick Start -Before you can parse the stream, you must subscribe to the parser. If you do not subscribe in advance, the parser will likely stop and terminate before you receive the first chunk. +Each logical stream uses an instance ID. `getInstance()` is asynchronous because the first call initializes `web-tree-sitter` and loads both grammars. -First import the parser and initialize it with an `instance-id`. (you can have as many parallel parsers as you want, just make sure to use different `instance-id`s) +Subscribe before calling `startParsing()`, feed string chunks with `parseToken()`, and finish with `stopParsing()` so buffered content is flushed. ```typescript -import { MarkdownStreamParser } from '@lixpi/markdown-stream-parser' +import { + MarkdownStreamParser, + type Chunk, +} from '@lixpi/markdown-stream-parser' -// Get a parser instance (singleton per ID) -// First call loads WASM grammars — subsequent calls for the same ID return instantly -const parser = await MarkdownStreamParser.getInstance('session-1') -``` +const instanceId = 'response-42' +const parser = await MarkdownStreamParser.getInstance(instanceId) -#### Approach 1: The Simplest +let renderedText = '' -```typescript -// Subscribe to parsed output -parser.subscribeToTokenParse((streamingChunk, unsubscribe) => { - console.log(streamingChunk) // Happy little parsed chunk - - // Clean up when the stream ends - if (streamingChunk.status === 'END_STREAM') { - unsubscribe() - MarkdownStreamParser.removeInstance('session-1') +const unsubscribe = parser.subscribeToTokenParse((event) => { + if (event.status === 'START_STREAM') { + renderedText = '' + return } -}) -``` -#### Approach 2: Customizable + if (event.status === 'END_STREAM') { + return + } -```typescript -// Subscribe to the parser -const parserUnsubscribe = parser.subscribeToTokenParse((streamingChunk) => { - console.log(streamingChunk) // Happy little parsed chunk + applyChunk(event.chunk) }) -// When the stream has ended stop the parser to avoid issues and memory leaks. -// You can decide when to terminate the parser. -// For example, using your own logic or rely on the `parser.parsing` flag. -if (!parser.parsing) { - parserUnsubscribe() // Unsubscribe from the parser - MarkdownStreamParser.removeInstance('session-1') // Dispose of the parser instance -} -``` - -## Parsing the Stream - -Regardless of which subscription method you choose, feeding the stream into the parser does not change. -Once the subscription to the parser is initialized, you can start parsing the stream. - -Again, this can be done in the same file or in a different part of your application. Just make sure to refer to the same parser `instance-id`. - -```typescript -import { MarkdownStreamParser } from '@lixpi/markdown-stream-parser' +function applyChunk(chunk: Chunk): void { + if (chunk.backtrackOffset !== undefined) { + renderedText = renderedText.slice(0, chunk.backtrackOffset) + } -// Get a parser instance (singleton per ID) -const parser = await MarkdownStreamParser.getInstance('session-1') + renderedText += chunk.text +} -// Start the parser parser.startParsing() -// Your iterator function here -for await (const chunk of ["Hello", " ~~world~~", "!", " \n"]) { - parser.parseToken(chunk) +for (const token of ['## ', 'Hello ', '**world**', '\n']) { + parser.parseToken(token) } -// Make sure to stop the parser at the end of the stream. It will flush any remaining content from the buffer. parser.stopParsing() +unsubscribe() +MarkdownStreamParser.removeInstance(instanceId) ``` -The output is a series of `StreamingChunk` objects. Each chunk contains the text content, UTF-16 offset from the start of the stream, block context, and **orthogonal span information**. +`getInstance()` returns the same parser for repeated calls with the same instance ID. Use different IDs for independent streams, and call `removeInstance()` when a stream no longer needs to be retained. -```javascript -{ status: 'START_STREAM' } -{ - status: 'STREAMING', - chunk: { - text: 'Hello ', - offset: 0, - length: 6, - block: { type: 'paragraph' }, - opening: [], // Spans that open but don't close in this chunk - closing: [], // Spans that close in this chunk (opened earlier) - contained: [] // Spans fully contained within this chunk - } -} -{ - status: 'STREAMING', - chunk: { - text: 'world', - offset: 6, - length: 5, - block: { type: 'paragraph' }, - opening: [], - closing: [], - contained: [ - { type: 'strikethrough', offset: 6, length: 5 } // ~~world~~ - ] - } -} -{ - status: 'STREAMING', - chunk: { - text: '! ', - offset: 11, - length: 3, - block: { type: 'paragraph' }, - opening: [], - closing: [], - contained: [] - } -} -{ status: 'END_STREAM' } -``` +## Stream Lifecycle -### Key Concepts +The subscriber receives a discriminated union: -- **`text`**: Plain text with markdown formatting syntax removed -- **`offset`**: UTF-16 code unit offset from the start of the stream -- **`length`**: UTF-16 code unit length of the text -- **`block`**: Block-level context (`paragraph`, `heading`, `code_block`, `list_item`, `table`, `blockquote`, etc.) -- **`opening`**: Spans that start in this chunk but don't close (span continues to next chunks) -- **`closing`**: Spans that close in this chunk (were opened in earlier chunks) -- **`contained`**: Spans fully contained within this chunk -- **`backtrackOffset`**: If present, the parser corrected previous output — discard everything from this offset onwards +```typescript +type StreamingChunk = + | { status: 'START_STREAM' } + | { status: 'STREAMING'; chunk: Chunk } + | { status: 'END_STREAM' } +``` +The lifecycle is: -## Consumer Span State Management +1. `subscribeToTokenParse(listener)` registers a listener and returns an unsubscribe function. +2. `startParsing()` resets accumulated parser state and emits `START_STREAM`. +3. `parseToken(chunk)` adds a string to the input buffer. Complete buffered segments may produce `STREAMING` events. +4. `stopParsing()` flushes buffered input and incomplete inline content, emits `END_STREAM`, and stops the session. +5. `removeInstance(instanceId)` stops and removes the retained parser instance. -The API uses an **orthogonal model** where chunks and spans are completely independent. Spans can cross chunk boundaries. Consumers must track open spans to properly render styled content. +Calling `parseToken()` before `startParsing()` returns an `Error`. Calling `startParsing()` while the parser is running leaves the active session in place. -### How to Track Span State +Multiple listeners can subscribe to one parser instance. Each listener receives the event and an unsubscribe function as arguments: ```typescript -import { MarkdownStreamParser, OpenSpan, ClosedSpan, Chunk } from '@lixpi/markdown-stream-parser' - -const parser = await MarkdownStreamParser.getInstance('session-1') +const unsubscribe = parser.subscribeToTokenParse((event, unsubscribeListener) => { + if (event.status === 'END_STREAM') { + unsubscribeListener() + } +}) +``` -// Track currently open spans -let openSpans: OpenSpan[] = [] +## Output Model -parser.subscribeToTokenParse((streamingChunk, unsubscribe) => { - if (streamingChunk.status === 'START_STREAM') { - openSpans = [] // Reset on new stream - return - } +A `STREAMING` event contains a `Chunk`: - if (streamingChunk.status === 'END_STREAM') { - unsubscribe() - return - } +```typescript +type Chunk = { + text: string + offset: number + length: number + block: BlockContext + opening: OpenSpan[] + closing: ClosedSpan[] + contained: ClosedSpan[] + backtrackOffset?: number + original?: string +} +``` - const chunk = streamingChunk.chunk +`text` contains rendered text with recognized Markdown syntax removed. `offset`, `length`, span positions, and `backtrackOffset` use UTF-16 code units in the rendered output coordinate space. This matches JavaScript string indexing and `String.prototype.slice()`. - // Handle backtracking (parser corrected previous output) - if (chunk.backtrackOffset !== undefined) { - // Remove content from offset `chunk.backtrackOffset` onwards - // Your render buffer should be truncated to this offset - // Also filter out any open spans that started after backtrackOffset - openSpans = openSpans.filter(s => s.openOffset < chunk.backtrackOffset!) - } +`block` describes the surrounding block: - // 1. Add new opening spans to our tracking list - openSpans.push(...chunk.opening) +```typescript +type BlockContext = { + type: + | 'paragraph' + | 'heading' + | 'code_block' + | 'list_item' + | 'table' + | 'table_row' + | 'table_cell' + | 'blockquote' + level?: number + language?: string +} +``` - // 2. Process closing spans (remove from tracking, render complete span) - for (const closingSpan of chunk.closing) { - // Find and remove the matching open span - const openIndex = openSpans.findIndex(s => s.type === closingSpan.type) - if (openIndex !== -1) { - openSpans.splice(openIndex, 1) - } - // Now you have a complete span with offset and length - // Use closingSpan.offset and closingSpan.length to apply styling - } +`level` applies to headings. `language` contains the info string detected on a fenced code block. - // 3. Contained spans are already complete (no tracking needed) - // Just apply their styling: contained.offset, contained.length +When `includeRawStreamedToken` is enabled, `original` contains the raw Markdown source associated with the emitted chunk. It is separate from the rendered UTF-16 coordinate space. - // 4. Render the chunk text with active styles - const activeStyles = [ - ...openSpans.map(s => s.type), - ...chunk.contained.map(s => s.type) - ] - renderText(chunk.text, chunk.block, activeStyles) -}) -``` +## Inline Spans -### Span Types +Chunks and inline spans are independent. A span can be contained by one chunk or cross chunk boundaries. ```typescript -// The base Span is a discriminated union — links carry URLs, images carry src/alt type Span = | { type: 'bold' } | { type: 'italic' } @@ -262,381 +204,212 @@ type Span = | { type: 'link'; url: string } | { type: 'image'; src: string; alt?: string } -type SpanType = Span['type'] // 'bold' | 'italic' | 'code' | 'strikethrough' | 'link' | 'image' - -// Opening span: we know where it starts, but it's not closed yet -type OpenSpan = { type: SpanType; openOffset: number } +type OpenSpan = { + type: Span['type'] + openOffset: number +} -// Closed/contained span: full span data + position info -type ClosedSpan = Span & { offset: number; length: number } +type ClosedSpan = Span & { + offset: number + length: number +} ``` -### Handling Backtracking +- `opening` lists spans that start in the chunk and remain open. +- `closing` lists spans that started in an earlier chunk and close in this chunk. +- `contained` lists complete spans represented within the chunk. -The parser may sometimes need to **correct** previously emitted chunks. This happens when tree-sitter reinterprets the content as more tokens arrive. - -When `chunk.backtrackOffset` is present: -1. **Discard content** from that offset onwards in your render buffer -2. **Filter open spans** to remove any that started after the backtrack offset -3. **Apply the new chunk** which contains the corrected content +Consumers that maintain active span state can match a closing span to an opening span by `type` and by comparing `OpenSpan.openOffset` with `ClosedSpan.offset`. ```typescript -if (chunk.backtrackOffset !== undefined) { - // Truncate your output buffer to backtrackOffset - outputBuffer = outputBuffer.slice(0, chunk.backtrackOffset) +import type { Chunk, OpenSpan } from '@lixpi/markdown-stream-parser' - // Remove spans that are no longer valid - openSpans = openSpans.filter(s => s.openOffset < chunk.backtrackOffset!) -} -``` +let openSpans: OpenSpan[] = [] -### Configuration Options +function updateSpanState(chunk: Chunk): void { + if (chunk.backtrackOffset !== undefined) { + openSpans = openSpans.filter( + (span) => span.openOffset < chunk.backtrackOffset! + ) + } -```typescript -// Optional: configure WASM grammar paths before creating any instance -// (only needed if you host the .wasm files at a non-default location) -MarkdownStreamParser.configureWasmPath('/custom/path/tree-sitter-markdown.wasm') + openSpans.push(...chunk.opening) -const parser = await MarkdownStreamParser.getInstance('session-1', { - windowSize: 500, // Lookback window for backtrack detection (UTF-16 code units) - includeRawStreamedToken: true // Include original markdown source in chunk.original -}) + for (const closed of chunk.closing) { + const index = openSpans.findIndex( + (open) => + open.type === closed.type && + open.openOffset === closed.offset + ) -// Or configure after creation -parser.setConfig({ windowSize: 1000 }) + if (index !== -1) { + openSpans.splice(index, 1) + } + } +} ``` -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `windowSize` | `number` | `undefined` (unlimited) | Requested lookback window in rendered UTF-16 code units. Recovery beyond this window does not yet have a strict public overflow contract; see [Known issues and limitations](#known-issues-and-limitations). | -| `includeRawStreamedToken` | `boolean` | `false` | When `true`, each chunk includes the original markdown source in `chunk.original`. Useful as a fallback for unsupported formatting. | +Links include their URL when closed. Images include `src` and may include `alt`. +## Error Recovery -## Is that it? What am I supposed to do with that? +Markdown structure can change as more source arrives. A line initially emitted as a paragraph can become part of a table or fenced code block, for example. -Good question. You can use this stream to render styled content in your application in real time. Having a `block type` and `span information` is enough to style it however you want. +When previously emitted output is affected, the first replacement chunk includes `backtrackOffset`. The consumer must: -It will **always remain `render-agnostic`** — whatever you use to render your styled text is entirely up to you. +1. Remove rendered output from `backtrackOffset` onward. +2. Remove derived block and span state at or after that offset. +3. Apply the replacement chunk and subsequent chunks in order. +```typescript +if (chunk.backtrackOffset !== undefined) { + output = output.slice(0, chunk.backtrackOffset) + openSpans = openSpans.filter( + (span) => span.openOffset < chunk.backtrackOffset! + ) +} -## Features +output += chunk.text +``` -- [x] Headers (`# H1`, `## H2`, etc.) -- [x] Paragraphs -- [x] Inline styles - - [x] Inline Italic (`*text*`) - - [x] Inline Bold (`**text**`) - - [x] Inline Bold & Italic (`***text***`) - - [x] Inline Strikethrough (`~~text~~`) - - [x] Inline Code (`` `code` ``) -- [x] Code Blocks (```` ```code-block``` ````) with language detection -- [x] Links (`[text](url)`) — with URL extraction *(no test coverage yet)* -- [x] Images (`![alt](url)`) — with src and alt extraction *(no test coverage yet)* -- [ ] Blockquotes (`> quote`) [Issue #2](https://github.com/Lixpi/markdown-stream-parser/issues/2) -- [ ] //TODO: PRIORITY: Ordered Lists (`1. item`) [Issue #3](https://github.com/Lixpi/markdown-stream-parser/issues/3) -- [ ] //TODO: PRIORITY: Unordered Lists (`- item`, `* item`, `+ item`) *BLOCKED BY:* [Issue #3](https://github.com/Lixpi/markdown-stream-parser/issues/3) -- [ ] //TODO: Task Lists (`- [ ] item`) *BLOCKED BY:* [Issue #3](https://github.com/Lixpi/markdown-stream-parser/issues/3) -- [ ] //TODO: PRIORITY: Tables [Issue #7](https://github.com/Lixpi/markdown-stream-parser/issues/7) -- [ ] //TODO: Horizontal Rules (`---`, `***`, `___`) -- [ ] //TODO: Footnotes -- [ ] //TODO: HTML blocks -- [ ] //TODO: Escaping (`\*literal asterisks\*`) -- [ ] //TODO: Automatic Links (``) -- [ ] //TODO: Emoji (`:smile:`) -- [ ] //TODO: Superscript (`x^2^`) -- [ ] //TODO: Subscript (`H~2~O`) +A correction may contain an empty `text` value when stale output must be deleted without replacement. Apply `backtrackOffset` even when `chunk.length` is zero. +## Configuration -## Running examples +Pass configuration when creating an instance or merge it into an existing instance with `setConfig()`: -To try out the parser with example streams, look inside the `demo/llm-streams-examples` directory. This folder contains real LLM responses collected from various providers. Each response has two versions: +```typescript +const parser = await MarkdownStreamParser.getInstance('response-42', { + includeRawStreamedToken: true, + windowSize: 500, +}) -- `*.json`: An array of items used for streaming -- `*.txt`: The same stream combined into a single file +parser.setConfig({ windowSize: 1000 }) +const config = parser.getConfig() +``` -Having the `*.txt` version is handy for visual comparison and debugging the parser. +| Option | Type | Default | Behavior | +| --- | --- | --- | --- | +| `includeRawStreamedToken` | `boolean` | `false` | Adds the associated raw Markdown source to `chunk.original`. | +| `windowSize` | `number` | `undefined` | Requests a maximum correction distance in rendered UTF-16 code units. See the recovery limitation below. | +`setConfig()` performs a shallow merge, so omitted properties retain their values. -Inside the repository root dir run: +## Supported Markdown -1. Start the Docker container: - ```bash - docker compose up -d - ``` +The parser handles these structures in its exercised parsing paths: -2. Run the tree-sitter debug parser inside the container: - ```bash - docker exec -it lixpi-markdown-stream-parser-demo pnpm run debug-parser-tree-sitter --file= - ``` +- Paragraphs and ATX headings (`#` through `######`) +- Fenced code blocks with language detection +- Ordered and unordered list items +- Bold, italic, bold-italic, strikethrough, and inline code spans +- Pipe-table cells and delimiter suppression for covered table forms - Replace `` with the relative path to any `.json` file. Examples: - - `--file=demo/llm-streams-examples/claude-3.5-1-quantum-physics.json` - - `--file=demo/llm-stream-examples-manually-simulated/long-consecutive-sequence.json` +Link and image span extraction is implemented, including URL and image metadata, but dedicated coverage is still needed for those paths. - > There's also `debug-parser` which runs the legacy state-machine parser for comparison. +These structures are incomplete or unsupported: -3. **Creating custom test streams**: You can also create your own chunked streams from arbitrary text files using the `split-sample-into-chunks` script: - ```bash - docker exec -it lixpi-markdown-stream-parser-demo pnpm run split-sample-into-chunks -- --file= --chunkSize= --outputPath= - ``` +- Blockquote marker stripping and nested blockquotes +- Full table behavior across all valid table shapes +- Task lists +- Horizontal rules +- Footnotes +- HTML blocks +- Autolinks +- Emoji shortcodes +- Superscript and subscript extensions - Example: - ```bash - docker exec -it lixpi-markdown-stream-parser-demo pnpm run split-sample-into-chunks -- --file=demo/llm-input-examples-raw-text/long-consecutive-sequence.txt --chunkSize=2 --outputPath=demo/llm-stream-examples-manually-simulated/long-consecutive-sequence.json - ``` +Escaped inline markers pass through the delimiter logic, but escaping behavior does not yet have complete feature coverage. -This will execute the parser against the selected example stream and print parsed chunks to the console. +## Limitations -## Running tests +### Recovery Window -The project includes comprehensive test coverage with 235+ tests across all core functionality. To run the tests: +`windowSize` is measured in rendered UTF-16 code units. The parser does not expose a recovery-limit event when the structurally valid checkpoint is older than the configured window. It may choose a later checkpoint to remain inside the window, which can omit part of a structural correction. -1. Start the Docker container: - ```bash - docker compose up -d - ``` +Leave `windowSize` undefined when a consumer requires complete recovery. A strict contract needs to select a checkpoint at or before the earliest affected source position and report or explicitly replace output when that checkpoint exceeds the consumer's window. -2. Run all tests: - ```bash - docker exec -it lixpi-markdown-stream-parser-demo pnpm test:run - ``` +### Recovery Coverage -3. Run tests in watch mode during development: - ```bash - docker exec -it lixpi-markdown-stream-parser-demo pnpm test - ``` +Recovery is covered for table and code-fence reclassification, rendered offsets, raw source output, and bounded lookback behavior. Dedicated cases are still needed for: ---- +- Inline delimiter replay with opening and closing spans +- Corrections that only delete stale rendered output +### Long Streams -## How It Works +Checkpoint history is copied as segments are emitted and searched linearly during recovery. Error detection also traverses the syntax tree after streamed input. These paths can accumulate disproportionate work as a document grows. -The parser uses **tree-sitter** for AST-based parsing. Instead of trying to match patterns with regex, we let tree-sitter build a syntax tree and then walk it to extract the content we need. +Long uninterrupted input is emitted in bounded chunks by the token buffer, but output can still be delayed until the buffer reaches its internal threshold or receives whitespace. -### High-Level Data Flow +## Runtime Design -```mermaid -%%{init: {'theme': 'base', 'themeVariables': { 'primaryColor': '#F6C7B3', 'primaryTextColor': '#5a3a2a', 'primaryBorderColor': '#d4956a', 'secondaryColor': '#C3DEDD', 'secondaryTextColor': '#1a3a47', 'secondaryBorderColor': '#4a8a9d', 'tertiaryColor': '#DCECE9', 'tertiaryTextColor': '#1a3a47', 'tertiaryBorderColor': '#82B2C0', 'lineColor': '#d4956a', 'textColor': '#5a3a2a'}}}%% -flowchart LR - A[LLM Token] --> B[TokensStreamBuffer] - B --> C[Accumulate Content] - C --> D[Tree-sitter Parse] - D --> E[AST Traversal] - E --> F[Emit Chunks] - F --> G[Subscribers] -``` - -### Module Architecture - -The tree-sitter parsing pipeline is organized into four layers. Each layer has a single direction of dependency — top layers call into lower layers, never the reverse. +The parser maintains one block syntax tree and one inline parser per instance. Incoming strings pass through a token buffer before incremental parsing. Generated chunks carry rendered offsets, while internal state also retains source offsets for replay. ```mermaid -%%{init: {'theme': 'base', 'themeVariables': { 'primaryColor': '#F6C7B3', 'primaryTextColor': '#5a3a2a', 'primaryBorderColor': '#d4956a', 'secondaryColor': '#C3DEDD', 'secondaryTextColor': '#1a3a47', 'secondaryBorderColor': '#4a8a9d', 'tertiaryColor': '#DCECE9', 'tertiaryTextColor': '#1a3a47', 'tertiaryBorderColor': '#82B2C0', 'lineColor': '#d4956a', 'textColor': '#5a3a2a'}}}%% -graph TB - subgraph "Public API" - Parser[MarkdownStreamParser
Singleton · Pub/Sub] - end - - subgraph "Stream Processing" - Backend[TreeSitterStreamParser
Parser backend] - Buffer[TokensStreamBuffer
Token accumulation] - end - - subgraph "Orchestration" - SegGen[SegmentGenerator
Chunk generation] - end - - subgraph "Analysis" - BD[BlockDetection
Block type classification] - ID[InlineDetection
Span detection] - CE[ContentExtraction
Syntax stripping] - SB[SegmentBuilder
Chunk/Span construction] - end - - subgraph "Foundation" - TN[TreeNavigation
AST traversal utilities] - Types[Types
Chunk · Span · BlockContext] - end - - subgraph "External Grammars" - TS[(web-tree-sitter)] - TS --> MD[(tree-sitter-markdown)] - TS --> MDI[(tree-sitter-markdown-inline)] - end - - Parser --> Backend - Backend --> Buffer - Backend --> SegGen - Backend -.-> TS - SegGen --> BD - SegGen --> ID - SegGen --> CE - SegGen --> SB - BD --> TN - ID --> TN +flowchart LR + A[Input strings] --> B[Token buffer] + B --> C[Incremental block parse] + C --> D[Block and inline analysis] + D --> E[Rendered chunks] + C --> F[Changed ranges and errors] + F --> G[Checkpoint replay] + G --> E ``` -> `types.ts` is a shared dependency imported by every module in the pipeline — arrows omitted for clarity. +Tree-sitter changed ranges and syntax errors identify source positions that may invalidate emitted output. Recovery selects a stored checkpoint, reconstructs generator state, and replays source from that checkpoint. Public offsets remain in rendered UTF-16 coordinates even though recovery decisions use raw source positions internally. -| Layer | Module | Responsibility | -|-------|--------|----------------| -| Stream Processing | `tree-sitter-markdown-stream-parser.ts` | Parser backend — manages tree-sitter lifecycle, drives the pipeline | -| Stream Processing | `tokens-stream-buffer.ts` | Accumulates incoming tokens into parseable content windows | -| Orchestration | `segment-generator.ts` | Central orchestrator — generates chunks with block context and span info | -| Analysis | `block-detection.ts` | Classifies block type: `heading`, `paragraph`, `code_block`, `list_item`, `table` | -| Analysis | `inline-detection.ts` | Detects inline spans: bold, italic, code, strikethrough, link, image | -| Analysis | `content-extraction.ts` | Strips markdown delimiters, extracts clean text content | -| Analysis | `segment-builder.ts` | Constructs `Chunk` and `Span` objects with UTF-16 offsets | -| Foundation | `tree-navigation.ts` | AST traversal — finds nodes at positions, walks inline trees | -| Foundation | `types.ts` | Shared type definitions: `Chunk`, `Span`, `BlockContext`, `SpanType` | +## Development -### Parser API Flow +The repository's Docker service installs the root and demo dependencies. Start it from the repository root: -```mermaid -%%{init: {'theme': 'base', 'themeVariables': { 'noteBkgColor': '#82B2C0', 'noteTextColor': '#1a3a47', 'noteBorderColor': '#5a9aad', 'actorBkg': '#F6C7B3', 'actorBorder': '#d4956a', 'actorTextColor': '#5a3a2a', 'actorLineColor': '#d4956a', 'signalColor': '#d4956a', 'signalTextColor': '#5a3a2a', 'labelBoxBkgColor': '#F6C7B3', 'labelBoxBorderColor': '#d4956a', 'labelTextColor': '#5a3a2a', 'loopTextColor': '#5a3a2a', 'activationBorderColor': '#9DC49D', 'activationBkgColor': '#9DC49D', 'sequenceNumberColor': '#5a3a2a'}}}%% -sequenceDiagram - participant App as Your App - participant Parser as MarkdownStreamParser - participant Buffer as TokensStreamBuffer - participant TS as Tree-sitter - participant Gen as SegmentGenerator - - %% ═══════════════════════════════════════════════════════════════ - %% SETUP PHASE - %% ═══════════════════════════════════════════════════════════════ - rect rgb(220, 236, 233) - Note over App, Gen: PHASE 1 - Setup - App->>Parser: getInstance(sessionId, config?) - activate Parser - Parser->>TS: load WASM grammars - activate TS - TS-->>Parser: grammars loaded - deactivate TS - Parser-->>App: parser instance - deactivate Parser - end - - %% ═══════════════════════════════════════════════════════════════ - %% SUBSCRIPTION PHASE - %% ═══════════════════════════════════════════════════════════════ - rect rgb(195, 222, 221) - Note over App, Gen: PHASE 2 - Subscription - App->>Parser: subscribeToTokenParse(listener) - activate Parser - App->>Parser: startParsing() - Parser-->>App: START_STREAM event - deactivate Parser - end - - %% ═══════════════════════════════════════════════════════════════ - %% STREAMING PHASE - %% ═══════════════════════════════════════════════════════════════ - rect rgb(246, 199, 179) - Note over App, Gen: PHASE 3 - Streaming - loop For each LLM token - App->>Parser: parseToken(chunk) - activate Parser - Parser->>Buffer: receiveChunk(chunk) - activate Buffer - Buffer-->>Parser: content ready - deactivate Buffer - Parser->>TS: parse(content) - activate TS - TS-->>Parser: AST - deactivate TS - Parser->>Gen: generateSegments(range, state) - activate Gen - Gen-->>Parser: Chunk[] with spans - deactivate Gen - Parser-->>App: notify(StreamingChunk) - deactivate Parser - end - end - - %% ═══════════════════════════════════════════════════════════════ - %% CLEANUP PHASE - %% ═══════════════════════════════════════════════════════════════ - rect rgb(242, 234, 224) - Note over App, Gen: PHASE 4 - Cleanup - App->>Parser: stopParsing() - activate Parser - Parser->>Buffer: flushBuffer() - activate Buffer - Buffer-->>Parser: buffer flushed - deactivate Buffer - Parser-->>App: END_STREAM event - deactivate Parser - App->>Parser: removeInstance(sessionId) - end +```bash +docker compose up -d ``` +Run the test suite in the service container: -### How Content Gets Processed - -#### 1. Token Buffering - -Incoming tokens are accumulated in a `TokensStreamBuffer`. The buffer waits for word boundaries (whitespace) before emitting, so the parser always has enough context to produce meaningful chunks rather than character-by-character. - -#### 2. AST-Based Parsing - -The core parsing is done by `web-tree-sitter` with the `tree-sitter-markdown` grammar. When content comes in, we parse it incrementally (editing the existing tree) and get an AST that tells us exactly what we're dealing with — headers, paragraphs, code blocks, lists, bold text, etc. - -Tree-sitter handles incomplete/malformed markdown gracefully. It uses error recovery and can still produce a usable tree even when the input is partial or slightly broken (which happens constantly with LLM streams). - -#### 3. Handling Incomplete Inline Markers - -A tricky problem with streaming is that inline markers can arrive split across chunks. For example, you might get `**hello` in one chunk and `**` in the next. - -The parser buffers content when it detects an unmatched delimiter. It checks whether the inline tree-sitter parser can see a complete structure (emphasis, code_span, etc.). If not, the content is held in `pendingInlineContent` and the parser waits for more tokens before emitting. - -This applies to inline code (`` ` ``), bold (`**`), italic (`*` or `_`), and strikethrough (`~~`). - -#### 4. Two-Parser Approach - -For inline content within blocks, we use a second tree-sitter parser with the `tree-sitter-markdown-inline` grammar. This gives us detailed AST info about emphasis delimiters, code spans, etc. +```bash +docker exec -it lixpi-markdown-stream-parser-demo pnpm test:run +``` -The two-parser approach (one for block structure, one for inline content) is how tree-sitter-markdown is designed to work. It lets us accurately detect things like whether a `*` is actually an italic marker or just a literal asterisk. +Run the package build: -### Pub/Sub and Singleton Patterns +```bash +docker exec -it lixpi-markdown-stream-parser-demo pnpm run build +``` -The parser uses a **publish/subscribe** pattern — you subscribe to get parsed chunks as they're ready. Parsing is decoupled from rendering, and multiple subscribers per parser instance are supported. +### Debug a Recorded Stream -Each logical stream gets its own parser instance via `await getInstance(instanceId)` (singleton pattern). This allows parallel processing of multiple streams without state conflicts. +Recorded streams live under `demo/llm-streams-examples`. JSON files preserve chunk boundaries; matching text files provide the combined Markdown for comparison. -```typescript -const parser = await MarkdownStreamParser.getInstance('session-1') -// ... use the parser ... -MarkdownStreamParser.removeInstance('session-1') // cleanup when done +```bash +docker exec -it lixpi-markdown-stream-parser-demo \ + pnpm run debug-parser-tree-sitter \ + --file=demo/llm-streams-examples/claude-3.5-long-regex.json ``` ---- - +Create a chunked JSON stream from a text fixture: -## Known issues and limitations - -- **Delayed processing for extremely long sequences of characters without whitespace**: Due to how token buffering works, extremely long uninterrupted sequences (like a huge regex) can delay output until the sequence completes. In practice this is rarely noticeable with modern LLM speeds, but it can happen. -- **Recovery beyond `windowSize` is not strictly defined yet**: `windowSize` is measured against rendered UTF-16 output, but the parser does not currently emit a dedicated overflow event when the structurally correct recovery point is older than the configured window. Consumers that require guaranteed correction should leave `windowSize` undefined until an explicit overflow/fallback contract is implemented. -- **Inline-delimiter replay needs more coverage**: Split inline delimiters are buffered during normal streaming, but recovery tests do not yet fully assert opening and closing span state when already-emitted inline content is replayed. -- **Deletion-only recovery needs direct coverage**: The parser can emit a zero-length correction chunk when stale rendered output must be removed without replacement, but this path does not yet have a dedicated recovery test. -- **Some Markdown structures remain incomplete**: Blockquotes and full table behavior still have skipped feature tests and are tracked in the feature list above. +```bash +docker exec -it lixpi-markdown-stream-parser-demo \ + pnpm run split-sample-into-chunks -- \ + --file=demo/llm-input-examples-raw-text/long-consecutive-sequence.txt \ + --chunkSize=2 \ + --outputPath=demo/llm-stream-examples-manually-simulated/long-consecutive-sequence.json +``` ---- +## Development Priorities +Recovery work focuses on a strict `windowSize` overflow contract, inline-span replay coverage, and deletion-only correction coverage. -## Contributions and Roadmap +Scaling work focuses on stable-boundary checkpoints, pruning and indexed lookup, parser-internal checkpoint storage, changed-subtree error inspection, tracked unresolved errors, and long-stream benchmarks. -- **Contributions:** - PRs and issues are *welcome*! Feel free to share your thoughts in **[discussions](https://github.com/Lixpi/markdown-stream-parser/discussions)**. +Markdown coverage work focuses on the incomplete structures listed in [Supported Markdown](#supported-markdown). -- **Roadmap:** - - Support for the missing markdown features listed earlier - - Define a strict `windowSize` overflow contract. Recovery must select a checkpoint at or before the earliest structurally affected source position and must never silently choose a later checkpoint merely to fit the window. The intended API should report that the recovery limit was exceeded or use an explicitly configured fallback such as a full snapshot replacement. - - Add recovery coverage for inline delimiter replay, including opening and closing spans, and for deletion-only corrections that emit a zero-length chunk. - - Improve checkpoint scaling: create checkpoints only at stable block boundaries or configured intervals, move checkpoint storage out of copied generator state, prune history outside the supported recovery range, retain an older baseline when unlimited recovery is enabled, and use indexed/binary-search lookup by source and rendered offsets. - - Limit error inspection to tree-sitter changed ranges, their containing blocks, a small surrounding recovery region, and explicitly tracked unresolved errors instead of recursively scanning the complete syntax tree after every token. - - Add long-stream benchmarks and recovery correctness tests to prevent quadratic checkpoint-copying and full-tree-scan regressions. +## Contributing ---- +Bug reports, implementation proposals, and pull requests are welcome. Use [GitHub Discussions](https://github.com/Lixpi/markdown-stream-parser/discussions) for design questions and the repository issue tracker for reproducible defects. ## License From 5379e20d577c02df9aa8da558ed957a575a9969a Mon Sep 17 00:00:00 2001 From: Dmitry Bondarenko Date: Wed, 24 Jun 2026 23:12:07 +0600 Subject: [PATCH 27/32] Implements the strict windowSize overflow recovery contract --- README.md | 22 ++++- demo/svelte-demo/src/routes/+page.svelte | 8 ++ src/markdown-stream-parser.ts | 1 + ...tree-sitter-markdown-stream-parser.test.ts | 66 +++++++++++++- src/tree-sitter-markdown-stream-parser.ts | 85 ++++++++++++++----- src/tree-sitter/segment-builder.ts | 4 + src/tree-sitter/types.ts | 10 +++ 7 files changed, 171 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index fad263c..b67b461 100644 --- a/README.md +++ b/README.md @@ -163,8 +163,16 @@ type Chunk = { closing: ClosedSpan[] contained: ClosedSpan[] backtrackOffset?: number + recovery?: RecoveryInfo original?: string } + +type RecoveryInfo = { + type: 'window_overflow' + windowSize: number + fullBacktrackOffset: number + appliedBacktrackOffset: number +} ``` `text` contains rendered text with recognized Markdown syntax removed. `offset`, `length`, span positions, and `backtrackOffset` use UTF-16 code units in the rendered output coordinate space. This matches JavaScript string indexing and `String.prototype.slice()`. @@ -293,7 +301,7 @@ const config = parser.getConfig() | `includeRawStreamedToken` | `boolean` | `false` | Adds the associated raw Markdown source to `chunk.original`. | | `windowSize` | `number` | `undefined` | Requests a maximum correction distance in rendered UTF-16 code units. See the recovery limitation below. | -`setConfig()` performs a shallow merge, so omitted properties retain their values. +`windowSize` must be finite and greater than or equal to `0`; invalid values throw `RangeError`. `setConfig()` performs a shallow merge, so omitted properties retain their values. ## Supported Markdown @@ -325,9 +333,17 @@ Escaped inline markers pass through the delimiter logic, but escaping behavior d ### Recovery Window -`windowSize` is measured in rendered UTF-16 code units. The parser does not expose a recovery-limit event when the structurally valid checkpoint is older than the configured window. It may choose a later checkpoint to remain inside the window, which can omit part of a structural correction. +`windowSize` is measured in rendered UTF-16 code units. When a complete correction would require replay before `lastEmittedOffset - windowSize`, the parser chooses the earliest stored checkpoint inside the configured window and attaches recovery metadata to the first replacement chunk: + +```typescript +if (chunk.recovery?.type === 'window_overflow') { + console.warn('Correction was truncated to the configured window', chunk.recovery) +} +``` + +`recovery.fullBacktrackOffset` is where complete replay would have started. `recovery.appliedBacktrackOffset` equals `chunk.backtrackOffset` and is the bounded offset consumers should apply. Output before `appliedBacktrackOffset` may remain stale because the parser did not ask the consumer to discard outside the configured window. -Leave `windowSize` undefined when a consumer requires complete recovery. A strict contract needs to select a checkpoint at or before the earliest affected source position and report or explicitly replace output when that checkpoint exceeds the consumer's window. +Leave `windowSize` undefined when a consumer requires complete recovery. ### Recovery Coverage diff --git a/demo/svelte-demo/src/routes/+page.svelte b/demo/svelte-demo/src/routes/+page.svelte index 978842d..5eb252c 100644 --- a/demo/svelte-demo/src/routes/+page.svelte +++ b/demo/svelte-demo/src/routes/+page.svelte @@ -145,6 +145,7 @@ backtrackOffset: chunk.backtrackOffset, chunkText: chunk.text, chunkOffset: chunk.offset, + recovery: chunk.recovery, discarding: parsedSegments .filter( (seg) => @@ -157,6 +158,13 @@ ), }); + if (chunk.recovery?.type === "window_overflow") { + console.warn( + "⚠️ Recovery exceeded windowSize; only the bounded suffix was replaced.", + chunk.recovery, + ); + } + parsedSegments = parsedSegments.filter((seg) => { if (seg.status !== "STREAMING") return true; return ( diff --git a/src/markdown-stream-parser.ts b/src/markdown-stream-parser.ts index cfa0e88..b1847c4 100644 --- a/src/markdown-stream-parser.ts +++ b/src/markdown-stream-parser.ts @@ -8,6 +8,7 @@ export type { BlockType, BlockContext, Chunk, + RecoveryInfo, StreamingChunk, ParserConfig, } from './tree-sitter-markdown-stream-parser.ts' diff --git a/src/tree-sitter-markdown-stream-parser.test.ts b/src/tree-sitter-markdown-stream-parser.test.ts index 379185c..680b1fb 100644 --- a/src/tree-sitter-markdown-stream-parser.test.ts +++ b/src/tree-sitter-markdown-stream-parser.test.ts @@ -877,7 +877,7 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { it('should respect windowSize configuration', async () => { const windowInstanceId = 'test-window-size' const windowParser = await MarkdownStreamParser.getInstance(windowInstanceId, { - windowSize: 5, + windowSize: 500, }) const windowChunks: Chunk[] = [] @@ -904,13 +904,75 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { const lastEmitted = Math.max(...priorChunks.map(c => c.offset + c.length)) const distance = lastEmitted - btChunk.backtrackOffset! // The backtrack distance should not exceed windowSize - expect(distance).toBeLessThanOrEqual(5) + expect(distance).toBeLessThanOrEqual(500) } + + expect(btChunk.recovery).toBeUndefined() } MarkdownStreamParser.removeInstance(windowInstanceId) }) + it('should report recovery metadata when required correction exceeds windowSize', async () => { + const windowInstanceId = 'test-window-size-overflow' + const windowParser = await MarkdownStreamParser.getInstance(windowInstanceId, { + windowSize: 256, + }) + + const windowChunks: Chunk[] = [] + windowParser.subscribeToTokenParse((chunk) => { + if (chunk.status === 'STREAMING' && chunk.chunk) { + windowChunks.push(chunk.chunk) + } + }) + + windowParser.startParsing() + + const columnCount = 40 + const headerCells = Array.from({ length: columnCount }, (_, i) => `column${i}`).join(' | ') + const delimiterCells = Array.from({ length: columnCount }, () => '---').join(' | ') + windowParser.parseToken(`| ${headerCells} |\n`) + windowParser.parseToken(`| ${delimiterCells} |\n`) + windowParser.stopParsing() + + const overflowChunk = windowChunks.find(c => c.recovery?.type === 'window_overflow') + expect(overflowChunk).toBeDefined() + expect(overflowChunk?.backtrackOffset).toBeDefined() + + const overflowIdx = windowChunks.indexOf(overflowChunk!) + const priorChunks = windowChunks.slice(0, overflowIdx).filter(c => c.backtrackOffset === undefined) + expect(priorChunks.length).toBeGreaterThan(0) + + const lastEmitted = Math.max(...priorChunks.map(c => c.offset + c.length)) + expect(overflowChunk!.backtrackOffset!).toBeGreaterThanOrEqual(lastEmitted - 256) + expect(overflowChunk!.recovery).toMatchObject({ + type: 'window_overflow', + windowSize: 256, + appliedBacktrackOffset: overflowChunk!.backtrackOffset, + }) + expect(overflowChunk!.recovery!.fullBacktrackOffset).toBeLessThan(overflowChunk!.recovery!.appliedBacktrackOffset) + + MarkdownStreamParser.removeInstance(windowInstanceId) + }) + + it('should reject invalid windowSize configuration', async () => { + await expect(MarkdownStreamParser.getInstance('test-window-size-negative', { + windowSize: -1, + })).rejects.toThrow(RangeError) + + await expect(MarkdownStreamParser.getInstance('test-window-size-nan', { + windowSize: NaN, + })).rejects.toThrow(RangeError) + + await expect(MarkdownStreamParser.getInstance('test-window-size-infinity', { + windowSize: Infinity, + })).rejects.toThrow(RangeError) + + expect(() => parser.setConfig({ windowSize: -1 })).toThrow(RangeError) + expect(() => parser.setConfig({ windowSize: NaN })).toThrow(RangeError) + expect(() => parser.setConfig({ windowSize: Infinity })).toThrow(RangeError) + }) + it('should backtrack using rendered offsets after preceding markdown syntax', async () => { const recoveryId = 'test-rendered-backtrack-offset' const recoveryParser = await MarkdownStreamParser.getInstance(recoveryId) diff --git a/src/tree-sitter-markdown-stream-parser.ts b/src/tree-sitter-markdown-stream-parser.ts index 6439ed1..3f481d0 100644 --- a/src/tree-sitter-markdown-stream-parser.ts +++ b/src/tree-sitter-markdown-stream-parser.ts @@ -1,8 +1,20 @@ import { Parser, Language, type Tree, type Node } from 'web-tree-sitter' import TokensStreamBuffer from './tokens-stream-buffer.ts' -import type { StreamingChunk, ParserConfig, SegmentGeneratorState } from './tree-sitter/types.ts' +import type { + Chunk, + RecoveryInfo, + StreamingChunk, + ParserConfig, + SegmentGeneratorCheckpoint, + SegmentGeneratorState +} from './tree-sitter/types.ts' import { generateSegments, createInitialState, stateFromCheckpoint } from './tree-sitter/segment-generator.ts' +type RecoverySelection = { + requiredCheckpoint: SegmentGeneratorCheckpoint + appliedCheckpoint: SegmentGeneratorCheckpoint + recovery?: RecoveryInfo +} // Re-export types for external consumers export type { @@ -13,6 +25,7 @@ export type { BlockType, BlockContext, Chunk, + RecoveryInfo, StreamingChunk, ParserConfig } from './tree-sitter/types.ts' @@ -74,6 +87,8 @@ export class MarkdownStreamParser { // instanceId - Unique identifier for the parser instance // config - Optional parser configuration static async getInstance(instanceId: string, config?: ParserConfig): Promise { + MarkdownStreamParser.validateConfig(config) + // Initialize parser and language once for all instances if (!MarkdownStreamParser.parserInitialized) { if (!MarkdownStreamParser.parserInitPromise) { @@ -85,7 +100,7 @@ export class MarkdownStreamParser { if (!MarkdownStreamParser.instances.has(instanceId)) { const instance = new MarkdownStreamParser() if (config) { - instance.config = config + instance.config = { ...config } } await instance.initialize() MarkdownStreamParser.instances.set(instanceId, instance) @@ -170,6 +185,16 @@ export class MarkdownStreamParser { } } + private static validateConfig(config?: ParserConfig): void { + if (config?.windowSize === undefined) { + return + } + + if (!Number.isFinite(config.windowSize) || config.windowSize < 0) { + throw new RangeError('windowSize must be a finite number greater than or equal to 0') + } + } + constructor() { this.tokensStreamProcessor = new TokensStreamBuffer() } @@ -193,7 +218,9 @@ export class MarkdownStreamParser { // Update parser configuration. // config - New parser configuration setConfig(config: ParserConfig): void { - this.config = { ...this.config, ...config } + const nextConfig = { ...this.config, ...config } + MarkdownStreamParser.validateConfig(nextConfig) + this.config = nextConfig } // Get current parser configuration. @@ -367,7 +394,8 @@ export class MarkdownStreamParser { } if (affectedSourceOffset !== undefined) { - const checkpoint = this.findRecoveryCheckpoint(affectedSourceOffset) + const recoverySelection = this.selectRecoveryCheckpoint(affectedSourceOffset) + const checkpoint = recoverySelection.appliedCheckpoint let state: SegmentGeneratorState = stateFromCheckpoint(checkpoint) let backtrackOffset = checkpoint.renderedOffset @@ -401,6 +429,7 @@ export class MarkdownStreamParser { const firstSeg = allBacktrackSegments[0] if (firstSeg.status === 'STREAMING' && firstSeg.chunk) { firstSeg.chunk.backtrackOffset = backtrackOffset + firstSeg.chunk.recovery = recoverySelection.recovery } } else if (backtrackOffset < this.generatorState.lastEmittedOffset) { allBacktrackSegments.push({ @@ -414,6 +443,7 @@ export class MarkdownStreamParser { closing: [], contained: [], backtrackOffset, + recovery: recoverySelection.recovery, } }) } @@ -442,8 +472,8 @@ export class MarkdownStreamParser { return result.segments } - private findRecoveryCheckpoint(sourceOffset: number): SegmentGeneratorState['checkpoints'][number] { - const baseCheckpoint: SegmentGeneratorState['checkpoints'][number] = { + private selectRecoveryCheckpoint(sourceOffset: number): RecoverySelection { + const baseCheckpoint: SegmentGeneratorCheckpoint = { sourceOffset: 0, renderedOffset: 0, lastEmittedSourceOffset: 0, @@ -455,29 +485,44 @@ export class MarkdownStreamParser { } const checkpoints = this.generatorState.checkpoints.length > 0 - ? this.generatorState.checkpoints + ? [baseCheckpoint, ...this.generatorState.checkpoints] : [baseCheckpoint] - let checkpoint = baseCheckpoint + let requiredCheckpoint = baseCheckpoint for (const candidate of checkpoints) { - if (candidate.sourceOffset <= sourceOffset && candidate.sourceOffset >= checkpoint.sourceOffset) { - checkpoint = candidate + if (candidate.sourceOffset <= sourceOffset && candidate.sourceOffset >= requiredCheckpoint.sourceOffset) { + requiredCheckpoint = candidate } } - if (this.config.windowSize !== undefined) { - const windowStart = Math.max(0, this.generatorState.lastEmittedOffset - this.config.windowSize) - if (checkpoint.renderedOffset < windowStart) { - for (const candidate of checkpoints) { - if (candidate.renderedOffset >= windowStart) { - checkpoint = candidate - break - } - } + if (this.config.windowSize === undefined) { + return { + requiredCheckpoint, + appliedCheckpoint: requiredCheckpoint, } } - return checkpoint + const windowStart = Math.max(0, this.generatorState.lastEmittedOffset - this.config.windowSize) + if (requiredCheckpoint.renderedOffset >= windowStart) { + return { + requiredCheckpoint, + appliedCheckpoint: requiredCheckpoint, + } + } + + const appliedCheckpoint = checkpoints.find(candidate => candidate.renderedOffset >= windowStart) + ?? checkpoints[checkpoints.length - 1] + + return { + requiredCheckpoint, + appliedCheckpoint, + recovery: { + type: 'window_overflow', + windowSize: this.config.windowSize, + fullBacktrackOffset: requiredCheckpoint.renderedOffset, + appliedBacktrackOffset: appliedCheckpoint.renderedOffset, + }, + } } private findEarliestErrorOffset(): number | undefined { diff --git a/src/tree-sitter/segment-builder.ts b/src/tree-sitter/segment-builder.ts index 4359cb7..55cc18c 100644 --- a/src/tree-sitter/segment-builder.ts +++ b/src/tree-sitter/segment-builder.ts @@ -6,6 +6,7 @@ import type { BlockType, OpenSpan, ClosedSpan, + RecoveryInfo, ParserConfig } from './types.ts' @@ -110,6 +111,7 @@ export function createChunk( closing?: ClosedSpan[] contained?: ClosedSpan[] backtrackOffset?: number + recovery?: RecoveryInfo original?: string } ): Chunk { @@ -122,6 +124,7 @@ export function createChunk( closing: options?.closing ?? [], contained: options?.contained ?? [], backtrackOffset: options?.backtrackOffset, + recovery: options?.recovery, original: options?.original, } } @@ -144,6 +147,7 @@ export function createChunkFromBlockInfo( closing?: ClosedSpan[] contained?: ClosedSpan[] backtrackOffset?: number + recovery?: RecoveryInfo original?: string } ): StreamingChunk { diff --git a/src/tree-sitter/types.ts b/src/tree-sitter/types.ts index 65c97fa..c5352ed 100644 --- a/src/tree-sitter/types.ts +++ b/src/tree-sitter/types.ts @@ -91,11 +91,21 @@ export type Chunk = { // offset onwards and replace with this chunk + subsequent chunks. backtrackOffset?: number + // Present when a configured recovery window prevented complete replay. + recovery?: RecoveryInfo + // Original markdown source (only if includeRawStreamedToken config is true). // Useful as fallback when parser messes up or for unsupported formats. original?: string } +export type RecoveryInfo = { + type: 'window_overflow' + windowSize: number + fullBacktrackOffset: number + appliedBacktrackOffset: number +} + // Stream status wrapper for chunks. export type StreamingChunk = | { status: 'STREAMING'; chunk: Chunk } From ebad7ef41a4a011794d07a7215e1c46147713cec Mon Sep 17 00:00:00 2001 From: Dmitry Bondarenko Date: Wed, 1 Jul 2026 22:20:04 +0600 Subject: [PATCH 28/32] Improves lists parsing --- README.md | 12 +- demo/svelte-demo/src/routes/+page.svelte | 16 +- lists-support-plan.md | 124 ++++++++++ ...tree-sitter-markdown-stream-parser.test.ts | 232 +++++++++++++++++ src/tree-sitter/block-detection.ts | 24 +- src/tree-sitter/content-extraction.ts | 4 +- src/tree-sitter/list-support.ts | 233 ++++++++++++++++++ src/tree-sitter/segment-builder.ts | 3 + src/tree-sitter/segment-generator.ts | 87 +++++-- src/tree-sitter/types.ts | 12 + 10 files changed, 709 insertions(+), 38 deletions(-) create mode 100644 lists-support-plan.md create mode 100644 src/tree-sitter/list-support.ts diff --git a/README.md b/README.md index b67b461..c4284f1 100644 --- a/README.md +++ b/README.md @@ -192,11 +192,20 @@ type BlockContext = { | 'blockquote' level?: number language?: string + list?: { + type: 'ordered' | 'unordered' + depth: number + marker: '-' | '+' | '*' | '.' | ')' + ordinal?: number + task?: { checked: boolean } + } } ``` `level` applies to headings. `language` contains the info string detected on a fenced code block. +`list` is present when the chunk is inside a list item. `depth` is zero-based. Unordered items use `marker` for the bullet character (`-`, `+`, or `*`). Ordered items use `marker` for the delimiter only (`.` or `)`) and put the number in `ordinal` when it is safely representable as a JavaScript number. Task list items omit the `[x]`, `[X]`, or `[ ]` marker from rendered text and expose `task.checked`. + When `includeRawStreamedToken` is enabled, `original` contains the raw Markdown source associated with the emitted chunk. It is separate from the rendered UTF-16 coordinate space. ## Inline Spans @@ -309,7 +318,7 @@ The parser handles these structures in its exercised parsing paths: - Paragraphs and ATX headings (`#` through `######`) - Fenced code blocks with language detection -- Ordered and unordered list items +- Ordered, unordered, nested, loose, and task list items - Bold, italic, bold-italic, strikethrough, and inline code spans - Pipe-table cells and delimiter suppression for covered table forms @@ -319,7 +328,6 @@ These structures are incomplete or unsupported: - Blockquote marker stripping and nested blockquotes - Full table behavior across all valid table shapes -- Task lists - Horizontal rules - Footnotes - HTML blocks diff --git a/demo/svelte-demo/src/routes/+page.svelte b/demo/svelte-demo/src/routes/+page.svelte index 5eb252c..7613ed1 100644 --- a/demo/svelte-demo/src/routes/+page.svelte +++ b/demo/svelte-demo/src/routes/+page.svelte @@ -331,11 +331,8 @@ } else if (blockType === "heading" && blockLevel !== lastBlockLevel) { isNewBlock = true; } else if (blockType === "list_item" && lastOffset >= 0) { - // New list item if there's a significant gap in offset (indicates newline/new item) - // Or if the text starts after a newline marker - const gap = chunk.offset - lastOffset; - if (gap > 50) { - // Heuristic: large gap suggests new list item + const previousChunk = currentBlock[currentBlock.length - 1]; + if (previousChunk?.text.endsWith("\n") && chunk.text.trim().length > 0) { isNewBlock = true; } } @@ -666,8 +663,15 @@ {/each} {:else if blockType === "list_item"} + {@const list = block[0]?.block.list} - • + + {#if list?.type === "ordered"} + {list.ordinal ?? ""}{list.marker} + {:else} + • + {/if} + {#each block as chunk} {@const styles = [ ...chunk.contained.map((s) => s.type), diff --git a/lists-support-plan.md b/lists-support-plan.md new file mode 100644 index 0000000..3ae9705 --- /dev/null +++ b/lists-support-plan.md @@ -0,0 +1,124 @@ +# Reliable List Support + +## Summary + +Add robust ordered, unordered, nested, loose, and task-list handling to the tree-sitter parser. Keep `block.type === 'list_item'` for list item text, and add optional list metadata so consumers can render bullets, numbers, nesting, and checked state without parsing raw Markdown. + +## Public API Changes + +Extend `BlockContext` with optional metadata: + +```ts +list?: { + type: 'ordered' | 'unordered' + depth: number + marker: '-' | '+' | '*' | '.' | ')' + ordinal?: number + task?: { checked: boolean } +} +``` + +- `depth` is zero-based: top-level list items use `0`, nested list items use `1+`. Compute it as the number of enclosing `list` ancestors minus one. +- For **unordered** lists, `marker` is the bullet character (`-`, `+`, `*`) and `ordinal` is absent. +- For **ordered** lists, `marker` is the *delimiter only* (`.` or `)`); the number lives in `ordinal` when it is safely representable as a JavaScript number. This is documented explicitly because `marker` alone does not reconstruct the source (`"1."` = `ordinal: 1` + `marker: '.'`). +- Task list items strip `[x]`, `[X]`, or `[ ]` (and the following space) from rendered text and expose `task.checked`. + +## Grammar Facts (verified against the bundled WASM) + +These node shapes drive the design and were confirmed by dumping the AST for representative inputs: + +``` +"- [x] done\n" + list_item + list_marker_minus [0,2] "- " + task_list_marker_checked [2,5] "[x]" ← node covers "[x]" only, not the trailing space + paragraph [6,11] "done\n" ← index 5 (the space) belongs to NO node + inline [6,10] "done" + +"- Parent\n - Child\n" + list_item + list_marker_minus [0,2] + paragraph [2,11] "Parent\n " + inline [2,8] "Parent" + block_continuation [9,11] " " ← indent hangs on the PARENT paragraph; \n at 8 is a gap + list (nested) … +``` + +Consequences: + +- Every list/task marker is a **discrete sibling node**, exactly like the markers `SUPPRESSED_SYNTAX_TYPES` already drops — so suppression can be *extended*, not replaced. +- `task_list_marker_*` nodes exclude the trailing space; that space (index 5 above) is an orphan gap owned by no node and must be handled explicitly. +- `block_continuation` is a standalone leaf node, but it is not safe to drop with the existing whole-segment suppression path because a streamed range can start on the continuation and extend into real content. It is also **not list-only** (blockquotes and loose-list blanks produce it), so range-aware filtering must be scoped to list ancestry. +- Ordered `1)` and `1.` parse as **separate `list` nodes**; derive metadata from the marker node, never from assumed list continuity. + +## Implementation Changes + +The approach is **extend the existing node-suppression path first**, and use small scoped range filtering only for source slices that bypass node-at-position suppression. Node suppression already keeps rendered-offset accounting correct for free (it advances `sourceOffset` without advancing `totalUtf16Offset`), which preserves chunk offsets, span offsets, backtracking, and checkpoints without broad new offset math. + +1. **List metadata helpers (AST-derived, stateless).** + - Add helpers to find the enclosing `list_item`, read the marker node type/text, compute `ordered`/`unordered`, extract `ordinal` from the ordered marker text, read `task.checked` from a `task_list_marker_checked` / `task_list_marker_unchecked` sibling, and compute zero-based `depth` as `listAncestorCount - 1`. + - Metadata is a pure function of the node, so it is re-derived per chunk and is inherently backtrack-safe (no dependence on checkpoint state). + - `ordinal` is parsed from the leading digits of the marker text. If the value is greater than `Number.MAX_SAFE_INTEGER`, omit `ordinal` rather than emitting a lossy number. + +2. **Thread metadata through the internal types.** + - Extend internal `BlockInfo` (and `BlockState`, if list data is retained across chunks) with the list fields. + - Derive list metadata by scanning the full ancestor chain independently from block type selection. Do not rely on `getBlockInfo` reaching `list_item`, because nested blocks such as `fenced_code_block` currently return before the walk reaches their enclosing list item. + - Nested blocks (e.g. a code block inside a list) keep their natural `BlockInfo.type` and *also* receive list metadata. + - `createBlockContext` / `mapBlockType` (segment-builder.ts) copy the list metadata onto the public `BlockContext`. `BlockContext.list` is only set inside list context. + +3. **Extend suppression to the new marker nodes.** + - Add `task_list_marker_checked` and `task_list_marker_unchecked` to the suppressed syntax types. + - Do **not** add `block_continuation` to the existing whole-segment early-suppression branch. That branch drops the entire incoming range when `nodeAtPosition` is suppressed; for list-contained code blocks, a streamed range can start on a structural continuation node and continue into real code text. + - Handle list-scoped `block_continuation` with range-aware filtering/splitting instead: remove only the exact continuation-node range, or early-suppress only when `[actualFromIndex, actualToIndex)` is fully contained within that continuation node. + - After stripping a *leading* continuation prefix, re-derive the block/node for the remainder from the post-continuation position. Do not classify or extract the remaining content against the `block_continuation` node it started on. + - Apply this only when the `block_continuation` has an enclosing `list_item` ancestor before any enclosing `blockquote` ancestor. Do not use "nearest block ancestor" because `paragraph` is also a block type. This is the one conditional suppression and must not over-reach. + +4. **Handle the orphan task-marker space.** + - The space between a task marker and its paragraph is owned by no node; without handling it leaks as a lone `" "` `list_item` chunk. Strip it by extending the suppressed task-marker range to swallow following spaces up to, but not including, a newline. Do **not** strip the item's trailing `\n` — existing behavior retains it (e.g. `'Run npm install now\n'`), and that contract stays. + +5. **Filter post-inline structural tails.** + - Current generation appends source text after the inline node directly. That path can leak list structural text such as `\n ` / `block_continuation` even when node suppression handles normal marker ranges. + - Before appending `content.substring(Math.max(actualFromIndex, hostInlineNode.endIndex), actualToIndex)`, remove list-scoped suppressed ranges from that tail using the same suppression decision as marker/task/block-continuation handling. + - Keep real rendered newlines that belong to item text; only remove structural continuation indentation and stripped task-marker spaces. + +6. **Filter list continuations inside code-block extraction.** + - The motivating list-contained code-block case flows through the `getCodeBlockContent` branch, not the generic inline/tail path. + - `getCodeBlockContent` currently skips only `fenced_code_block_delimiter` and `info_string`; also remove list-scoped `block_continuation` ranges there so list indentation around fences and code lines does not leak. + - Preserve code content after a stripped continuation prefix. For example, when a source range starts with structural list indentation and then real code text, strip only the structural prefix and emit the remaining code text as `block.type: 'code_block'`. + +7. **Consolidate duplicated constants (DRY, done as part of this change).** + - Replace the local copies `SUPPRESSED_SYNTAX_TYPES_LOCAL` and `HEADER_MARKER_LEVELS_LOCAL` in `segment-generator.ts` with the exported constants from `types.ts`, so the new marker types are added in exactly one place. + +8. **Docs.** Update README supported-Markdown/API docs, including moving task lists out of Limitations and documenting the `block.list` shape and the ordered `marker`/`ordinal` split. + +## Offset & Recovery Notes + +- Because node suppression handles marker nodes directly, `rawToRenderedOffset` / `collectInlineDelimiterRanges` need **no** list awareness for markers that are their own nodes. Range filtering is limited to orphan task-marker spaces and list-scoped structural continuations, including code-block extraction and post-inline tails that bypass node-at-position suppression. +- The one place to verify carefully is **spans inside a task item**: the `inline` node starts after `[x] ` / `[X] ` / `[ ] `, so a span must render as if the prefix never existed. Confirm `chunkStartUtf16` base math holds when the suppressed prefix and the span text fall in the same word-range/chunk. This is the primary correctness risk and gets a dedicated exact-offset test. +- Split-marker streaming (`-` then ` `; `1` then `.` then ` `) briefly parses as paragraph/other, then backtracks once the marker resolves. Metadata is re-derived from the post-backtrack AST, so assertions target post-backtrack output. +- Checkpoints shallow-copy `currentBlock`; if list metadata is stored there, clone the nested `list` object in checkpoint creation/restoration or treat it as immutable for the full generator lifecycle. + +## Test Plan + +- Unordered markers `-`, `+`, `*`; ordered markers `1.`, `10.`, `1)`, asserting exact `ordinal` (e.g. `10`) and `marker` values, not just detection. +- Ordered marker with a number greater than `Number.MAX_SAFE_INTEGER`: assert `marker` is present and `ordinal` is omitted. +- Nested lists: assert exact zero-based `depth` per level and that no structural indentation (`block_continuation`) leaks into rendered text. +- Loose lists: blank-line items still classify as list items and no indentation leaks. +- Task lists: checked and unchecked items, including uppercase `[X]`, rendered text without `[x]` / `[X]` / `[ ]` **and** without the following space, `task.checked` correct, and the item's trailing `\n` preserved. +- Streaming split-marker tests: `'-'`, `' '`, `'Item\n'`; `'1'`, `'.'`, `' '`, `'Item\n'`; split task-marker chunks — asserting post-backtrack metadata and text. +- Inline span inside a list/task item: assert exact span `offset` and `length` after the stripped list/task syntax (the core offset regression guard). +- List-contained code block: list indentation/fence syntax does not leak through `getCodeBlockContent`, code text after stripped continuation prefixes is preserved, and `block.list` metadata is preserved alongside `block.type: 'code_block'`. +- **Negative tests (guard over-broad suppression):** non-list blockquote continuation still renders its text, and a loose-list blank line does not swallow adjacent content. These ensure the list-scoped `block_continuation` rule does not strip non-list continuations. +- **List inside a blockquote** (`> - a\n> - b\n`): the continuation node's text is `"> "` (it carries the `>` marker), and because it is structural continuation inside a list item, suppress it — assert neither the indent nor the `>` leaks into rendered item text. +- Run verification inside `lixpi-markdown-stream-parser-demo`: + +```sh +docker exec lixpi-markdown-stream-parser-demo pnpm test:run +``` + +## Assumptions + +- The change targets the tree-sitter parser path, which is the public documented parser. +- Existing consumers remain compatible because list metadata is optional and existing `block.type` values are preserved. +- Blockquote behavior inside lists remains limited to current blockquote support unless separately requested. +- The bundled grammar emits `task_list_marker_checked` / `task_list_marker_unchecked` (verified against the WASM in `demo/svelte-demo/static`); no GFM extension toggle is required. diff --git a/src/tree-sitter-markdown-stream-parser.test.ts b/src/tree-sitter-markdown-stream-parser.test.ts index 680b1fb..ad14059 100644 --- a/src/tree-sitter-markdown-stream-parser.test.ts +++ b/src/tree-sitter-markdown-stream-parser.test.ts @@ -1,6 +1,7 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' import { MarkdownStreamParser } from './tree-sitter-markdown-stream-parser' import type { Chunk, ClosedSpan, SpanType } from './tree-sitter/types.ts' +import { getListMetadata } from './tree-sitter/list-support.ts' import path from 'path' import { fileURLToPath } from 'url' import fs from 'fs' @@ -256,6 +257,237 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { const listChunks = parsedChunks.filter(c => c.block.type === 'list_item') expect(listChunks.length).toBeGreaterThan(0) }) + + it('should expose unordered list metadata for each marker type', async () => { + parser.parseToken('- Dash\n') + parser.parseToken('+ Plus\n') + parser.parseToken('* Star\n') + parser.stopParsing() + + const activeChunks = applyBacktracks(parsedChunks) + const items = activeChunks.filter(c => c.block.type === 'list_item' && c.text.trim().length > 0) + + expect(items.map(c => c.block.list?.marker)).toEqual(['-', '+', '*']) + expect(items.every(c => c.block.list?.type === 'unordered')).toBe(true) + expect(items.every(c => c.block.list?.depth === 0)).toBe(true) + expect(items.every(c => c.block.list?.ordinal === undefined)).toBe(true) + }) + + it('should expose ordered list metadata with delimiters and ordinals', async () => { + parser.parseToken('1. First\n') + parser.parseToken('10. Tenth\n') + parser.parseToken('1) Parenthesis\n') + parser.stopParsing() + + const activeChunks = applyBacktracks(parsedChunks) + const items = activeChunks.filter(c => c.block.type === 'list_item' && c.text.trim().length > 0) + + expect(items.map(c => c.block.list?.marker)).toEqual(['.', '.', ')']) + expect(items.map(c => c.block.list?.ordinal)).toEqual([1, 10, 1]) + expect(items.every(c => c.block.list?.type === 'ordered')).toBe(true) + }) + + it('should omit unsafe ordered ordinals from metadata', async () => { + const marker: any = { + type: 'list_marker_dot', + text: '9007199254740993.', + children: [], + parent: undefined, + } + const listItem: any = { + type: 'list_item', + children: [marker], + parent: undefined, + } + const list: any = { + type: 'list', + children: [listItem], + parent: null, + } + marker.parent = listItem + listItem.parent = list + + const metadata = getListMetadata(listItem) + + expect(metadata?.type).toBe('ordered') + expect(metadata?.marker).toBe('.') + expect(metadata?.ordinal).toBeUndefined() + }) + + it('should expose nested list depth without structural indentation', async () => { + parser.parseToken('- Parent\n') + parser.parseToken(' - Child\n') + parser.stopParsing() + + const activeChunks = applyBacktracks(parsedChunks) + const fullText = activeChunks.map(c => c.text).join('') + const parent = activeChunks.find(c => c.text.includes('Parent')) + const child = activeChunks.find(c => c.text.includes('Child')) + + expect(fullText).toBe('Parent\nChild\n') + expect(parent?.block.list?.depth).toBe(0) + expect(child?.block.list?.depth).toBe(1) + }) + + it('should strip task markers and expose checked state', async () => { + parser.parseToken('- [x] Done\n') + parser.parseToken('- [X] Upper\n') + parser.parseToken('- [ ] Todo\n') + parser.stopParsing() + + const activeChunks = applyBacktracks(parsedChunks) + const fullText = activeChunks.map(c => c.text).join('') + const items = activeChunks.filter(c => c.block.type === 'list_item' && c.text.trim().length > 0) + + expect(fullText).toBe('Done\nUpper\nTodo\n') + expect(fullText).not.toContain('[x]') + expect(fullText).not.toContain('[X]') + expect(fullText).not.toContain('[ ]') + expect(items.map(c => c.block.list?.task?.checked)).toEqual([true, true, false]) + }) + + it('should handle split list and task markers after backtracking', async () => { + parser.parseToken('-') + parser.parseToken(' ') + parser.parseToken('[ ] ') + parser.parseToken('Item\n') + parser.stopParsing() + + const activeChunks = applyBacktracks(parsedChunks) + const fullText = activeChunks.map(c => c.text).join('') + const item = activeChunks.find(c => c.text.includes('Item')) + + expect(fullText).toBe('Item\n') + expect(item?.block.type).toBe('list_item') + expect(item?.block.list).toMatchObject({ + type: 'unordered', + marker: '-', + depth: 0, + task: { checked: false }, + }) + }) + + it('should keep inline span offsets correct inside task items', async () => { + parser.parseToken('- [x] Run `npm install` now\n') + parser.stopParsing() + + const activeChunks = applyBacktracks(parsedChunks) + const fullText = activeChunks.map(c => c.text).join('') + const codeSpan = getClosedSpans(activeChunks).find(s => s.type === 'code') + + expect(fullText).toBe('Run npm install now\n') + expect(codeSpan?.offset).toBe('Run '.length) + expect(codeSpan?.length).toBe('npm install'.length) + }) + + it('should preserve list metadata and strip continuation prefixes in list-contained code blocks', async () => { + parser.parseToken('- Example\n') + parser.parseToken(' ```ts\n') + parser.parseToken(' const x = 1\n') + parser.parseToken(' ```\n') + parser.stopParsing() + + const activeChunks = applyBacktracks(parsedChunks) + const codeChunks = activeChunks.filter(c => c.block.type === 'code_block') + const codeText = codeChunks.map(c => c.text).join('') + + expect(codeText).toBe('const x = 1\n') + expect(codeText).not.toContain(' ') + expect(codeChunks.length).toBeGreaterThan(0) + expect(codeChunks.every(c => c.block.list?.depth === 0)).toBe(true) + expect(codeChunks.every(c => c.block.list?.marker === '-')).toBe(true) + }) + + it('should handle a loose list with a blank line between items', async () => { + parser.parseToken('- one\n') + parser.parseToken('\n') + parser.parseToken('- two\n') + parser.stopParsing() + + const activeChunks = applyBacktracks(parsedChunks) + const fullText = activeChunks.map(c => c.text).join('') + const items = activeChunks.filter(c => c.block.type === 'list_item' && c.text.trim().length > 0) + + expect(fullText).toBe('one\n\ntwo\n') + expect(items.every(c => c.block.list?.depth === 0)).toBe(true) + expect(items.every(c => c.block.list?.marker === '-')).toBe(true) + }) + + it('should split a plain unordered marker across chunks', async () => { + parser.parseToken('-') + parser.parseToken(' ') + parser.parseToken('Item\n') + parser.stopParsing() + + const activeChunks = applyBacktracks(parsedChunks) + const fullText = activeChunks.map(c => c.text).join('') + const item = activeChunks.find(c => c.text.includes('Item')) + + expect(fullText).toBe('Item\n') + expect(item?.block.list).toMatchObject({ type: 'unordered', marker: '-', depth: 0 }) + expect(item?.block.list?.task).toBeUndefined() + }) + + it('should split a plain ordered marker across chunks', async () => { + parser.parseToken('1') + parser.parseToken('.') + parser.parseToken(' ') + parser.parseToken('Item\n') + parser.stopParsing() + + const activeChunks = applyBacktracks(parsedChunks) + const fullText = activeChunks.map(c => c.text).join('') + const item = activeChunks.find(c => c.text.includes('Item')) + + expect(fullText).toBe('Item\n') + expect(item?.block.list).toMatchObject({ type: 'ordered', marker: '.', ordinal: 1, depth: 0 }) + }) + + it('should not treat non-checkbox bracket text as a task marker', async () => { + parser.parseToken('- Some ') + parser.parseToken('[ ') + parser.parseToken('note] ') + parser.parseToken('text\n') + parser.stopParsing() + + const activeChunks = applyBacktracks(parsedChunks) + const fullText = activeChunks.map(c => c.text).join('') + + expect(fullText).toBe('Some [ note] text\n') + expect(activeChunks.every(c => c.block.list?.task === undefined)).toBe(true) + }) + + it('should not suppress a non-list blockquote continuation (negative test)', async () => { + parser.parseToken('> line one\n') + parser.parseToken('> line two\n') + parser.stopParsing() + + const activeChunks = applyBacktracks(parsedChunks) + const fullText = activeChunks.map(c => c.text).join('') + + // Blockquote marker stripping is a separate, unimplemented feature (see + // the skipped tests in the "Blockquotes" describe block above). This + // asserts the new list-scoped block_continuation suppression does not + // reach into a plain (non-list) blockquote and swallow its text. + expect(fullText).toContain('line one') + expect(fullText).toContain('line two') + }) + + it('should suppress a list-scoped continuation nested inside a blockquote', async () => { + parser.parseToken('> - a\n') + parser.parseToken('> - b\n') + parser.stopParsing() + + const activeChunks = applyBacktracks(parsedChunks) + const fullText = activeChunks.map(c => c.text).join('') + + // The continuation line carries the blockquote's "> " prefix, but + // because it also structurally continues the list item, it must be + // suppressed the same as an unquoted nested list continuation. + expect(fullText).not.toContain('> ') + expect(fullText).toContain('a') + expect(fullText).toContain('b') + }) }) describe('Nested Inline Styles', () => { diff --git a/src/tree-sitter/block-detection.ts b/src/tree-sitter/block-detection.ts index a3f3deb..03677a5 100644 --- a/src/tree-sitter/block-detection.ts +++ b/src/tree-sitter/block-detection.ts @@ -1,10 +1,12 @@ import type { Node } from 'web-tree-sitter' import { HEADER_MARKER_LEVELS, type BlockInfo, type BlockState } from './types.ts' import { findBlockNode } from './tree-navigation.ts' +import { getListMetadata } from './list-support.ts' // Get the block type and properties from a tree-sitter node. // Walks up the tree to find the enclosing block structure. export function getBlockInfo(node: Node): BlockInfo { + const list = getListMetadata(node) let current: Node | null = node let foundParagraph = false let foundTableCell = false @@ -15,7 +17,8 @@ export function getBlockInfo(node: Node): BlockInfo { case 'atx_heading': return { type: 'header', - level: getHeadingLevel(current) + level: getHeadingLevel(current), + list, } case 'paragraph': // Don't return immediately - check if we're inside a list_item or blockquote @@ -24,14 +27,15 @@ export function getBlockInfo(node: Node): BlockInfo { case 'fenced_code_block': return { type: 'codeBlock', // camelCase for consistency - language: getCodeBlockLanguage(current) + language: getCodeBlockLanguage(current), + list, } case 'list_item': // If we found a paragraph inside a list_item, return list_item - return { type: 'list_item' } + return { type: 'list_item', list } case 'blockquote': // If we found a paragraph inside a blockquote, return blockquote - return { type: 'blockquote' } + return { type: 'blockquote', list } // Table types case 'pipe_table_cell': foundTableCell = true @@ -40,22 +44,22 @@ export function getBlockInfo(node: Node): BlockInfo { isInHeader = true // If we found a cell inside a header, return table_header_cell if (foundTableCell) { - return { type: 'table_header_cell', id: current.id } + return { type: 'table_header_cell', id: current.id, list } } break case 'pipe_table_row': // If we found a cell inside a regular row, return table_cell if (foundTableCell) { - return { type: 'table_cell', id: current.id } + return { type: 'table_cell', id: current.id, list } } break case 'pipe_table': // Found the table - if we have a cell, determine type based on header flag if (foundTableCell) { - return { type: isInHeader ? 'table_header_cell' : 'table_cell', id: current.id } + return { type: isInHeader ? 'table_header_cell' : 'table_cell', id: current.id, list } } // Otherwise just return table - return { type: 'table' } + return { type: 'table', list } } current = current.parent @@ -63,10 +67,10 @@ export function getBlockInfo(node: Node): BlockInfo { // If we found a paragraph but no enclosing list_item/blockquote, return paragraph if (foundParagraph) { - return { type: 'paragraph' } + return { type: 'paragraph', list } } - return { type: 'paragraph' } + return { type: 'paragraph', list } } // Check if the given node represents a new block compared to the current block state. diff --git a/src/tree-sitter/content-extraction.ts b/src/tree-sitter/content-extraction.ts index e92f73c..d229294 100644 --- a/src/tree-sitter/content-extraction.ts +++ b/src/tree-sitter/content-extraction.ts @@ -1,4 +1,5 @@ import type { Node, Tree } from 'web-tree-sitter' +import { stripListSuppressedRanges } from './list-support.ts' // Extract header content from a chunk, excluding marker nodes (# symbols). // Requires tree-sitter node for accurate extraction. @@ -66,7 +67,8 @@ export function getCodeBlockContent( if (overlapStart < overlapEnd) { const relativeStart = overlapStart - startByte const relativeEnd = overlapEnd - startByte - extractedText += content.substring(relativeStart, relativeEnd) + const text = content.substring(relativeStart, relativeEnd) + extractedText += stripListSuppressedRanges(text, child, content, overlapStart, overlapEnd) } } } diff --git a/src/tree-sitter/list-support.ts b/src/tree-sitter/list-support.ts new file mode 100644 index 0000000..6615612 --- /dev/null +++ b/src/tree-sitter/list-support.ts @@ -0,0 +1,233 @@ +import type { Node } from 'web-tree-sitter' +import type { ListMetadata } from './types.ts' + +type Range = { start: number; end: number } + +const LIST_MARKER_TYPES = [ + 'list_marker_minus', + 'list_marker_plus', + 'list_marker_star', + 'list_marker_dot', + 'list_marker_parenthesis', +] as const + +const TASK_MARKER_TYPES = [ + 'task_list_marker_checked', + 'task_list_marker_unchecked', +] as const + +function isListMarkerType(type: string): boolean { + return LIST_MARKER_TYPES.indexOf(type as typeof LIST_MARKER_TYPES[number]) !== -1 +} + +function isTaskMarkerType(type: string): boolean { + return TASK_MARKER_TYPES.indexOf(type as typeof TASK_MARKER_TYPES[number]) !== -1 +} + +function findEnclosingListItem(node: Node): Node | null { + let current: Node | null = node + + while (current) { + if (current.type === 'list_item') { + return current + } + current = current.parent + } + + return null +} + +function getListDepth(node: Node): number { + let listCount = 0 + let current: Node | null = node + + while (current) { + if (current.type === 'list') { + listCount++ + } + current = current.parent + } + + return Math.max(0, listCount - 1) +} + +function getTaskMetadata(listItem: Node): ListMetadata['task'] | undefined { + for (const child of listItem.children) { + if (child.type === 'task_list_marker_checked') { + return { checked: true } + } + if (child.type === 'task_list_marker_unchecked') { + return { checked: false } + } + } + + return undefined +} + +export function getListMetadata(node: Node): ListMetadata | undefined { + const listItem = findEnclosingListItem(node) + if (!listItem) { + return undefined + } + + const markerNode = listItem.children.find(child => isListMarkerType(child.type)) + if (!markerNode) { + return undefined + } + + const task = getTaskMetadata(listItem) + const depth = getListDepth(listItem) + + let unorderedMarker: '-' | '+' | '*' | undefined + switch (markerNode.type) { + case 'list_marker_minus': unorderedMarker = '-'; break + case 'list_marker_plus': unorderedMarker = '+'; break + case 'list_marker_star': unorderedMarker = '*'; break + } + + if (unorderedMarker) { + const metadata: ListMetadata = { type: 'unordered', depth, marker: unorderedMarker } + if (task) { + metadata.task = task + } + return metadata + } + + if (markerNode.type === 'list_marker_dot' || markerNode.type === 'list_marker_parenthesis') { + const marker = markerNode.type === 'list_marker_dot' ? '.' : ')' + const ordinalText = markerNode.text.trim().match(/^(\d+)/)?.[1] + const ordinal = ordinalText ? Number(ordinalText) : undefined + const metadata: ListMetadata = { type: 'ordered', depth, marker } + + if (ordinal !== undefined && Number.isSafeInteger(ordinal)) { + metadata.ordinal = ordinal + } + if (task) { + metadata.task = task + } + + return metadata + } + + return undefined +} + +// A task marker can only ever appear as the first thing in a list item's +// content, immediately after the list marker node (which itself includes +// its trailing space). Scoping the pending-bracket check to this exact +// position avoids treating ordinary list text like "[ note] text" as a +// possible in-progress checkbox. +export function isAtListItemContentStart(node: Node, position: number): boolean { + const listItem = findEnclosingListItem(node) + if (!listItem) { + return false + } + + const markerNode = listItem.children.find(child => isListMarkerType(child.type)) + return markerNode !== undefined && position === markerNode.endIndex +} + +export function isListScopedBlockContinuation(node: Node): boolean { + if (node.type !== 'block_continuation') { + return false + } + + let current = node.parent + while (current) { + if (current.type === 'list_item') { + return true + } + if (current.type === 'blockquote') { + return false + } + current = current.parent + } + + return false +} + +function extendTaskMarkerRange(content: string, markerEnd: number): number { + let end = markerEnd + + while (end < content.length && content[end] !== '\n' && /\s/.test(content[end])) { + end++ + } + + return end +} + +function collectSuppressedRanges( + node: Node, + content: string, + startIndex: number, + endIndex: number, + ranges: Range[] +): void { + if (node.endIndex <= startIndex || node.startIndex >= endIndex) { + return + } + + if (isTaskMarkerType(node.type)) { + ranges.push({ + start: node.startIndex, + end: extendTaskMarkerRange(content, node.endIndex), + }) + return + } + + if (isListScopedBlockContinuation(node)) { + ranges.push({ start: node.startIndex, end: node.endIndex }) + return + } + + for (const child of node.children) { + collectSuppressedRanges(child, content, startIndex, endIndex, ranges) + } +} + +export function getListSuppressedRanges( + root: Node, + content: string, + startIndex: number, + endIndex: number +): Range[] { + const ranges: Range[] = [] + collectSuppressedRanges(root, content, startIndex, endIndex, ranges) + + return ranges + .map(range => ({ + start: Math.max(range.start, startIndex), + end: Math.min(range.end, endIndex), + })) + .filter(range => range.start < range.end) + .sort((a, b) => a.start - b.start) +} + +export function stripListSuppressedRanges( + text: string, + root: Node, + content: string, + startIndex: number, + endIndex: number +): string { + const ranges = getListSuppressedRanges(root, content, startIndex, endIndex) + if (ranges.length === 0) { + return text + } + + let result = '' + let cursor = startIndex + + for (const range of ranges) { + if (cursor < range.start) { + result += content.substring(cursor, range.start) + } + cursor = Math.max(cursor, range.end) + } + + if (cursor < endIndex) { + result += content.substring(cursor, endIndex) + } + + return result +} diff --git a/src/tree-sitter/segment-builder.ts b/src/tree-sitter/segment-builder.ts index 55cc18c..90eabed 100644 --- a/src/tree-sitter/segment-builder.ts +++ b/src/tree-sitter/segment-builder.ts @@ -93,6 +93,9 @@ export function createBlockContext(blockInfo: BlockInfo): BlockContext { if (blockInfo.language !== undefined) { context.language = blockInfo.language } + if (blockInfo.list !== undefined) { + context.list = blockInfo.list + } return context } diff --git a/src/tree-sitter/segment-generator.ts b/src/tree-sitter/segment-generator.ts index 1de14de..21686d9 100644 --- a/src/tree-sitter/segment-generator.ts +++ b/src/tree-sitter/segment-generator.ts @@ -8,6 +8,7 @@ import type { SpanType, ParserConfig } from './types.ts' +import { HEADER_MARKER_LEVELS, SUPPRESSED_SYNTAX_TYPES } from './types.ts' import { findActiveNodeAtPosition, findInlineNodeAtPosition, findBlockNode } from './tree-navigation.ts' import { getBlockInfo } from './block-detection.ts' import { @@ -32,22 +33,12 @@ import { createLinkSpan, createImageSpan } from './segment-builder.ts' - -// Re-import the constant that we need locally -const HEADER_MARKER_LEVELS_LOCAL: Record = { - 'atx_h1_marker': 1, - 'atx_h2_marker': 2, - 'atx_h3_marker': 3, - 'atx_h4_marker': 4, - 'atx_h5_marker': 5, - 'atx_h6_marker': 6, -} - -const SUPPRESSED_SYNTAX_TYPES_LOCAL = [ - 'list_marker_minus', 'list_marker_plus', 'list_marker_star', - 'list_marker_dot', 'list_marker_parenthesis', - '|' // Table pipe delimiters -] +import { + getListSuppressedRanges, + isAtListItemContentStart, + isListScopedBlockContinuation, + stripListSuppressedRanges +} from './list-support.ts' export type SegmentGeneratorContext = { content: string @@ -515,8 +506,57 @@ export function generateSegments( return { segments: [chunk], state } } + const leadingSuppressedRange = getListSuppressedRanges( + currentTree.rootNode, + content, + actualFromIndex, + actualToIndex + ).find(range => range.start === actualFromIndex) + + if (leadingSuppressedRange) { + const suppressedEnd = leadingSuppressedRange.end + const suppressedPrefix = content.substring(actualFromIndex, suppressedEnd) + state = { + ...state, + sourceOffset: suppressedEnd, + accumulatedContent: state.accumulatedContent + suppressedPrefix + } + + if (suppressedEnd >= actualToIndex) { + state = withCheckpoint(state) + return { segments, state } + } + + return generateSegments(suppressedEnd, actualToIndex, { + ...context, + state, + disableBlockBoundarySplit: true, + }) + } + + if (isListScopedBlockContinuation(nodeAtPosition)) { + const continuationEnd = Math.min(nodeAtPosition.endIndex, actualToIndex) + const suppressedPrefix = content.substring(actualFromIndex, continuationEnd) + state = { + ...state, + sourceOffset: continuationEnd, + accumulatedContent: state.accumulatedContent + suppressedPrefix + } + + if (continuationEnd >= actualToIndex) { + state = withCheckpoint(state) + return { segments, state } + } + + return generateSegments(continuationEnd, actualToIndex, { + ...context, + state, + disableBlockBoundarySplit: true, + }) + } + // Check if the node is a suppressed syntax type - if (SUPPRESSED_SYNTAX_TYPES_LOCAL.indexOf(nodeAtPosition.type) !== -1) { + if (SUPPRESSED_SYNTAX_TYPES.indexOf(nodeAtPosition.type as typeof SUPPRESSED_SYNTAX_TYPES[number]) !== -1) { state = { ...state, sourceOffset: actualToIndex, @@ -545,6 +585,13 @@ export function generateSegments( // Determine the block type and properties const blockInfo = getBlockInfo(nodeAtPosition) + if (blockInfo.list && isAtListItemContentStart(nodeAtPosition, actualFromIndex) && /^\[[ xX]?$/.test(newContent)) { + state.pendingInlineContent = newContent + state.pendingInlineStartIndex = actualFromIndex + state.sourceOffset = actualToIndex + return { segments, state } + } + // Process content based on block type let processedContent = newContent @@ -587,7 +634,7 @@ export function generateSegments( } } else if (blockInfo.type === 'paragraph') { // Handle incomplete header markers - if (nodeAtPosition.type in HEADER_MARKER_LEVELS_LOCAL) { + if (nodeAtPosition.type in HEADER_MARKER_LEVELS) { state = { ...state, sourceOffset: actualToIndex, @@ -638,7 +685,9 @@ export function generateSegments( ) if (hostInlineNode && blockInfo.type !== 'header' && actualToIndex > hostInlineNode.endIndex) { - strippedContent += content.substring(Math.max(actualFromIndex, hostInlineNode.endIndex), actualToIndex) + const tailStart = Math.max(actualFromIndex, hostInlineNode.endIndex) + const tailText = content.substring(tailStart, actualToIndex) + strippedContent += stripListSuppressedRanges(tailText, currentTree.rootNode, content, tailStart, actualToIndex) } } diff --git a/src/tree-sitter/types.ts b/src/tree-sitter/types.ts index c5352ed..b631c9b 100644 --- a/src/tree-sitter/types.ts +++ b/src/tree-sitter/types.ts @@ -55,6 +55,16 @@ export type BlockContext = { level?: number // For code blocks: language identifier language?: string + // For content inside a Markdown list item. + list?: ListMetadata +} + +export type ListMetadata = { + type: 'ordered' | 'unordered' + depth: number + marker: '-' | '+' | '*' | '.' | ')' + ordinal?: number + task?: { checked: boolean } } // ============================================================================ @@ -156,6 +166,7 @@ export const BLOCK_TYPES = [ export const SUPPRESSED_SYNTAX_TYPES = [ 'list_marker_minus', 'list_marker_plus', 'list_marker_star', 'list_marker_dot', 'list_marker_parenthesis', + 'task_list_marker_checked', 'task_list_marker_unchecked', '|' // Table pipe delimiters ] as const @@ -174,6 +185,7 @@ export type BlockInfo = { type: string level?: number language?: string + list?: ListMetadata id?: number } From b8757bebf551af91d369ac4ab251f07ac19256c9 Mon Sep 17 00:00:00 2001 From: Dmitry Bondarenko Date: Sat, 4 Jul 2026 12:15:38 +0600 Subject: [PATCH 29/32] Improves table parsing --- README.md | 12 + TABLES_IMPROVEMENT_PLAN.md | 70 ++++++ demo/svelte-demo/src/routes/+page.svelte | 141 +++++++++-- src/markdown-stream-parser.ts | 2 + ...tree-sitter-markdown-stream-parser.test.ts | 226 +++++++++++++++++- src/tree-sitter-markdown-stream-parser.ts | 2 + src/tree-sitter/block-detection.ts | 32 +-- src/tree-sitter/segment-builder.ts | 6 +- src/tree-sitter/segment-generator.ts | 20 +- src/tree-sitter/table-support.ts | 153 ++++++++++++ src/tree-sitter/types.ts | 15 +- 11 files changed, 613 insertions(+), 66 deletions(-) create mode 100644 TABLES_IMPROVEMENT_PLAN.md create mode 100644 src/tree-sitter/table-support.ts diff --git a/README.md b/README.md index c4284f1..80cf6a6 100644 --- a/README.md +++ b/README.md @@ -188,6 +188,7 @@ type BlockContext = { | 'list_item' | 'table' | 'table_row' + | 'table_header_cell' | 'table_cell' | 'blockquote' level?: number @@ -199,6 +200,13 @@ type BlockContext = { ordinal?: number task?: { checked: boolean } } + table?: { + tableId: string + rowIndex: number + columnIndex: number + cellId: string + align?: 'left' | 'center' | 'right' + } } ``` @@ -206,6 +214,10 @@ type BlockContext = { `list` is present when the chunk is inside a list item. `depth` is zero-based. Unordered items use `marker` for the bullet character (`-`, `+`, or `*`). Ordered items use `marker` for the delimiter only (`.` or `)`) and put the number in `ordinal` when it is safely representable as a JavaScript number. Task list items omit the `[x]`, `[X]`, or `[ ]` marker from rendered text and expose `task.checked`. +`table` is present when the chunk is inside a table cell. `tableId` identifies the enclosing table, `rowIndex` is zero-based with the header row at `0`, `columnIndex` is zero-based within the row, `cellId` is a stable `${tableId}:${rowIndex}:${columnIndex}` grouping key, and `align` reflects the parsed delimiter row when specified. + +Compatibility note: header cell chunks now use `block.type === 'table_header_cell'`. Consumers that previously treated all table header content as `table_cell` should update that branch. + When `includeRawStreamedToken` is enabled, `original` contains the raw Markdown source associated with the emitted chunk. It is separate from the rendered UTF-16 coordinate space. ## Inline Spans diff --git a/TABLES_IMPROVEMENT_PLAN.md b/TABLES_IMPROVEMENT_PLAN.md new file mode 100644 index 0000000..6fa2356 --- /dev/null +++ b/TABLES_IMPROVEMENT_PLAN.md @@ -0,0 +1,70 @@ +# Plan: Improve Markdown Table Parsing (LIX-MDSP-7/tables-support) + +## Context + +The tree-sitter streaming parser has partial table support: it detects `table`/`table_row`/`table_cell` blocks, suppresses pipes and delimiter rows, and handles header reclassification via backtrack. Gaps: header cells are folded into `table_cell` (renderers can't emit ``), column alignment from the delimiter row is discarded, streaming edge cases are under-tested, and table logic is scattered across four files. Legacy `src/state-machine/` is dead code — ignored entirely. All code runs inside the `lixpi-markdown-stream-parser-demo` docker service. + +## Steps + +### 1. Types and public contract +- In `src/tree-sitter/types.ts`: add `'table_header_cell'` to `BlockType` union (~line 46), add `TableAlignment = 'left' | 'center' | 'right'`, add `TableMetadata = { tableId: string; rowIndex: number; columnIndex: number; cellId: string; align?: TableAlignment }`, and add `table?: TableMetadata` to `BlockContext` and internal `BlockInfo` +- `tableId` must be deterministic from table structure and stable across backtrack replay; use a source-derived key such as the enclosing `pipe_table.startIndex`, not a raw tree-sitter `Node.id`. +- `cellId` must be deterministic and stream-unique, e.g. `${tableId}:${rowIndex}:${columnIndex}`. This prevents adjacent tables from merging when two tables both contain a local `0:0` cell and gives consumers a stable way to group word-sized chunks from the same markdown cell. +- In `src/tree-sitter-markdown-stream-parser.ts` and `src/markdown-stream-parser.ts`: re-export `TableAlignment` and `TableMetadata` if they are named public types +- In `README.md`: document `table_header_cell`, `block.table.tableId`, `block.table.rowIndex`, `block.table.columnIndex`, `block.table.cellId`, and `block.table.align`; add a compatibility note for consumers that previously treated all header chunks as `table_cell` + +Breaking only for consumers that expect header cells as `table_cell` — that's the point of the change. There is no changelog file currently; record the compatibility note in README or add a dedicated changelog/release note file as part of this change. + +### 2. New module `src/tree-sitter/table-support.ts` (precedent: `list-support.ts`) +Pure Node-walking functions: +- `isInsideTableDelimiterRow(node)` — extracted from `segment-generator.ts:569-583` +- `getTableAlignments(tableNode)` — parse `pipe_table_delimiter_cell` text (`:---`/`:---:`/`---:`) → normalized alignment per column +- `getColumnIndex(cellNode)` — count preceding `pipe_table_cell` siblings only +- `getRowIndex(rowNode)` — count preceding `pipe_table_header`/`pipe_table_row` siblings only; header row is `0` +- `getTableId(tableNode)` — deterministic public table key, e.g. `table:${tableNode.startIndex}` +- `getCellId(tableId, rowIndex, columnIndex)` — deterministic public grouping key, e.g. `${tableId}:${rowIndex}:${columnIndex}` +- `getTableBlockInfo(node, list?)` — the table branch of `getBlockInfo` moved here; returns `BlockInfo` incl. `table: { tableId, rowIndex, columnIndex, cellId, align? }` + +Keep `isHeader` out of public `TableMetadata` unless a consumer need appears; `table_header_cell` already exposes header-ness. + +### 3. Delegation +- `block-detection.ts:39-62`: replace table case cluster with `getTableBlockInfo(node, list)` call (same pattern as `getListMetadata`); keep `getListMetadata(node)` owned by `getBlockInfo()` so table support does not duplicate list walks/imports +- Remove the now-dead `id` field: the table branches (`block-detection.ts:47,53,59`) are the only place it is ever set and nothing reads it (`isNewBlock` compares type/level/startIndex; `createBlockContext` never copies it), so also delete `id?: number` from `BlockInfo` in `types.ts:189` +- `segment-builder.ts:67-75`: map `table_header_cell` → `table_header_cell` (stop folding); copy `blockInfo.table` in `createBlockContext` (like `list`/`language`) +- Carry explicit table metadata through `BlockContext`; do not use internal `BlockInfo.id` for public grouping + +### 4. `segment-generator.ts` +- Replace inline delimiter walk (569-583) with `isInsideTableDelimiterRow()` +- Alignment flows automatically via existing BlockInfo → BlockContext → chunk path. Timing: header cells emitted pre-delimiter parse as paragraph; the existing backtrack re-emits them as `table_header_cell` with `align` once the delimiter row is in the tree. Body cells carry `tableId`, `rowIndex`, `columnIndex`, `cellId`, and `align` when specified by the delimiter row. +- Robustness fixes driven by Step 6 tests. Known suspects: + - Cell identity/grouping: current chunks are word-sized, so one markdown cell may produce multiple chunks. Group by explicit `table.cellId`; never rely on `blockInfo.id` + - Delimiter row split across chunks transiently parsing as body row (leaking `---`) — verify backtrack corrects; fix empirically + - `windowSize` overflow during header reclassification → must yield `recovery: window_overflow`, not corruption + +### 5. Demo (`demo/svelte-demo/src/routes/+page.svelte`) +- Include `table_header_cell` in `hasTableCells` (~line 521) +- Add header-cell rendering branch (~line 689, bold/th-style) and apply `chunk.block.table?.align` as text alignment on cells +- Do not render every chunk as its own bordered cell. The token buffer emits word-sized chunks, so cells like `New York` can become multiple `table_cell` chunks. Build table display groups by `chunk.block.table.tableId`, then `rowIndex`, then `cellId`; render rows/cells from those groups. This also prevents adjacent tables with matching local row/column positions from merging. +- Keep alignment application closed over known values (`left`/`center`/`right`) via classes or controlled style values; never pass raw delimiter text into a `style` attribute + +### 6. Tests (`src/tree-sitter-markdown-stream-parser.test.ts`, new `describe('Table Support')`) +- Header cells → `table_header_cell`, body → `table_cell` (update existing test at 971-983) +- Alignment left/center/right/undefined + `tableId` + `rowIndex` + `columnIndex` + stable stream-unique `cellId` +- Chunked streaming helper, sizes 1/2/3: no `|`/`---` leakage in reconstructed active output; pipe split across chunks; delimiter row split mid-cell; table at stream start/end; table after paragraph and after list (no `list` metadata bleed) +- Backtrack: re-emitted header chunks carry type + align; small-`windowSize` recovery test +- When asserting leakage or final content, reconstruct the active stream after applying `backtrackOffset`; raw emitted event history may contain transient paragraph chunks before table reclassification +- Multi-word cell test: verify chunks in one cell can be grouped/rendered as one cell, not separate bordered cells +- Adjacent tables test: two separate tables with local cell `0:0` must have different `tableId`/`cellId` and must not merge in demo grouping +- Optional: table fixture JSON in `demo/llm-streams-examples/` for debug runs + +## Files +- New: `src/tree-sitter/table-support.ts` +- Modify: `src/tree-sitter/types.ts`, `block-detection.ts`, `segment-builder.ts`, `segment-generator.ts`, `src/tree-sitter-markdown-stream-parser.ts`, `src/markdown-stream-parser.ts`, `src/tree-sitter-markdown-stream-parser.test.ts`, `README.md`, `demo/svelte-demo/src/routes/+page.svelte` + +## Verification (all in docker) +``` +docker compose up -d +docker exec lixpi-markdown-stream-parser-demo pnpm test:run +docker exec lixpi-markdown-stream-parser-demo pnpm run debug-parser-tree-sitter --file=demo/llm-streams-examples/.json +``` +Visual check: svelte demo dev server (imports `src/` directly) — header row styled, alignment applied. diff --git a/demo/svelte-demo/src/routes/+page.svelte b/demo/svelte-demo/src/routes/+page.svelte index 7613ed1..e7cfa81 100644 --- a/demo/svelte-demo/src/routes/+page.svelte +++ b/demo/svelte-demo/src/routes/+page.svelte @@ -10,6 +10,18 @@ } from "../../../../src/markdown-stream-parser.ts"; type ExampleFile = { base: string; json: string; txt: string }; + type TableAlign = "left" | "center" | "right" | undefined; + type TableCellGroup = { + cellId: string; + columnIndex: number; + type: "table_header_cell" | "table_cell"; + align: TableAlign; + chunks: Chunk[]; + }; + type TableRowGroup = { + rowIndex: number; + cells: TableCellGroup[]; + }; MarkdownStreamParser.configureWasmPath("/tree-sitter-markdown.wasm"); @@ -305,12 +317,74 @@ openSpans = []; } + function getTableAlignClass(align: TableAlign): string { + if (align === "center") return "text-center"; + if (align === "right") return "text-right"; + return "text-left"; + } + + function getTableCellAlignClass(cell: TableCellGroup): string { + if (cell.align) { + return getTableAlignClass(cell.align); + } + + return cell.type === "table_header_cell" ? "text-center" : "text-left"; + } + + function buildTableRows(block: Chunk[]): TableRowGroup[] { + const rows = new Map>(); + + for (const chunk of block) { + const table = chunk.block.table; + if (!table) continue; + + let row = rows.get(table.rowIndex); + if (!row) { + row = new Map(); + rows.set(table.rowIndex, row); + } + + let cell = row.get(table.cellId); + if (!cell) { + cell = { + cellId: table.cellId, + columnIndex: table.columnIndex, + type: + chunk.block.type === "table_header_cell" + ? "table_header_cell" + : "table_cell", + align: table.align, + chunks: [], + }; + row.set(table.cellId, cell); + } + + cell.chunks = [...cell.chunks, chunk]; + } + + return [...rows.entries()] + .sort((a, b) => a[0] - b[0]) + .map(([rowIndex, cells]) => ({ + rowIndex, + cells: [...cells.values()].sort( + (a, b) => a.columnIndex - b.columnIndex, + ), + })); + } + + function isTableCellBlockType(blockType: string | undefined): boolean { + return ( + blockType === "table_header_cell" || blockType === "table_cell" + ); + } + // Group chunks into blocks based on block type changes $: parsedBlocks = (() => { const blocks: Chunk[][] = []; let currentBlock: Chunk[] = []; let lastBlockType: string | undefined = undefined; let lastBlockLevel: number | undefined = undefined; + let lastTableId: string | undefined = undefined; let lastOffset: number = -1; for (const seg of parsedSegments) { @@ -321,12 +395,22 @@ const chunk = seg.chunk; const blockType = chunk.block.type; const blockLevel = chunk.block.level; + const tableId = chunk.block.table?.tableId; // Detect new block: type change, or heading level change // For list items, use gap in offset to detect new item let isNewBlock = false; - if (blockType !== lastBlockType) { + if ( + blockType !== lastBlockType && + !( + isTableCellBlockType(blockType) && + isTableCellBlockType(lastBlockType) && + tableId === lastTableId + ) + ) { + isNewBlock = true; + } else if (tableId !== lastTableId) { isNewBlock = true; } else if (blockType === "heading" && blockLevel !== lastBlockLevel) { isNewBlock = true; @@ -345,6 +429,7 @@ currentBlock.push(chunk); lastBlockType = blockType; lastBlockLevel = blockLevel; + lastTableId = tableId; lastOffset = chunk.offset + chunk.length; } @@ -519,7 +604,8 @@ {@const blockLevel = block[0]?.block.level} {@const blockLanguage = block[0]?.block.language} {@const hasTableCells = - blockType === "table_cell" || blockType === "table_row"} + blockType === "table_header_cell" || blockType === "table_cell"} + {@const tableRows = hasTableCells ? buildTableRows(block) : []}
{#if blockType === "heading"} @@ -686,24 +772,39 @@ {/if} {/each} - {:else if blockType === "table_cell" || blockType === "table_row"} - {#each block as chunk} - {@const styles = [ - ...chunk.contained.map((s) => s.type), - ...chunk.opening.map((s) => s.type), - ]} - - {#if hasCodeStyle(styles)} - {chunk.text} - {:else} - {chunk.text} - {/if} - - {/each} + {:else if blockType === "table_header_cell" || blockType === "table_cell"} +
+ + {#each tableRows as row} + + {#each row.cells as cell} + + {#each cell.chunks as chunk} + {@const styles = [ + ...chunk.contained.map((s) => s.type), + ...chunk.opening.map((s) => s.type), + ]} + {#if hasCodeStyle(styles)} + {chunk.text} + {:else} + {chunk.text} + {/if} + {/each} + + {/each} + + {/each} +
+
{:else} diff --git a/src/markdown-stream-parser.ts b/src/markdown-stream-parser.ts index b1847c4..7fe34d5 100644 --- a/src/markdown-stream-parser.ts +++ b/src/markdown-stream-parser.ts @@ -7,6 +7,8 @@ export type { ClosedSpan, BlockType, BlockContext, + TableAlignment, + TableMetadata, Chunk, RecoveryInfo, StreamingChunk, diff --git a/src/tree-sitter-markdown-stream-parser.test.ts b/src/tree-sitter-markdown-stream-parser.test.ts index ad14059..3fb26f5 100644 --- a/src/tree-sitter-markdown-stream-parser.test.ts +++ b/src/tree-sitter-markdown-stream-parser.test.ts @@ -38,6 +38,26 @@ function applyBacktracks(chunks: Chunk[]): Chunk[] { return activeChunks } +async function parseMarkdownInChunks(instanceId: string, markdown: string, chunkSize: number): Promise { + const parser = await MarkdownStreamParser.getInstance(instanceId) + const chunks: Chunk[] = [] + + parser.subscribeToTokenParse((chunk) => { + if (chunk.status === 'STREAMING' && chunk.chunk) { + chunks.push(chunk.chunk) + } + }) + + parser.startParsing() + for (let index = 0; index < markdown.length; index += chunkSize) { + parser.parseToken(markdown.slice(index, index + chunkSize)) + } + parser.stopParsing() + MarkdownStreamParser.removeInstance(instanceId) + + return chunks +} + describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { let parser: MarkdownStreamParser let parsedChunks: Chunk[] = [] @@ -954,7 +974,7 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { expect(after?.offset).toBe('code\n'.length) }) }) - describe('Table Inline Code', () => { + describe('Table Support', () => { it('should strip backticks from inline code inside tables', async () => { parser.parseToken('| Col | `code` |\n') parser.stopParsing() @@ -974,12 +994,12 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { parser.parseToken('| 1 | 2 |\n') parser.stopParsing() - // Should have table-related chunks - const tableChunks = parsedChunks.filter(c => - c.block.type === 'table' || c.block.type === 'table_row' || c.block.type === 'table_cell' - ) + const activeChunks = applyBacktracks(parsedChunks) + const headerChunks = activeChunks.filter(c => c.block.type === 'table_header_cell') + const bodyChunks = activeChunks.filter(c => c.block.type === 'table_cell') - expect(tableChunks.length).toBeGreaterThan(0) + expect(headerChunks.length).toBeGreaterThan(0) + expect(bodyChunks.length).toBeGreaterThan(0) }) it('should suppress pipe delimiters from output', async () => { @@ -1020,6 +1040,152 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { const hasStrippedCode = codeChunks.some(c => c.text.trim() === 'code') expect(hasStrippedCode).toBe(true) }) + + it('should expose header/body table metadata and alignments', async () => { + parser.parseToken('| Left | Center | Right | Plain |\n') + parser.parseToken('| :--- | :---: | ---: | --- |\n') + parser.parseToken('| a | b | c | d |\n') + parser.stopParsing() + + const activeChunks = applyBacktracks(parsedChunks) + const headerChunks = activeChunks.filter(c => c.block.type === 'table_header_cell') + const bodyChunks = activeChunks.filter(c => c.block.type === 'table_cell') + + expect(headerChunks.length).toBeGreaterThan(0) + expect(bodyChunks.length).toBeGreaterThan(0) + + const leftHeader = headerChunks.find(c => c.text.includes('Left')) + const centerHeader = headerChunks.find(c => c.text.includes('Center')) + const rightHeader = headerChunks.find(c => c.text.includes('Right')) + const plainHeader = headerChunks.find(c => c.text.includes('Plain')) + const bodyCell = bodyChunks.find(c => c.text.trim() === 'c') + + expect(leftHeader?.block.table?.rowIndex).toBe(0) + expect(leftHeader?.block.table?.columnIndex).toBe(0) + expect(leftHeader?.block.table?.align).toBe('left') + expect(centerHeader?.block.table?.align).toBe('center') + expect(rightHeader?.block.table?.align).toBe('right') + expect(plainHeader?.block.table?.align).toBeUndefined() + + expect(bodyCell?.block.table?.rowIndex).toBe(1) + expect(bodyCell?.block.table?.columnIndex).toBe(2) + expect(bodyCell?.block.table?.cellId).toBe( + `${bodyCell?.block.table?.tableId}:1:2` + ) + }) + + it('should keep table parsing stable across small streaming chunk sizes', async () => { + const markdown = '| A | B |\n| --- | --- |\n| New York | 42 |\n' + + for (const chunkSize of [1, 2, 3]) { + const chunks = await parseMarkdownInChunks(`table-stream-${chunkSize}`, markdown, chunkSize) + const activeChunks = applyBacktracks(chunks) + const rendered = activeChunks.map(c => c.text).join('') + + expect(rendered).toContain('A') + expect(rendered).toContain('B') + expect(rendered).toContain('New York') + expect(rendered).not.toContain('|') + expect(rendered).not.toContain('---') + + const tableCells = activeChunks.filter(c => c.block.type === 'table_header_cell' || c.block.type === 'table_cell') + expect(tableCells.length).toBeGreaterThan(0) + expect(new Set(tableCells.map(c => c.block.table?.tableId)).size).toBe(1) + + // "New York" is two words; when split across chunks it may land in + // multiple emitted pieces, but they must all share one cellId so + // consumers can group them back into a single rendered cell. + const cityChunks = tableCells.filter(c => c.text.includes('New') || c.text.includes('York')) + expect(cityChunks.length).toBeGreaterThan(0) + expect(new Set(cityChunks.map(c => c.block.table?.cellId)).size).toBe(1) + } + }) + + it('should carry table metadata for a table at the very start of the stream', async () => { + parser.parseToken('| A |\n') + parser.parseToken('| --- |\n') + parser.parseToken('| 1 |\n') + parser.stopParsing() + + const activeChunks = applyBacktracks(parsedChunks) + const tableChunks = activeChunks.filter( + c => c.block.type === 'table_header_cell' || c.block.type === 'table_cell' + ) + + expect(tableChunks.length).toBeGreaterThan(0) + expect(tableChunks.every(c => typeof c.block.table?.tableId === 'string')).toBe(true) + }) + + it('should carry table metadata for a table at the very end of the stream with no trailing newline', async () => { + parser.parseToken('| A |\n') + parser.parseToken('| --- |\n') + parser.parseToken('| 1 |') + parser.stopParsing() + + const activeChunks = applyBacktracks(parsedChunks) + const tableChunks = activeChunks.filter( + c => c.block.type === 'table_header_cell' || c.block.type === 'table_cell' + ) + + expect(tableChunks.length).toBeGreaterThan(0) + expect(tableChunks.some(c => c.text.includes('1'))).toBe(true) + }) + + it('should not bleed paragraph metadata into a following table', async () => { + parser.parseToken('Some intro text.\n') + parser.parseToken('\n') + parser.parseToken('| A |\n') + parser.parseToken('| --- |\n') + parser.parseToken('| B |\n') + parser.stopParsing() + + const activeChunks = applyBacktracks(parsedChunks) + const paragraphChunks = activeChunks.filter(c => c.block.type === 'paragraph') + const tableChunks = activeChunks.filter( + c => c.block.type === 'table_header_cell' || c.block.type === 'table_cell' + ) + + expect(paragraphChunks.length).toBeGreaterThan(0) + expect(tableChunks.length).toBeGreaterThan(0) + expect(tableChunks.every(c => c.block.list === undefined)).toBe(true) + expect(tableChunks.every(c => typeof c.block.table?.tableId === 'string')).toBe(true) + }) + + it('should not bleed list metadata into a following table', async () => { + parser.parseToken('- item\n') + parser.parseToken('\n') + parser.parseToken('| A |\n') + parser.parseToken('| --- |\n') + parser.parseToken('| B |\n') + parser.stopParsing() + + const activeChunks = applyBacktracks(parsedChunks) + const tableChunks = activeChunks.filter( + c => c.block.type === 'table_header_cell' || c.block.type === 'table_cell' + ) + + expect(tableChunks.length).toBeGreaterThan(0) + expect(tableChunks.every(c => c.block.list === undefined)).toBe(true) + }) + + it('should assign different tableId and cellId values to adjacent tables', async () => { + parser.parseToken('| A |\n') + parser.parseToken('| --- |\n') + parser.parseToken('| 1 |\n') + parser.parseToken('\n') + parser.parseToken('| B |\n') + parser.parseToken('| --- |\n') + parser.parseToken('| 2 |\n') + parser.stopParsing() + + const activeChunks = applyBacktracks(parsedChunks) + const headerChunks = activeChunks.filter(c => c.block.type === 'table_header_cell') + const tableIds = new Set(headerChunks.map(c => c.block.table?.tableId)) + const cellIds = new Set(headerChunks.map(c => c.block.table?.cellId)) + + expect(tableIds.size).toBe(2) + expect(cellIds.size).toBe(2) + }) }) describe('Error Recovery', () => { @@ -1061,6 +1227,14 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { expect(typeof firstBacktrack.backtrackOffset).toBe('number') expect(firstBacktrack.backtrackOffset!).toBeGreaterThanOrEqual(0) + const activeChunks = applyBacktracks(tableChunks) + const headerChunks = activeChunks.filter(c => c.block.type === 'table_header_cell') + expect(headerChunks.length).toBeGreaterThan(0) + const headerText = headerChunks.map(c => c.text).join('') + expect(headerText).toContain('Col A') + expect(headerText).toContain('Col B') + expect(headerChunks.every(c => c.block.table?.rowIndex === 0)).toBe(true) + MarkdownStreamParser.removeInstance(tableId) }) @@ -1103,6 +1277,46 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { expect(fullText).toContain('Age') expect(fullText).toContain('Alice') + const headerChunks = activeChunks.filter(c => c.block.type === 'table_header_cell') + expect(headerChunks.length).toBeGreaterThan(0) + expect(headerChunks.every(c => c.block.table?.rowIndex === 0)).toBe(true) + + MarkdownStreamParser.removeInstance(tableId) + }) + + it('should re-emit header chunks with alignment metadata after table reclassification', async () => { + const tableId = 'test-table-reemit-align' + const tableParser = await MarkdownStreamParser.getInstance(tableId) + const tableChunks: Chunk[] = [] + + tableParser.subscribeToTokenParse((chunk) => { + if (chunk.status === 'STREAMING' && chunk.chunk) { + tableChunks.push(chunk.chunk) + } + }) + + tableParser.startParsing() + tableParser.parseToken('| Left | Center | Right |\n') + tableParser.parseToken('| :--- | :---: | ---: |\n') + tableParser.parseToken('| a | b | c |\n') + tableParser.stopParsing() + + const backtrackChunks = tableChunks.filter(c => c.backtrackOffset !== undefined) + expect(backtrackChunks.length).toBeGreaterThan(0) + + const activeChunks = applyBacktracks(tableChunks) + const headerChunks = activeChunks.filter(c => c.block.type === 'table_header_cell') + + expect(headerChunks.length).toBeGreaterThan(0) + + const leftHeader = headerChunks.find(c => c.text.includes('Left')) + const centerHeader = headerChunks.find(c => c.text.includes('Center')) + const rightHeader = headerChunks.find(c => c.text.includes('Right')) + + expect(leftHeader?.block.table?.align).toBe('left') + expect(centerHeader?.block.table?.align).toBe('center') + expect(rightHeader?.block.table?.align).toBe('right') + MarkdownStreamParser.removeInstance(tableId) }) diff --git a/src/tree-sitter-markdown-stream-parser.ts b/src/tree-sitter-markdown-stream-parser.ts index 3f481d0..9cec637 100644 --- a/src/tree-sitter-markdown-stream-parser.ts +++ b/src/tree-sitter-markdown-stream-parser.ts @@ -24,6 +24,8 @@ export type { ClosedSpan, BlockType, BlockContext, + TableAlignment, + TableMetadata, Chunk, RecoveryInfo, StreamingChunk, diff --git a/src/tree-sitter/block-detection.ts b/src/tree-sitter/block-detection.ts index 03677a5..0fd1038 100644 --- a/src/tree-sitter/block-detection.ts +++ b/src/tree-sitter/block-detection.ts @@ -2,15 +2,19 @@ import type { Node } from 'web-tree-sitter' import { HEADER_MARKER_LEVELS, type BlockInfo, type BlockState } from './types.ts' import { findBlockNode } from './tree-navigation.ts' import { getListMetadata } from './list-support.ts' +import { getTableBlockInfo } from './table-support.ts' // Get the block type and properties from a tree-sitter node. // Walks up the tree to find the enclosing block structure. export function getBlockInfo(node: Node): BlockInfo { const list = getListMetadata(node) + const tableBlockInfo = getTableBlockInfo(node, list) + if (tableBlockInfo) { + return tableBlockInfo + } + let current: Node | null = node let foundParagraph = false - let foundTableCell = false - let isInHeader = false while (current) { switch (current.type) { @@ -36,30 +40,6 @@ export function getBlockInfo(node: Node): BlockInfo { case 'blockquote': // If we found a paragraph inside a blockquote, return blockquote return { type: 'blockquote', list } - // Table types - case 'pipe_table_cell': - foundTableCell = true - break - case 'pipe_table_header': - isInHeader = true - // If we found a cell inside a header, return table_header_cell - if (foundTableCell) { - return { type: 'table_header_cell', id: current.id, list } - } - break - case 'pipe_table_row': - // If we found a cell inside a regular row, return table_cell - if (foundTableCell) { - return { type: 'table_cell', id: current.id, list } - } - break - case 'pipe_table': - // Found the table - if we have a cell, determine type based on header flag - if (foundTableCell) { - return { type: isInHeader ? 'table_header_cell' : 'table_cell', id: current.id, list } - } - // Otherwise just return table - return { type: 'table', list } } current = current.parent diff --git a/src/tree-sitter/segment-builder.ts b/src/tree-sitter/segment-builder.ts index 90eabed..42b075e 100644 --- a/src/tree-sitter/segment-builder.ts +++ b/src/tree-sitter/segment-builder.ts @@ -69,8 +69,9 @@ function mapBlockType(type: string): BlockType { return 'table' case 'pipe_table_row': return 'table_row' - case 'pipe_table_cell': case 'table_header_cell': + return 'table_header_cell' + case 'pipe_table_cell': case 'table_cell': return 'table_cell' case 'blockquote': @@ -96,6 +97,9 @@ export function createBlockContext(blockInfo: BlockInfo): BlockContext { if (blockInfo.list !== undefined) { context.list = blockInfo.list } + if (blockInfo.table !== undefined) { + context.table = blockInfo.table + } return context } diff --git a/src/tree-sitter/segment-generator.ts b/src/tree-sitter/segment-generator.ts index 21686d9..bb1d6f5 100644 --- a/src/tree-sitter/segment-generator.ts +++ b/src/tree-sitter/segment-generator.ts @@ -24,6 +24,7 @@ import { isInsideCodeBlock } from './inline-detection.ts' import { getHeaderContent, getCodeBlockContent, getInlineContent } from './content-extraction.ts' +import { isInsideTableDelimiterRow } from './table-support.ts' import { createChunkFromBlockInfo, createPlainTextChunk, @@ -567,19 +568,14 @@ export function generateSegments( } // Check if we're inside a table delimiter row - let currentForDelimiter: Node | null = nodeAtPosition - while (currentForDelimiter) { - if (currentForDelimiter.type === 'pipe_table_delimiter_row' || - currentForDelimiter.type === 'pipe_table_delimiter_cell') { - state = { - ...state, - sourceOffset: actualToIndex, - accumulatedContent: state.accumulatedContent + newContent - } - state = withCheckpoint(state) - return { segments, state } + if (isInsideTableDelimiterRow(nodeAtPosition)) { + state = { + ...state, + sourceOffset: actualToIndex, + accumulatedContent: state.accumulatedContent + newContent } - currentForDelimiter = currentForDelimiter.parent + state = withCheckpoint(state) + return { segments, state } } // Determine the block type and properties diff --git a/src/tree-sitter/table-support.ts b/src/tree-sitter/table-support.ts new file mode 100644 index 0000000..cfa0b4b --- /dev/null +++ b/src/tree-sitter/table-support.ts @@ -0,0 +1,153 @@ +import type { Node } from 'web-tree-sitter' +import type { BlockInfo, ListMetadata, TableAlignment, TableMetadata } from './types.ts' + +export function isInsideTableDelimiterRow(node: Node): boolean { + let current: Node | null = node + + while (current) { + if (current.type === 'pipe_table_delimiter_row' || current.type === 'pipe_table_delimiter_cell') { + return true + } + current = current.parent + } + + return false +} + +function getEnclosingTable(node: Node): Node | null { + let current: Node | null = node + + while (current) { + if (current.type === 'pipe_table') { + return current + } + current = current.parent + } + + return null +} + +function normalizeAlignment(text: string): TableAlignment | undefined { + const trimmed = text.trim() + const startsWithColon = trimmed.startsWith(':') + const endsWithColon = trimmed.endsWith(':') + + if (startsWithColon && endsWithColon) { + return 'center' + } + if (endsWithColon) { + return 'right' + } + if (startsWithColon) { + return 'left' + } + + return undefined +} + +export function getTableAlignments(tableNode: Node): Array { + const delimiterRow = tableNode.children.find(child => child.type === 'pipe_table_delimiter_row') + if (!delimiterRow) { + return [] + } + + return delimiterRow.children + .filter(child => child.type === 'pipe_table_delimiter_cell') + .map(child => normalizeAlignment(child.text)) +} + +export function getColumnIndex(cellNode: Node): number { + let count = 0 + let sibling = cellNode.previousSibling + + while (sibling) { + if (sibling.type === 'pipe_table_cell') { + count++ + } + sibling = sibling.previousSibling + } + + return count +} + +export function getRowIndex(rowNode: Node): number { + let count = 0 + let sibling = rowNode.previousSibling + + while (sibling) { + if (sibling.type === 'pipe_table_header' || sibling.type === 'pipe_table_row') { + count++ + } + sibling = sibling.previousSibling + } + + return count +} + +export function getTableId(tableNode: Node): string { + return `table:${tableNode.startIndex}` +} + +export function getCellId(tableId: string, rowIndex: number, columnIndex: number): string { + return `${tableId}:${rowIndex}:${columnIndex}` +} + +function createTableMetadata(tableNode: Node, rowNode: Node, cellNode: Node): TableMetadata { + const rowIndex = getRowIndex(rowNode) + const columnIndex = getColumnIndex(cellNode) + const tableId = getTableId(tableNode) + const alignments = getTableAlignments(tableNode) + + return { + tableId, + rowIndex, + columnIndex, + cellId: getCellId(tableId, rowIndex, columnIndex), + align: alignments[columnIndex], + } +} + +export function getTableBlockInfo(node: Node, list?: ListMetadata): BlockInfo | null { + const tableNode = getEnclosingTable(node) + if (!tableNode) { + return null + } + + let current: Node | null = node + let rowNode: Node | null = null + let cellNode: Node | null = null + let isHeaderCell = false + + while (current && current !== tableNode) { + if (current.type === 'pipe_table_cell' && !cellNode) { + cellNode = current + } + if ((current.type === 'pipe_table_header' || current.type === 'pipe_table_row') && !rowNode) { + rowNode = current + } + if (current.type === 'pipe_table_header') { + isHeaderCell = true + } + current = current.parent + } + + if (cellNode && rowNode) { + return { + type: isHeaderCell ? 'table_header_cell' : 'table_cell', + list, + table: createTableMetadata(tableNode, rowNode, cellNode), + } + } + + if (rowNode) { + return { + type: 'table_row', + list, + } + } + + return { + type: 'table', + list, + } +} diff --git a/src/tree-sitter/types.ts b/src/tree-sitter/types.ts index b631c9b..c5d8c35 100644 --- a/src/tree-sitter/types.ts +++ b/src/tree-sitter/types.ts @@ -45,6 +45,7 @@ export type BlockType = | 'list_item' | 'table' | 'table_row' + | 'table_header_cell' | 'table_cell' | 'blockquote' @@ -57,6 +58,8 @@ export type BlockContext = { language?: string // For content inside a Markdown list item. list?: ListMetadata + // For content inside a Markdown table cell. + table?: TableMetadata } export type ListMetadata = { @@ -67,6 +70,16 @@ export type ListMetadata = { task?: { checked: boolean } } +export type TableAlignment = 'left' | 'center' | 'right' + +export type TableMetadata = { + tableId: string + rowIndex: number + columnIndex: number + cellId: string + align?: TableAlignment +} + // ============================================================================ // CHUNK TYPE - Core output unit // ============================================================================ @@ -186,7 +199,7 @@ export type BlockInfo = { level?: number language?: string list?: ListMetadata - id?: number + table?: TableMetadata } // Context passed to inline style extractors From ea65f5eb2bd823301557ce64b0fb22630bf93b00 Mon Sep 17 00:00:00 2001 From: Dmitry Bondarenko Date: Sat, 4 Jul 2026 12:27:12 +0600 Subject: [PATCH 30/32] Updates documentation --- README.md | 6 ++-- TABLES_IMPROVEMENT_PLAN.md | 70 -------------------------------------- 2 files changed, 4 insertions(+), 72 deletions(-) delete mode 100644 TABLES_IMPROVEMENT_PLAN.md diff --git a/README.md b/README.md index 80cf6a6..da3d512 100644 --- a/README.md +++ b/README.md @@ -216,6 +216,8 @@ type BlockContext = { `table` is present when the chunk is inside a table cell. `tableId` identifies the enclosing table, `rowIndex` is zero-based with the header row at `0`, `columnIndex` is zero-based within the row, `cellId` is a stable `${tableId}:${rowIndex}:${columnIndex}` grouping key, and `align` reflects the parsed delimiter row when specified. +Chunks are streamed at content boundaries, not at Markdown table-cell boundaries. One Markdown cell can produce multiple chunks. Consumers that need to rebuild a visual table should group chunks by `block.table.tableId`, then by `rowIndex`, then by `cellId`. + Compatibility note: header cell chunks now use `block.type === 'table_header_cell'`. Consumers that previously treated all table header content as `table_cell` should update that branch. When `includeRawStreamedToken` is enabled, `original` contains the raw Markdown source associated with the emitted chunk. It is separate from the rendered UTF-16 coordinate space. @@ -332,14 +334,14 @@ The parser handles these structures in its exercised parsing paths: - Fenced code blocks with language detection - Ordered, unordered, nested, loose, and task list items - Bold, italic, bold-italic, strikethrough, and inline code spans -- Pipe-table cells and delimiter suppression for covered table forms +- Pipe tables with header-cell detection, delimiter suppression, alignment metadata, and stable per-cell grouping keys for covered table forms Link and image span extraction is implemented, including URL and image metadata, but dedicated coverage is still needed for those paths. These structures are incomplete or unsupported: - Blockquote marker stripping and nested blockquotes -- Full table behavior across all valid table shapes +- Full coverage for every valid Markdown table shape - Horizontal rules - Footnotes - HTML blocks diff --git a/TABLES_IMPROVEMENT_PLAN.md b/TABLES_IMPROVEMENT_PLAN.md deleted file mode 100644 index 6fa2356..0000000 --- a/TABLES_IMPROVEMENT_PLAN.md +++ /dev/null @@ -1,70 +0,0 @@ -# Plan: Improve Markdown Table Parsing (LIX-MDSP-7/tables-support) - -## Context - -The tree-sitter streaming parser has partial table support: it detects `table`/`table_row`/`table_cell` blocks, suppresses pipes and delimiter rows, and handles header reclassification via backtrack. Gaps: header cells are folded into `table_cell` (renderers can't emit ``), column alignment from the delimiter row is discarded, streaming edge cases are under-tested, and table logic is scattered across four files. Legacy `src/state-machine/` is dead code — ignored entirely. All code runs inside the `lixpi-markdown-stream-parser-demo` docker service. - -## Steps - -### 1. Types and public contract -- In `src/tree-sitter/types.ts`: add `'table_header_cell'` to `BlockType` union (~line 46), add `TableAlignment = 'left' | 'center' | 'right'`, add `TableMetadata = { tableId: string; rowIndex: number; columnIndex: number; cellId: string; align?: TableAlignment }`, and add `table?: TableMetadata` to `BlockContext` and internal `BlockInfo` -- `tableId` must be deterministic from table structure and stable across backtrack replay; use a source-derived key such as the enclosing `pipe_table.startIndex`, not a raw tree-sitter `Node.id`. -- `cellId` must be deterministic and stream-unique, e.g. `${tableId}:${rowIndex}:${columnIndex}`. This prevents adjacent tables from merging when two tables both contain a local `0:0` cell and gives consumers a stable way to group word-sized chunks from the same markdown cell. -- In `src/tree-sitter-markdown-stream-parser.ts` and `src/markdown-stream-parser.ts`: re-export `TableAlignment` and `TableMetadata` if they are named public types -- In `README.md`: document `table_header_cell`, `block.table.tableId`, `block.table.rowIndex`, `block.table.columnIndex`, `block.table.cellId`, and `block.table.align`; add a compatibility note for consumers that previously treated all header chunks as `table_cell` - -Breaking only for consumers that expect header cells as `table_cell` — that's the point of the change. There is no changelog file currently; record the compatibility note in README or add a dedicated changelog/release note file as part of this change. - -### 2. New module `src/tree-sitter/table-support.ts` (precedent: `list-support.ts`) -Pure Node-walking functions: -- `isInsideTableDelimiterRow(node)` — extracted from `segment-generator.ts:569-583` -- `getTableAlignments(tableNode)` — parse `pipe_table_delimiter_cell` text (`:---`/`:---:`/`---:`) → normalized alignment per column -- `getColumnIndex(cellNode)` — count preceding `pipe_table_cell` siblings only -- `getRowIndex(rowNode)` — count preceding `pipe_table_header`/`pipe_table_row` siblings only; header row is `0` -- `getTableId(tableNode)` — deterministic public table key, e.g. `table:${tableNode.startIndex}` -- `getCellId(tableId, rowIndex, columnIndex)` — deterministic public grouping key, e.g. `${tableId}:${rowIndex}:${columnIndex}` -- `getTableBlockInfo(node, list?)` — the table branch of `getBlockInfo` moved here; returns `BlockInfo` incl. `table: { tableId, rowIndex, columnIndex, cellId, align? }` - -Keep `isHeader` out of public `TableMetadata` unless a consumer need appears; `table_header_cell` already exposes header-ness. - -### 3. Delegation -- `block-detection.ts:39-62`: replace table case cluster with `getTableBlockInfo(node, list)` call (same pattern as `getListMetadata`); keep `getListMetadata(node)` owned by `getBlockInfo()` so table support does not duplicate list walks/imports -- Remove the now-dead `id` field: the table branches (`block-detection.ts:47,53,59`) are the only place it is ever set and nothing reads it (`isNewBlock` compares type/level/startIndex; `createBlockContext` never copies it), so also delete `id?: number` from `BlockInfo` in `types.ts:189` -- `segment-builder.ts:67-75`: map `table_header_cell` → `table_header_cell` (stop folding); copy `blockInfo.table` in `createBlockContext` (like `list`/`language`) -- Carry explicit table metadata through `BlockContext`; do not use internal `BlockInfo.id` for public grouping - -### 4. `segment-generator.ts` -- Replace inline delimiter walk (569-583) with `isInsideTableDelimiterRow()` -- Alignment flows automatically via existing BlockInfo → BlockContext → chunk path. Timing: header cells emitted pre-delimiter parse as paragraph; the existing backtrack re-emits them as `table_header_cell` with `align` once the delimiter row is in the tree. Body cells carry `tableId`, `rowIndex`, `columnIndex`, `cellId`, and `align` when specified by the delimiter row. -- Robustness fixes driven by Step 6 tests. Known suspects: - - Cell identity/grouping: current chunks are word-sized, so one markdown cell may produce multiple chunks. Group by explicit `table.cellId`; never rely on `blockInfo.id` - - Delimiter row split across chunks transiently parsing as body row (leaking `---`) — verify backtrack corrects; fix empirically - - `windowSize` overflow during header reclassification → must yield `recovery: window_overflow`, not corruption - -### 5. Demo (`demo/svelte-demo/src/routes/+page.svelte`) -- Include `table_header_cell` in `hasTableCells` (~line 521) -- Add header-cell rendering branch (~line 689, bold/th-style) and apply `chunk.block.table?.align` as text alignment on cells -- Do not render every chunk as its own bordered cell. The token buffer emits word-sized chunks, so cells like `New York` can become multiple `table_cell` chunks. Build table display groups by `chunk.block.table.tableId`, then `rowIndex`, then `cellId`; render rows/cells from those groups. This also prevents adjacent tables with matching local row/column positions from merging. -- Keep alignment application closed over known values (`left`/`center`/`right`) via classes or controlled style values; never pass raw delimiter text into a `style` attribute - -### 6. Tests (`src/tree-sitter-markdown-stream-parser.test.ts`, new `describe('Table Support')`) -- Header cells → `table_header_cell`, body → `table_cell` (update existing test at 971-983) -- Alignment left/center/right/undefined + `tableId` + `rowIndex` + `columnIndex` + stable stream-unique `cellId` -- Chunked streaming helper, sizes 1/2/3: no `|`/`---` leakage in reconstructed active output; pipe split across chunks; delimiter row split mid-cell; table at stream start/end; table after paragraph and after list (no `list` metadata bleed) -- Backtrack: re-emitted header chunks carry type + align; small-`windowSize` recovery test -- When asserting leakage or final content, reconstruct the active stream after applying `backtrackOffset`; raw emitted event history may contain transient paragraph chunks before table reclassification -- Multi-word cell test: verify chunks in one cell can be grouped/rendered as one cell, not separate bordered cells -- Adjacent tables test: two separate tables with local cell `0:0` must have different `tableId`/`cellId` and must not merge in demo grouping -- Optional: table fixture JSON in `demo/llm-streams-examples/` for debug runs - -## Files -- New: `src/tree-sitter/table-support.ts` -- Modify: `src/tree-sitter/types.ts`, `block-detection.ts`, `segment-builder.ts`, `segment-generator.ts`, `src/tree-sitter-markdown-stream-parser.ts`, `src/markdown-stream-parser.ts`, `src/tree-sitter-markdown-stream-parser.test.ts`, `README.md`, `demo/svelte-demo/src/routes/+page.svelte` - -## Verification (all in docker) -``` -docker compose up -d -docker exec lixpi-markdown-stream-parser-demo pnpm test:run -docker exec lixpi-markdown-stream-parser-demo pnpm run debug-parser-tree-sitter --file=demo/llm-streams-examples/.json -``` -Visual check: svelte demo dev server (imports `src/` directly) — header row styled, alignment applied. From 0719f87a5237b974061440ee8923383742d7965a Mon Sep 17 00:00:00 2001 From: Dmitry Bondarenko Date: Sat, 4 Jul 2026 12:32:57 +0600 Subject: [PATCH 31/32] Updates documentation --- README.md | 19 ++++++- lists-support-plan.md | 124 ------------------------------------------ 2 files changed, 17 insertions(+), 126 deletions(-) delete mode 100644 lists-support-plan.md diff --git a/README.md b/README.md index c4284f1..0a28338 100644 --- a/README.md +++ b/README.md @@ -204,7 +204,11 @@ type BlockContext = { `level` applies to headings. `language` contains the info string detected on a fenced code block. -`list` is present when the chunk is inside a list item. `depth` is zero-based. Unordered items use `marker` for the bullet character (`-`, `+`, or `*`). Ordered items use `marker` for the delimiter only (`.` or `)`) and put the number in `ordinal` when it is safely representable as a JavaScript number. Task list items omit the `[x]`, `[X]`, or `[ ]` marker from rendered text and expose `task.checked`. +`list` is present when the chunk is inside a list item, including nested blocks such as fenced code blocks contained by a list item. `depth` is zero-based: top-level items use `0`, and nested items use `1` or greater. + +For unordered items, `marker` is the bullet character from the source: `-`, `+`, or `*`. For ordered items, `marker` is the delimiter only: `.` or `)`. The list number is exposed separately as `ordinal` when it is safely representable as a JavaScript number, so `10.` becomes `{ ordinal: 10, marker: '.' }`. + +Task list items omit the checkbox marker and following space from rendered text. `[x]` and `[X]` produce `task: { checked: true }`; `[ ]` produces `task: { checked: false }`. When `includeRawStreamedToken` is enabled, `original` contains the raw Markdown source associated with the emitted chunk. It is separate from the rendered UTF-16 coordinate space. @@ -318,12 +322,23 @@ The parser handles these structures in its exercised parsing paths: - Paragraphs and ATX headings (`#` through `######`) - Fenced code blocks with language detection -- Ordered, unordered, nested, loose, and task list items +- Ordered list items with `.` and `)` delimiters +- Unordered list items with `-`, `+`, and `*` markers +- Nested and loose list items +- Task list items with checked and unchecked state - Bold, italic, bold-italic, strikethrough, and inline code spans - Pipe-table cells and delimiter suppression for covered table forms Link and image span extraction is implemented, including URL and image metadata, but dedicated coverage is still needed for those paths. +### Lists + +List marker syntax is removed from `text`. Consumers should use `block.list` to render bullets, ordered numbers, nesting, and task state instead of parsing the original Markdown source. + +List metadata is attached to all chunks emitted inside a list item. A fenced code block inside a list keeps `block.type === 'code_block'` and also receives `block.list`, so consumers can preserve list indentation while rendering the nested block with its natural block type. + +The parser removes structural list indentation from rendered output. Rendered list item text keeps meaningful content newlines, including blank lines in loose lists and the trailing newline at the end of an item. + These structures are incomplete or unsupported: - Blockquote marker stripping and nested blockquotes diff --git a/lists-support-plan.md b/lists-support-plan.md deleted file mode 100644 index 3ae9705..0000000 --- a/lists-support-plan.md +++ /dev/null @@ -1,124 +0,0 @@ -# Reliable List Support - -## Summary - -Add robust ordered, unordered, nested, loose, and task-list handling to the tree-sitter parser. Keep `block.type === 'list_item'` for list item text, and add optional list metadata so consumers can render bullets, numbers, nesting, and checked state without parsing raw Markdown. - -## Public API Changes - -Extend `BlockContext` with optional metadata: - -```ts -list?: { - type: 'ordered' | 'unordered' - depth: number - marker: '-' | '+' | '*' | '.' | ')' - ordinal?: number - task?: { checked: boolean } -} -``` - -- `depth` is zero-based: top-level list items use `0`, nested list items use `1+`. Compute it as the number of enclosing `list` ancestors minus one. -- For **unordered** lists, `marker` is the bullet character (`-`, `+`, `*`) and `ordinal` is absent. -- For **ordered** lists, `marker` is the *delimiter only* (`.` or `)`); the number lives in `ordinal` when it is safely representable as a JavaScript number. This is documented explicitly because `marker` alone does not reconstruct the source (`"1."` = `ordinal: 1` + `marker: '.'`). -- Task list items strip `[x]`, `[X]`, or `[ ]` (and the following space) from rendered text and expose `task.checked`. - -## Grammar Facts (verified against the bundled WASM) - -These node shapes drive the design and were confirmed by dumping the AST for representative inputs: - -``` -"- [x] done\n" - list_item - list_marker_minus [0,2] "- " - task_list_marker_checked [2,5] "[x]" ← node covers "[x]" only, not the trailing space - paragraph [6,11] "done\n" ← index 5 (the space) belongs to NO node - inline [6,10] "done" - -"- Parent\n - Child\n" - list_item - list_marker_minus [0,2] - paragraph [2,11] "Parent\n " - inline [2,8] "Parent" - block_continuation [9,11] " " ← indent hangs on the PARENT paragraph; \n at 8 is a gap - list (nested) … -``` - -Consequences: - -- Every list/task marker is a **discrete sibling node**, exactly like the markers `SUPPRESSED_SYNTAX_TYPES` already drops — so suppression can be *extended*, not replaced. -- `task_list_marker_*` nodes exclude the trailing space; that space (index 5 above) is an orphan gap owned by no node and must be handled explicitly. -- `block_continuation` is a standalone leaf node, but it is not safe to drop with the existing whole-segment suppression path because a streamed range can start on the continuation and extend into real content. It is also **not list-only** (blockquotes and loose-list blanks produce it), so range-aware filtering must be scoped to list ancestry. -- Ordered `1)` and `1.` parse as **separate `list` nodes**; derive metadata from the marker node, never from assumed list continuity. - -## Implementation Changes - -The approach is **extend the existing node-suppression path first**, and use small scoped range filtering only for source slices that bypass node-at-position suppression. Node suppression already keeps rendered-offset accounting correct for free (it advances `sourceOffset` without advancing `totalUtf16Offset`), which preserves chunk offsets, span offsets, backtracking, and checkpoints without broad new offset math. - -1. **List metadata helpers (AST-derived, stateless).** - - Add helpers to find the enclosing `list_item`, read the marker node type/text, compute `ordered`/`unordered`, extract `ordinal` from the ordered marker text, read `task.checked` from a `task_list_marker_checked` / `task_list_marker_unchecked` sibling, and compute zero-based `depth` as `listAncestorCount - 1`. - - Metadata is a pure function of the node, so it is re-derived per chunk and is inherently backtrack-safe (no dependence on checkpoint state). - - `ordinal` is parsed from the leading digits of the marker text. If the value is greater than `Number.MAX_SAFE_INTEGER`, omit `ordinal` rather than emitting a lossy number. - -2. **Thread metadata through the internal types.** - - Extend internal `BlockInfo` (and `BlockState`, if list data is retained across chunks) with the list fields. - - Derive list metadata by scanning the full ancestor chain independently from block type selection. Do not rely on `getBlockInfo` reaching `list_item`, because nested blocks such as `fenced_code_block` currently return before the walk reaches their enclosing list item. - - Nested blocks (e.g. a code block inside a list) keep their natural `BlockInfo.type` and *also* receive list metadata. - - `createBlockContext` / `mapBlockType` (segment-builder.ts) copy the list metadata onto the public `BlockContext`. `BlockContext.list` is only set inside list context. - -3. **Extend suppression to the new marker nodes.** - - Add `task_list_marker_checked` and `task_list_marker_unchecked` to the suppressed syntax types. - - Do **not** add `block_continuation` to the existing whole-segment early-suppression branch. That branch drops the entire incoming range when `nodeAtPosition` is suppressed; for list-contained code blocks, a streamed range can start on a structural continuation node and continue into real code text. - - Handle list-scoped `block_continuation` with range-aware filtering/splitting instead: remove only the exact continuation-node range, or early-suppress only when `[actualFromIndex, actualToIndex)` is fully contained within that continuation node. - - After stripping a *leading* continuation prefix, re-derive the block/node for the remainder from the post-continuation position. Do not classify or extract the remaining content against the `block_continuation` node it started on. - - Apply this only when the `block_continuation` has an enclosing `list_item` ancestor before any enclosing `blockquote` ancestor. Do not use "nearest block ancestor" because `paragraph` is also a block type. This is the one conditional suppression and must not over-reach. - -4. **Handle the orphan task-marker space.** - - The space between a task marker and its paragraph is owned by no node; without handling it leaks as a lone `" "` `list_item` chunk. Strip it by extending the suppressed task-marker range to swallow following spaces up to, but not including, a newline. Do **not** strip the item's trailing `\n` — existing behavior retains it (e.g. `'Run npm install now\n'`), and that contract stays. - -5. **Filter post-inline structural tails.** - - Current generation appends source text after the inline node directly. That path can leak list structural text such as `\n ` / `block_continuation` even when node suppression handles normal marker ranges. - - Before appending `content.substring(Math.max(actualFromIndex, hostInlineNode.endIndex), actualToIndex)`, remove list-scoped suppressed ranges from that tail using the same suppression decision as marker/task/block-continuation handling. - - Keep real rendered newlines that belong to item text; only remove structural continuation indentation and stripped task-marker spaces. - -6. **Filter list continuations inside code-block extraction.** - - The motivating list-contained code-block case flows through the `getCodeBlockContent` branch, not the generic inline/tail path. - - `getCodeBlockContent` currently skips only `fenced_code_block_delimiter` and `info_string`; also remove list-scoped `block_continuation` ranges there so list indentation around fences and code lines does not leak. - - Preserve code content after a stripped continuation prefix. For example, when a source range starts with structural list indentation and then real code text, strip only the structural prefix and emit the remaining code text as `block.type: 'code_block'`. - -7. **Consolidate duplicated constants (DRY, done as part of this change).** - - Replace the local copies `SUPPRESSED_SYNTAX_TYPES_LOCAL` and `HEADER_MARKER_LEVELS_LOCAL` in `segment-generator.ts` with the exported constants from `types.ts`, so the new marker types are added in exactly one place. - -8. **Docs.** Update README supported-Markdown/API docs, including moving task lists out of Limitations and documenting the `block.list` shape and the ordered `marker`/`ordinal` split. - -## Offset & Recovery Notes - -- Because node suppression handles marker nodes directly, `rawToRenderedOffset` / `collectInlineDelimiterRanges` need **no** list awareness for markers that are their own nodes. Range filtering is limited to orphan task-marker spaces and list-scoped structural continuations, including code-block extraction and post-inline tails that bypass node-at-position suppression. -- The one place to verify carefully is **spans inside a task item**: the `inline` node starts after `[x] ` / `[X] ` / `[ ] `, so a span must render as if the prefix never existed. Confirm `chunkStartUtf16` base math holds when the suppressed prefix and the span text fall in the same word-range/chunk. This is the primary correctness risk and gets a dedicated exact-offset test. -- Split-marker streaming (`-` then ` `; `1` then `.` then ` `) briefly parses as paragraph/other, then backtracks once the marker resolves. Metadata is re-derived from the post-backtrack AST, so assertions target post-backtrack output. -- Checkpoints shallow-copy `currentBlock`; if list metadata is stored there, clone the nested `list` object in checkpoint creation/restoration or treat it as immutable for the full generator lifecycle. - -## Test Plan - -- Unordered markers `-`, `+`, `*`; ordered markers `1.`, `10.`, `1)`, asserting exact `ordinal` (e.g. `10`) and `marker` values, not just detection. -- Ordered marker with a number greater than `Number.MAX_SAFE_INTEGER`: assert `marker` is present and `ordinal` is omitted. -- Nested lists: assert exact zero-based `depth` per level and that no structural indentation (`block_continuation`) leaks into rendered text. -- Loose lists: blank-line items still classify as list items and no indentation leaks. -- Task lists: checked and unchecked items, including uppercase `[X]`, rendered text without `[x]` / `[X]` / `[ ]` **and** without the following space, `task.checked` correct, and the item's trailing `\n` preserved. -- Streaming split-marker tests: `'-'`, `' '`, `'Item\n'`; `'1'`, `'.'`, `' '`, `'Item\n'`; split task-marker chunks — asserting post-backtrack metadata and text. -- Inline span inside a list/task item: assert exact span `offset` and `length` after the stripped list/task syntax (the core offset regression guard). -- List-contained code block: list indentation/fence syntax does not leak through `getCodeBlockContent`, code text after stripped continuation prefixes is preserved, and `block.list` metadata is preserved alongside `block.type: 'code_block'`. -- **Negative tests (guard over-broad suppression):** non-list blockquote continuation still renders its text, and a loose-list blank line does not swallow adjacent content. These ensure the list-scoped `block_continuation` rule does not strip non-list continuations. -- **List inside a blockquote** (`> - a\n> - b\n`): the continuation node's text is `"> "` (it carries the `>` marker), and because it is structural continuation inside a list item, suppress it — assert neither the indent nor the `>` leaks into rendered item text. -- Run verification inside `lixpi-markdown-stream-parser-demo`: - -```sh -docker exec lixpi-markdown-stream-parser-demo pnpm test:run -``` - -## Assumptions - -- The change targets the tree-sitter parser path, which is the public documented parser. -- Existing consumers remain compatible because list metadata is optional and existing `block.type` values are preserved. -- Blockquote behavior inside lists remains limited to current blockquote support unless separately requested. -- The bundled grammar emits `task_list_marker_checked` / `task_list_marker_unchecked` (verified against the WASM in `demo/svelte-demo/static`); no GFM extension toggle is required. From 6883f19acf154e42ca55ac11f33c579d469917cc Mon Sep 17 00:00:00 2001 From: Dmitry Bondarenko Date: Sat, 4 Jul 2026 14:48:49 +0600 Subject: [PATCH 32/32] Improves handling edge case scenarios and overall stability of error recovery --- README.md | 13 +- ...tree-sitter-markdown-stream-parser.test.ts | 95 ++++++++++++- src/tree-sitter-markdown-stream-parser.ts | 127 ++++++++++++++---- src/tree-sitter/segment-builder.ts | 2 +- src/tree-sitter/segment-generator.ts | 16 +-- src/tree-sitter/types.ts | 4 - 6 files changed, 204 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index b67b461..949cdf9 100644 --- a/README.md +++ b/README.md @@ -347,14 +347,13 @@ Leave `windowSize` undefined when a consumer requires complete recovery. ### Recovery Coverage -Recovery is covered for table and code-fence reclassification, rendered offsets, raw source output, and bounded lookback behavior. Dedicated cases are still needed for: +Recovery is covered for table and code-fence reclassification, rendered offsets, raw source output, bounded lookback behavior, and deletion-only corrections. Dedicated cases are still needed for: - Inline delimiter replay with opening and closing spans -- Corrections that only delete stale rendered output ### Long Streams -Checkpoint history is copied as segments are emitted and searched linearly during recovery. Error detection also traverses the syntax tree after streamed input. These paths can accumulate disproportionate work as a document grows. +Checkpoint history is copied as segments are emitted and searched linearly during recovery. Error detection walks only errored syntax subtrees, and replaced tree-sitter trees are released as the document changes. These paths can still accumulate disproportionate work as a document grows. Long uninterrupted input is emitted in bounded chunks by the token buffer, but output can still be delayed until the buffer reaches its internal threshold or receives whitespace. @@ -362,6 +361,8 @@ Long uninterrupted input is emitted in bounded chunks by the token buffer, but o The parser maintains one block syntax tree and one inline parser per instance. Incoming strings pass through a token buffer before incremental parsing. Generated chunks carry rendered offsets, while internal state also retains source offsets for replay. +When incremental parsing succeeds, the parser compares changed ranges against the prior syntax tree, then releases the replaced tree. If parsing does not produce a replacement tree, the parser keeps the existing tree so the next chunk can continue from a valid parser state. + ```mermaid flowchart LR A[Input strings] --> B[Token buffer] @@ -402,7 +403,7 @@ Recorded streams live under `demo/llm-streams-examples`. JSON files preserve chu ```bash docker exec -it lixpi-markdown-stream-parser-demo \ pnpm run debug-parser-tree-sitter \ - --file=demo/llm-streams-examples/claude-3.5-long-regex.json + --file=claude-3.5-long-regex.json ``` Create a chunked JSON stream from a text fixture: @@ -417,9 +418,9 @@ docker exec -it lixpi-markdown-stream-parser-demo \ ## Development Priorities -Recovery work focuses on a strict `windowSize` overflow contract, inline-span replay coverage, and deletion-only correction coverage. +Recovery work focuses on a strict `windowSize` overflow contract and inline-span replay coverage. -Scaling work focuses on stable-boundary checkpoints, pruning and indexed lookup, parser-internal checkpoint storage, changed-subtree error inspection, tracked unresolved errors, and long-stream benchmarks. +Scaling work focuses on stable-boundary checkpoints, indexed lookup, parser-internal checkpoint storage, tracked unresolved errors, and long-stream benchmarks. Markdown coverage work focuses on the incomplete structures listed in [Supported Markdown](#supported-markdown). diff --git a/src/tree-sitter-markdown-stream-parser.test.ts b/src/tree-sitter-markdown-stream-parser.test.ts index 680b1fb..8c18a7e 100644 --- a/src/tree-sitter-markdown-stream-parser.test.ts +++ b/src/tree-sitter-markdown-stream-parser.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' import { MarkdownStreamParser } from './tree-sitter-markdown-stream-parser' -import type { Chunk, ClosedSpan, SpanType } from './tree-sitter/types.ts' +import type { Chunk, ClosedSpan, SpanType, StreamingChunk } from './tree-sitter/types.ts' import path from 'path' import { fileURLToPath } from 'url' import fs from 'fs' @@ -37,6 +37,18 @@ function applyBacktracks(chunks: Chunk[]): Chunk[] { return activeChunks } +function processRawChunk(parser: MarkdownStreamParser, chunk: string): StreamingChunk[] { + return (parser as unknown as { processRawChunk(chunk: string): StreamingChunk[] }).processRawChunk(chunk) +} + +function processRawAndCollect(parser: MarkdownStreamParser, chunk: string, chunks: Chunk[]): void { + for (const segment of processRawChunk(parser, chunk)) { + if (segment.status === 'STREAMING') { + chunks.push(segment.chunk) + } + } +} + describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { let parser: MarkdownStreamParser let parsedChunks: Chunk[] = [] @@ -1032,6 +1044,87 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => { MarkdownStreamParser.removeInstance(rawId) }) + + it('should recover from the errored subtree instead of replaying the full document', async () => { + const recoveryId = 'test-error-offset-subtree' + const recoveryParser = await MarkdownStreamParser.getInstance(recoveryId) + const recoveryChunks: Chunk[] = [] + + recoveryParser.subscribeToTokenParse((chunk) => { + if (chunk.status === 'STREAMING') { + recoveryChunks.push(chunk.chunk) + } + }) + + recoveryParser.startParsing() + processRawAndCollect(recoveryParser, 'Intro text\n\n', recoveryChunks) + processRawAndCollect(recoveryParser, '# [', recoveryChunks) + processRawAndCollect(recoveryParser, ']', recoveryChunks) + recoveryParser.stopParsing() + + const backtrack = recoveryChunks.find(c => c.backtrackOffset !== undefined) + expect(backtrack).toBeDefined() + expect(backtrack!.backtrackOffset).toBeGreaterThan(0) + expect(backtrack!.backtrackOffset).toBeLessThanOrEqual('Intro text\n\n'.length) + + MarkdownStreamParser.removeInstance(recoveryId) + }) + + it('should emit deletion-only recovery chunks when replay produces no replacement segments', async () => { + const recoveryId = 'test-deletion-only-recovery' + const recoveryParser = await MarkdownStreamParser.getInstance(recoveryId) + const recoveryChunks: Chunk[] = [] + + recoveryParser.subscribeToTokenParse((chunk) => { + if (chunk.status === 'STREAMING') { + recoveryChunks.push(chunk.chunk) + } + }) + + recoveryParser.startParsing() + processRawAndCollect(recoveryParser, 'Intro\n\n', recoveryChunks) + processRawAndCollect(recoveryParser, '# [foo', recoveryChunks) + processRawAndCollect(recoveryParser, ']', recoveryChunks) + recoveryParser.stopParsing() + + const deletion = recoveryChunks.find(c => + c.backtrackOffset !== undefined && + c.text === '' && + c.length === 0 + ) + + expect(deletion).toBeDefined() + expect(deletion!.backtrackOffset).toBe('Intro\n\n'.length) + + MarkdownStreamParser.removeInstance(recoveryId) + }) + + it('should preserve older checkpoints across successive recoveries', async () => { + const recoveryId = 'test-successive-recovery-checkpoints' + const recoveryParser = await MarkdownStreamParser.getInstance(recoveryId) + const recoveryChunks: Chunk[] = [] + + recoveryParser.subscribeToTokenParse((chunk) => { + if (chunk.status === 'STREAMING') { + recoveryChunks.push(chunk.chunk) + } + }) + + recoveryParser.startParsing() + recoveryParser.parseToken('Intro\n\n') + recoveryParser.parseToken('| Name | Age |\n') + recoveryParser.parseToken('| --- | --- |\n') + recoveryParser.parseToken('\n# ') + recoveryParser.parseToken('[') + recoveryParser.parseToken('heading](https://example.com)\n') + recoveryParser.stopParsing() + + const backtracks = recoveryChunks.filter(c => c.backtrackOffset !== undefined) + expect(backtracks.length).toBeGreaterThanOrEqual(2) + expect(backtracks[1].backtrackOffset).toBeGreaterThan(0) + + MarkdownStreamParser.removeInstance(recoveryId) + }) }) describe('Code Fence Recovery', () => { diff --git a/src/tree-sitter-markdown-stream-parser.ts b/src/tree-sitter-markdown-stream-parser.ts index 3f481d0..257763e 100644 --- a/src/tree-sitter-markdown-stream-parser.ts +++ b/src/tree-sitter-markdown-stream-parser.ts @@ -9,6 +9,7 @@ import type { SegmentGeneratorState } from './tree-sitter/types.ts' import { generateSegments, createInitialState, stateFromCheckpoint } from './tree-sitter/segment-generator.ts' +import { mapBlockType } from './tree-sitter/segment-builder.ts' type RecoverySelection = { requiredCheckpoint: SegmentGeneratorCheckpoint @@ -53,7 +54,6 @@ export class MarkdownStreamParser { private parser: Parser | null = null private inlineParser: Parser | null = null private currentTree: Tree | null = null - private previousTree: Tree | null = null // Configuration private config: ParserConfig = {} @@ -61,6 +61,7 @@ export class MarkdownStreamParser { // Content state private content: string = '' private lastProcessedIndex: number = 0 + private endPosition: { row: number; column: number } = { row: 0, column: 0 } private allSegments: StreamingChunk[] = [] // Segment generator state @@ -293,11 +294,16 @@ export class MarkdownStreamParser { if (this.generatorState.pendingInlineContent) { const text = this.generatorState.pendingInlineContent + const currentBlock = this.generatorState.currentBlock const chunk: Chunk = { text, offset: this.generatorState.totalUtf16Offset, length: text.length, - block: { type: 'paragraph' }, + block: { + type: currentBlock ? mapBlockType(currentBlock.type) : 'paragraph', + level: currentBlock?.level, + language: currentBlock?.language, + }, opening: [], closing: [], contained: [], @@ -311,7 +317,6 @@ export class MarkdownStreamParser { lastEmittedSourceOffset: this.generatorState.sourceOffset, pendingInlineContent: '', pendingInlineStartIndex: undefined, - accumulatedContent: this.generatorState.accumulatedContent + text, } const streamingChunk: StreamingChunk = { status: 'STREAMING', chunk } @@ -337,30 +342,22 @@ export class MarkdownStreamParser { } const oldLength = this.content.length + const oldEndPosition = this.endPosition this.content += chunk this.lastProcessedIndex = this.content.length + this.endPosition = this.advancePosition(oldEndPosition, chunk) - // Store previous tree for change detection - this.previousTree = this.currentTree + const previousTree = this.currentTree // For proper incremental parsing, tell tree-sitter what changed if (this.currentTree) { - const getPosition = (index: number) => { - const textUpToIndex = this.content.substring(0, Math.min(index, this.content.length)) - const lines = textUpToIndex.split('\n') - return { - row: lines.length - 1, - column: lines[lines.length - 1].length - } - } - this.currentTree.edit({ startIndex: oldLength, oldEndIndex: oldLength, newEndIndex: this.content.length, - startPosition: getPosition(oldLength), - oldEndPosition: getPosition(oldLength), - newEndPosition: getPosition(this.content.length) + startPosition: oldEndPosition, + oldEndPosition, + newEndPosition: this.endPosition }) } @@ -374,8 +371,8 @@ export class MarkdownStreamParser { // Detect backtracking by checking changed source ranges. let affectedSourceOffset: number | undefined - if (this.previousTree && this.currentTree) { - const changedRanges = this.previousTree.getChangedRanges(this.currentTree) + if (previousTree && this.currentTree) { + const changedRanges = previousTree.getChangedRanges(this.currentTree) for (const range of changedRanges) { // range.startIndex is already a UTF-16 character offset in web-tree-sitter JS bindings @@ -387,6 +384,7 @@ export class MarkdownStreamParser { } } } + previousTree?.delete() const errorSourceOffset = this.findEarliestErrorOffset() if (errorSourceOffset !== undefined && errorSourceOffset < this.generatorState.lastEmittedSourceOffset) { @@ -396,7 +394,13 @@ export class MarkdownStreamParser { if (affectedSourceOffset !== undefined) { const recoverySelection = this.selectRecoveryCheckpoint(affectedSourceOffset) const checkpoint = recoverySelection.appliedCheckpoint + const priorCheckpoints = this.generatorState.checkpoints + const preRecoveryLastEmittedOffset = this.generatorState.lastEmittedOffset let state: SegmentGeneratorState = stateFromCheckpoint(checkpoint) + state = { + ...state, + checkpoints: this.restoreCheckpointHistory(priorCheckpoints, checkpoint), + } let backtrackOffset = checkpoint.renderedOffset // Re-generate all segments from backtrack point through end of content. @@ -431,7 +435,7 @@ export class MarkdownStreamParser { firstSeg.chunk.backtrackOffset = backtrackOffset firstSeg.chunk.recovery = recoverySelection.recovery } - } else if (backtrackOffset < this.generatorState.lastEmittedOffset) { + } else if (backtrackOffset < preRecoveryLastEmittedOffset) { allBacktrackSegments.push({ status: 'STREAMING', chunk: { @@ -481,7 +485,6 @@ export class MarkdownStreamParser { openSpans: [], currentBlock: null, pendingInlineContent: '', - accumulatedContent: '', } const checkpoints = this.generatorState.checkpoints.length > 0 @@ -528,19 +531,68 @@ export class MarkdownStreamParser { private findEarliestErrorOffset(): number | undefined { if (!this.currentTree) return undefined - let earliest: number | undefined - const visit = (node: Node) => { - if (node.hasError || node.isError || node.isMissing) { - earliest = Math.min(earliest ?? Infinity, node.startIndex) + let earliestConcrete: number | undefined + let fallback: { offset: number; width: number; depth: number } | undefined + + const visit = (node: Node, depth: number): boolean => { + if (!node.hasError && !node.isError && !node.isMissing) { + return false } + if (node.isError || node.isMissing) { + earliestConcrete = Math.min(earliestConcrete ?? Infinity, node.startIndex) + return true + } + + let recordedChild = false for (const child of node.children) { - visit(child) + recordedChild = visit(child, depth + 1) || recordedChild + } + + if (!recordedChild) { + const candidate = { + offset: node.startIndex, + width: node.endIndex - node.startIndex, + depth, + } + if ( + !fallback || + candidate.depth > fallback.depth || + (candidate.depth === fallback.depth && candidate.width < fallback.width) || + (candidate.depth === fallback.depth && candidate.width === fallback.width && candidate.offset < fallback.offset) + ) { + fallback = candidate + } + return true } + + return true } - visit(this.currentTree.rootNode) - return earliest + visit(this.currentTree.rootNode, 0) + return earliestConcrete ?? fallback?.offset + } + + private restoreCheckpointHistory( + priorCheckpoints: SegmentGeneratorCheckpoint[], + selectedCheckpoint: SegmentGeneratorCheckpoint + ): SegmentGeneratorCheckpoint[] { + const checkpoints = priorCheckpoints + .filter(checkpoint => checkpoint.sourceOffset <= selectedCheckpoint.sourceOffset) + + const selectedIndex = checkpoints.findIndex(checkpoint => + checkpoint.sourceOffset === selectedCheckpoint.sourceOffset && + checkpoint.renderedOffset === selectedCheckpoint.renderedOffset + ) + + const restored = selectedIndex === -1 + ? [...checkpoints, selectedCheckpoint] + : checkpoints.map((checkpoint, index) => index === selectedIndex ? selectedCheckpoint : checkpoint) + + return restored.sort((a, b) => + a.sourceOffset - b.sourceOffset || + a.renderedOffset - b.renderedOffset + ) } @@ -586,6 +638,22 @@ export class MarkdownStreamParser { return char === ' ' || char === '\t' || char === '\n' || char === '\r' } + private advancePosition(position: { row: number; column: number }, text: string): { row: number; column: number } { + let row = position.row + let column = position.column + + for (const char of text) { + if (char === '\n') { + row += 1 + column = 0 + } else { + column += char.length + } + } + + return { row, column } + } + // Get the current accumulated content. getCurrentContent(): string { return this.content @@ -622,8 +690,9 @@ export class MarkdownStreamParser { // Reset the parser state. reset(): void { this.content = '' + this.currentTree?.delete() this.currentTree = null - this.previousTree = null + this.endPosition = { row: 0, column: 0 } this.lastProcessedIndex = 0 this.allSegments = [] this.generatorState = createInitialState() diff --git a/src/tree-sitter/segment-builder.ts b/src/tree-sitter/segment-builder.ts index 55cc18c..7badc93 100644 --- a/src/tree-sitter/segment-builder.ts +++ b/src/tree-sitter/segment-builder.ts @@ -54,7 +54,7 @@ export function utf16ToByteOffset(text: string, utf16Offset: number): number { // ============================================================================ // Map internal block type strings to BlockType enum. -function mapBlockType(type: string): BlockType { +export function mapBlockType(type: string): BlockType { switch (type) { case 'header': case 'atx_heading': diff --git a/src/tree-sitter/segment-generator.ts b/src/tree-sitter/segment-generator.ts index 1de14de..a12c483 100644 --- a/src/tree-sitter/segment-generator.ts +++ b/src/tree-sitter/segment-generator.ts @@ -68,7 +68,6 @@ export function createInitialState(): SegmentGeneratorState { openSpans: [], currentBlock: null, pendingInlineContent: '', - accumulatedContent: '', checkpoints: [] } } @@ -83,7 +82,6 @@ export function createCheckpoint(state: SegmentGeneratorState): SegmentGenerator currentBlock: state.currentBlock ? { ...state.currentBlock } : null, pendingInlineContent: state.pendingInlineContent, pendingInlineStartIndex: state.pendingInlineStartIndex, - accumulatedContent: state.accumulatedContent, } } @@ -97,7 +95,6 @@ export function stateFromCheckpoint(checkpoint: SegmentGeneratorState['checkpoin currentBlock: checkpoint.currentBlock ? { ...checkpoint.currentBlock } : null, pendingInlineContent: checkpoint.pendingInlineContent, pendingInlineStartIndex: checkpoint.pendingInlineStartIndex, - accumulatedContent: checkpoint.accumulatedContent, checkpoints: [checkpoint], } } @@ -313,6 +310,7 @@ function processInlineSpans( chunkEndRaw: number, content: string, state: SegmentGeneratorState, + delimiterRanges: Array<{ start: number; end: number }>, baseRenderedOffset: number ): { opening: OpenSpan[]; closing: ClosedSpan[]; contained: ClosedSpan[]; newOpenSpans: OpenSpan[] } { const opening: OpenSpan[] = [] @@ -320,7 +318,6 @@ function processInlineSpans( const contained: ClosedSpan[] = [] const newOpenSpans = [...state.openSpans] const indicesToRemove: number[] = [] - const delimiterRanges = collectInlineDelimiterRanges(inlineTree) const styleNodeTypes = ['code_span', 'strong_emphasis', 'emphasis', 'strikethrough', 'inline_link', 'image'] @@ -509,7 +506,6 @@ export function generateSegments( lastEmittedOffset: chunkStartUtf16 + newContent.length, sourceOffset: actualToIndex, lastEmittedSourceOffset: actualToIndex, - accumulatedContent: state.accumulatedContent + newContent } state = withCheckpoint(state) return { segments: [chunk], state } @@ -520,7 +516,6 @@ export function generateSegments( state = { ...state, sourceOffset: actualToIndex, - accumulatedContent: state.accumulatedContent + newContent } state = withCheckpoint(state) return { segments, state } @@ -534,7 +529,6 @@ export function generateSegments( state = { ...state, sourceOffset: actualToIndex, - accumulatedContent: state.accumulatedContent + newContent } state = withCheckpoint(state) return { segments, state } @@ -562,7 +556,6 @@ export function generateSegments( state = { ...state, sourceOffset: actualToIndex, - accumulatedContent: state.accumulatedContent + newContent } state = withCheckpoint(state) return { segments, state } @@ -580,7 +573,6 @@ export function generateSegments( state = { ...state, sourceOffset: actualToIndex, - accumulatedContent: state.accumulatedContent + newContent } state = withCheckpoint(state) return { segments, state } @@ -591,7 +583,6 @@ export function generateSegments( state = { ...state, sourceOffset: actualToIndex, - accumulatedContent: state.accumulatedContent + newContent } state = withCheckpoint(state) return { segments, state } @@ -609,6 +600,7 @@ export function generateSegments( const hostInlineNode = findInlineNodeAtPosition(currentTree.rootNode, actualFromIndex) const inlineContent = hostInlineNode?.text ?? processedContent const inlineTree = inlineParser.parse(inlineContent) + const delimiterRanges = collectInlineDelimiterRanges(inlineTree) const chunkStartInInline = hostInlineNode ? Math.max(0, actualFromIndex - hostInlineNode.startIndex) : 0 @@ -622,7 +614,8 @@ export function generateSegments( chunkEndInInline, inlineContent, state, - chunkStartUtf16 - rawToRenderedOffset(chunkStartInInline, collectInlineDelimiterRanges(inlineTree)) + delimiterRanges, + chunkStartUtf16 - rawToRenderedOffset(chunkStartInInline, delimiterRanges) ) opening = spanResult.opening closing = spanResult.closing @@ -663,7 +656,6 @@ export function generateSegments( lastEmittedOffset: chunkStartUtf16 + strippedContent.length, sourceOffset: actualToIndex, lastEmittedSourceOffset: actualToIndex, - accumulatedContent: state.accumulatedContent + newContent, currentBlock: { type: blockInfo.type, level: blockInfo.level, diff --git a/src/tree-sitter/types.ts b/src/tree-sitter/types.ts index c5352ed..4244f40 100644 --- a/src/tree-sitter/types.ts +++ b/src/tree-sitter/types.ts @@ -267,9 +267,6 @@ export type SegmentGeneratorState = { // Start index for pending inline content pendingInlineStartIndex?: number - // Accumulated content for backtrack reference - accumulatedContent: string - // Stable replay points used to translate source recovery ranges to rendered offsets. checkpoints: SegmentGeneratorCheckpoint[] } @@ -283,5 +280,4 @@ export type SegmentGeneratorCheckpoint = { currentBlock: BlockState | null pendingInlineContent: string pendingInlineStartIndex?: number - accumulatedContent: string }