LIX-MDSP-22 # Bug fixes and test coverage for links

README.md

@@ -175,7 +175,7 @@ type RecoveryInfo = {
 }
 ```
 
-`text` contains rendered text with recognized Markdown syntax removed. `offset`, `length`, span positions, and `backtrackOffset` use UTF-16 code units in the rendered output coordinate space. This matches JavaScript string indexing and `String.prototype.slice()`.
+`text` removes structural markers and emphasis-style delimiters. Link and image source remains in `text`; their spans identify the covered range so a consumer can apply a link mark or replace an image with a node. `offset`, `length`, span positions, and `backtrackOffset` use UTF-16 code units in the rendered output coordinate space. This matches JavaScript string indexing and `String.prototype.slice()`.
 
 `block` describes the surrounding block:
 
@@ -284,7 +284,7 @@ function updateSpanState(chunk: Chunk): void {
 }
 ```
 
-Links include their URL when closed. Images include `src` and may include `alt`.
+Inline links (`[text](url)`) emit a closed `link` span with `url`. Inline images (`![alt](url)`) emit a closed `image` span with `src` and, when present, `alt`. The parser buffers an incomplete link or image until it closes; if the stream ends first, it emits the buffered source as ordinary text.
 
 ## Error Recovery
 
@@ -341,10 +341,9 @@ The parser handles these structures in its exercised parsing paths:
 - Nested and loose list items
 - Task list items with checked and unchecked state
 - Bold, italic, bold-italic, strikethrough, and inline code spans
+- Inline links and images with closed-span URL, source, and alt-text metadata
 - Pipe tables with header-cell detection, delimiter suppression, alignment metadata, and stable per-cell grouping keys for covered table forms
 
-Link and image span extraction is implemented, including URL and image metadata, but dedicated coverage is still needed for those paths.
-
 ### Lists
 
 List marker syntax is removed from `text`. Consumers should use `block.list` to render bullets, ordered numbers, nesting, and task state instead of parsing the original Markdown source.
@@ -361,10 +360,11 @@ These structures are incomplete or unsupported:
 - Footnotes
 - HTML blocks
 - Autolinks
+- Reference links and images
 - Emoji shortcodes
 - Superscript and subscript extensions
 
-Escaped inline markers pass through the delimiter logic, but escaping behavior does not yet have complete feature coverage.
+Escaped inline markers pass through the delimiter logic, but escaping behavior does not yet have complete feature coverage. Link destinations containing nested parentheses, such as `https://en.wikipedia.org/wiki/Foo_(bar)`, are emitted as ordinary text rather than link spans.
 
 ## Limitations
 

demo/llm-streams-examples/test-links-images.json

@@ -0,0 +1,5 @@
+[
+  "Read [doc",
+  "umentation](https://example.com/docs) and ![Project ",
+  "logo](https://example.com/logo.png).\n"
+]

demo/llm-streams-examples/test-links-images.txt

@@ -0,0 +1 @@
+Read [documentation](https://example.com/docs) and ![Project logo](https://example.com/logo.png).

demo/svelte-demo/src/lib/prosemirror/stream-assembly.test.ts

@@ -41,6 +41,11 @@ describe('stream assembly helpers', () => {
         expect(sanitizeLinkHref('https://example.com')).toBe('https://example.com')
         expect(sanitizeLinkHref('mailto:test@example.com')).toBe('mailto:test@example.com')
         expect(sanitizeLinkHref('javascript:alert(1)')).toBeNull()
+        expect(sanitizeLinkHref('JaVaScRiPt:alert(1)')).toBeNull()
+        expect(sanitizeLinkHref('java\tscript:alert(1)')).toBeNull()
+        expect(sanitizeLinkHref('vbscript:msgbox(1)')).toBeNull()
+        expect(sanitizeLinkHref('data:text/html;base64,AAAA')).toBeNull()
+        expect(sanitizeLinkHref('//example.com')).toBeNull()
         expect(sanitizeImageSrc('/asset.png')).toBe('/asset.png')
         expect(sanitizeImageSrc('data:image/png;base64,AAAA')).toBe('data:image/png;base64,AAAA')
         expect(sanitizeImageSrc('//example.com/image.png')).toBeNull()

demo/svelte-demo/src/lib/prosemirror/stream-examples.integration.test.ts

@@ -65,6 +65,22 @@ function hasMark(doc: ParsedExample['doc'], type: string): boolean {
     return found
 }
 
+function nodeAttrs(doc: ParsedExample['doc'], type: string): Record<string, unknown>[] {
+    const attrs: Record<string, unknown>[] = []
+    doc.descendants(node => {
+        if (node.type.name === type) attrs.push(node.attrs)
+    })
+    return attrs
+}
+
+function markAttrs(doc: ParsedExample['doc'], type: string): Record<string, unknown>[] {
+    const attrs: Record<string, unknown>[] = []
+    doc.descendants(node => {
+        for (const mark of node.marks) if (mark.type.name === type) attrs.push(mark.attrs)
+    })
+    return attrs
+}
+
 function taskAttrs(doc: ParsedExample['doc']): unknown[] {
     const attrs: unknown[] = []
     doc.descendants(node => {
@@ -130,15 +146,28 @@ describe('real stream examples to ProseMirror documents', () => {
         expect(taskAttrs(taskDoc)).toEqual([{ checked: true }, { checked: false }])
     })
 
+    it('renders streamed link marks and image nodes with their parsed metadata', async () => {
+        const parsed = await parseExample('test-links-images')
+        const links = markAttrs(parsed.doc, 'link')
+        const images = nodeAttrs(parsed.doc, 'image')
+
+        expect(links.some(link => link.href === 'https://example.com/docs')).toBe(true)
+        expect(images).toContainEqual(expect.objectContaining({ src: 'https://example.com/logo.png', alt: 'Project logo' }))
+        expect(parsed.doc.textContent).toContain('Read')
+    })
+
     it('supports reset, replay after completion, and switching examples at the buffer level', async () => {
         const partial = await parseExample('gpt-4.5-cat-coding', 5)
         const replayA = await parseExample('test-strikethrough')
         const replayB = await parseExample('test-strikethrough')
         const switched = await parseExample('test-error-recovery', 3)
+        const linksA = await parseExample('test-links-images')
+        const linksB = await parseExample('test-links-images')
 
         expect(partial.activeChunks.length).toBeGreaterThan(0)
         expect(buildDocFromChunks(schema, []).textContent).toBe('')
         expect(replayA.doc.eq(replayB.doc)).toBe(true)
+        expect(linksA.doc.eq(linksB.doc)).toBe(true)
         expect(switched.doc.textContent).not.toBe(partial.doc.textContent)
     })
 })

src/tree-sitter-markdown-stream-parser.test.ts

@@ -706,6 +706,174 @@ describe('Tree-Sitter MarkdownStreamParser - Phase 1: Quick Wins', () => {
     })
   })
 
+  describe('Links and Images', () => {
+    it('extracts metadata for complete inline links and images', async () => {
+      parser.parseToken('Read [documentation](https://example.com/docs) and ![project logo](https://example.com/logo.png)\n')
+      parser.stopParsing()
+
+      const activeChunks = applyBacktracks(parsedChunks)
+      const closedSpans = getClosedSpans(activeChunks)
+      const link = closedSpans.find((span): span is ClosedSpan & { type: 'link'; url: string } => span.type === 'link')
+      const image = closedSpans.find((span): span is ClosedSpan & { type: 'image'; src: string; alt?: string } => span.type === 'image')
+
+      expect(link).toMatchObject({
+        type: 'link',
+        url: 'https://example.com/docs',
+        offset: 'Read '.length,
+        length: '[documentation](https://example.com/docs)'.length,
+      })
+      expect(image).toMatchObject({
+        type: 'image',
+        src: 'https://example.com/logo.png',
+        alt: 'project logo',
+        offset: 'Read [documentation](https://example.com/docs) and '.length,
+        length: '![project logo](https://example.com/logo.png)'.length,
+      })
+    })
+
+    it('buffers links and images split across streamed tokens until they close', async () => {
+      parser.parseToken('Read [document')
+      parser.parseToken('ation](https://example.com/docs) and ![project ')
+      parser.parseToken('logo](https://example.com/logo.png)\n')
+      parser.stopParsing()
+
+      const activeChunks = applyBacktracks(parsedChunks)
+      const closedSpans = getClosedSpans(activeChunks)
+      const link = closedSpans.find((span): span is ClosedSpan & { type: 'link'; url: string } => span.type === 'link')
+      const image = closedSpans.find((span): span is ClosedSpan & { type: 'image'; src: string; alt?: string } => span.type === 'image')
+
+      expect(activeChunks.map(chunk => chunk.text).join('')).toBe(
+        'Read [documentation](https://example.com/docs) and ![project logo](https://example.com/logo.png)\n'
+      )
+      expect(link).toMatchObject({ type: 'link', url: 'https://example.com/docs' })
+      expect(image).toMatchObject({ type: 'image', src: 'https://example.com/logo.png', alt: 'project logo' })
+    })
+
+    it('flushes truncated links and images as plain text at end of stream', async () => {
+      parser.parseToken('See [docum')
+      parser.stopParsing()
+
+      expect(parsedChunks.map(chunk => chunk.text).join('')).toBe('See [docum')
+      expect(getClosedSpans(parsedChunks)).toHaveLength(0)
+
+      const imageChunks = await parseMarkdownInChunks('truncated-image', 'See ![lo', 2)
+      expect(imageChunks.map(chunk => chunk.text).join('')).toBe('See ![lo')
+      expect(getClosedSpans(imageChunks)).toHaveLength(0)
+    })
+
+    it('does not retain bracket-like text or images without destinations as open syntax', async () => {
+      parser.parseToken('See [citation 1] for details and ![alt] without a destination\n')
+      parser.stopParsing()
+
+      expect(parsedChunks.map(chunk => chunk.text).join('')).toBe(
+        'See [citation 1] for details and ![alt] without a destination\n'
+      )
+      expect(getClosedSpans(parsedChunks)).toHaveLength(0)
+    })
+
+    it('does not emit partial link or image syntax before the closing token arrives', async () => {
+      parser.parseToken('See [docum')
+      expect(parsedChunks.map(chunk => chunk.text).join('')).toBe('See ')
+
+      parser.parseToken('entation](https://example.com/docs) and ![lo')
+      expect(parsedChunks.map(chunk => chunk.text).join('')).toBe('See [documentation](https://example.com/docs) and ')
+
+      parser.parseToken('go](https://example.com/logo.png)\n')
+      parser.stopParsing()
+
+      const closedSpans = getClosedSpans(applyBacktracks(parsedChunks))
+      expect(closedSpans.filter(span => span.type === 'link')).toHaveLength(1)
+      expect(closedSpans.filter(span => span.type === 'image')).toHaveLength(1)
+    })
+
+    it('parses links and images across character-sized tokens without duplicate spans', async () => {
+      const markdown = 'Links: [one](https://example.com/one) [two](https://example.com/two) ![logo](https://example.com/logo.png)\n'
+      const chunks = applyBacktracks(await parseMarkdownInChunks('character-links-images', markdown, 1))
+      const closedSpans = getClosedSpans(chunks)
+
+      expect(chunks.map(chunk => chunk.text).join('')).toBe(markdown)
+      expect(closedSpans.filter(span => span.type === 'link')).toHaveLength(2)
+      expect(closedSpans.filter(span => span.type === 'image')).toHaveLength(1)
+    })
+
+    it('handles splits between link delimiters, inside URLs, and between ! and [', async () => {
+      parser.parseToken('[split]')
+      parser.parseToken('(https://example.')
+      parser.parseToken('com/path) !')
+      parser.parseToken('[logo](https://example.com/logo.png)')
+      parser.parseToken('\n')
+      parser.stopParsing()
+
+      const closedSpans = getClosedSpans(applyBacktracks(parsedChunks))
+      expect(closedSpans).toContainEqual(expect.objectContaining({ type: 'link', url: 'https://example.com/path' }))
+      expect(closedSpans).toContainEqual(expect.objectContaining({ type: 'image', src: 'https://example.com/logo.png', alt: 'logo' }))
+    })
+
+    it('keeps destinations and span placement for adjacent and nested inline syntax', async () => {
+      parser.parseToken('[a](https://example.com/a)[b](https://example.com/b) *see [docs](https://example.com/docs)* [![logo](logo.png)](https://example.com/home)\n')
+      parser.stopParsing()
+
+      const closedSpans = getClosedSpans(applyBacktracks(parsedChunks))
+      const links = closedSpans.filter((span): span is ClosedSpan & { type: 'link'; url: string } => span.type === 'link')
+      const images = closedSpans.filter((span): span is ClosedSpan & { type: 'image'; src: string; alt?: string } => span.type === 'image')
+
+      expect(links.map(link => link.url)).toEqual([
+        'https://example.com/a',
+        'https://example.com/b',
+        'https://example.com/docs',
+        'https://example.com/home',
+      ])
+      expect(links[1].offset).toBe('[a](https://example.com/a)'.length)
+      expect(images).toHaveLength(1)
+      expect(images[0]).toMatchObject({ src: 'logo.png', alt: 'logo' })
+      expect(closedSpans.some(span => span.type === 'italic')).toBe(true)
+    })
+
+    it('extracts empty, titled, and angle-bracket destinations while pinning parenthesized URLs', async () => {
+      const examples = await Promise.all([
+        parseMarkdownInChunks('empty-link-text', '[](https://example.com)', 100),
+        parseMarkdownInChunks('empty-link-url', '[text]()', 100),
+        parseMarkdownInChunks('empty-image-alt', '![](image.png)', 100),
+        parseMarkdownInChunks('link-title', '[title](https://example.com "A title")', 100),
+        parseMarkdownInChunks('parenthesized-link-url', '[wiki](https://en.wikipedia.org/wiki/Foo_(bar))', 100),
+        parseMarkdownInChunks('angle-link-url', '[space](<https://example.com/a%20b>)', 100),
+      ])
+      const closed = examples.map(chunks => getClosedSpans(applyBacktracks(chunks)))
+
+      expect(closed[0]).toContainEqual(expect.objectContaining({ type: 'link', url: 'https://example.com' }))
+      expect(closed[1]).toContainEqual(expect.objectContaining({ type: 'link', url: '' }))
+      expect(closed[2]).toContainEqual(expect.objectContaining({ type: 'image', src: 'image.png' }))
+      expect(closed[3]).toContainEqual(expect.objectContaining({ type: 'link', url: 'https://example.com' }))
+      // The current streaming parser leaves parenthesized destinations as text.
+      expect(closed[4]).toHaveLength(0)
+      expect(examples[4].map(chunk => chunk.text).join('')).toBe('[wiki](https://en.wikipedia.org/wiki/Foo_(bar))')
+      expect(closed[5]).toContainEqual(expect.objectContaining({ type: 'link', url: '<https://example.com/a%20b>' }))
+    })
+
+    it('leaves escaped, reference, and autolink syntax without inline link spans', async () => {
+      parser.parseToken('\\[not a link\\] [reference][ref] <https://example.com>\n')
+      parser.stopParsing()
+
+      expect(parsedChunks.map(chunk => chunk.text).join('')).toBe('\\[not a link\\] [reference][ref] <https://example.com>\n')
+      expect(getClosedSpans(parsedChunks).filter(span => span.type === 'link')).toHaveLength(0)
+    })
+
+    it('extracts links in headings, list items, blockquotes, and table cells', async () => {
+      parser.parseToken('## [heading](https://example.com/heading)\n- [item](https://example.com/item)\n> [quote](https://example.com/quote)\n\n| [cell](https://example.com/cell) |\n| --- |\n')
+      parser.stopParsing()
+
+      const links = getClosedSpans(applyBacktracks(parsedChunks))
+        .filter((span): span is ClosedSpan & { type: 'link'; url: string } => span.type === 'link')
+
+      expect(links.map(link => link.url)).toEqual([
+        'https://example.com/heading',
+        'https://example.com/item',
+        'https://example.com/quote',
+        'https://example.com/cell',
+      ])
+    })
+  })
+
   describe('Split Inline Code', () => {
     it('should buffer split inline code delimiters across chunks', async () => {
       parser.parseToken('Run `npm')

src/tree-sitter/inline-detection.ts

@@ -1,10 +1,19 @@
 import type { Parser, Tree, Node } from 'web-tree-sitter'
 import { findActiveNodeAtPosition, findInlineNodeAtPosition } from './tree-navigation.ts'
 
+function hasLinkDestination(node: Node): boolean {
+    // tree-sitter omits link_destination for a valid empty destination: [text]().
+    return node.descendantsOfType('link_destination').length > 0 || /\]\(\)$/.test(node.text)
+}
+
 // Check if there's a complete inline_link that overlaps with the given range.
 export function hasCompleteLinkAt(inlineRoot: Node, startPos: number, endPos: number): boolean {
     const linkNodes = inlineRoot.descendantsOfType('inline_link')
     for (const link of linkNodes) {
+        if (!hasLinkDestination(link)) {
+            continue
+        }
+
         // Check if this link overlaps with our range
         if (link.startIndex <= startPos && link.endIndex >= endPos) {
             return true
@@ -21,6 +30,12 @@ export function hasCompleteLinkAt(inlineRoot: Node, startPos: number, endPos: nu
 export function hasCompleteImageAt(inlineRoot: Node, startPos: number, endPos: number): boolean {
     const imageNodes = inlineRoot.descendantsOfType('image')
     for (const img of imageNodes) {
+        // An image description alone (for example, `![alt]` while streaming)
+        // is not a complete inline image yet. Wait for its destination.
+        if (img.descendantsOfType('link_destination').length === 0) {
+            continue
+        }
+
         // Check if this image overlaps with our range
         if (img.startIndex <= startPos && img.endIndex >= endPos) {
             return true
@@ -47,7 +62,9 @@ export function hasIncompleteLinkOpening(text: string, inlineParser: Parser | nu
     // Tree-sitter parses incomplete link structures.
     // Look for link_text nodes that aren't part of a complete inline_link
     const linkTexts = inlineTree.rootNode.descendantsOfType('link_text')
-    const completeLinks = inlineTree.rootNode.descendantsOfType('inline_link')
+    const completeLinks = inlineTree.rootNode
+        .descendantsOfType('inline_link')
+        .filter(hasLinkDestination)
     const completeImages = inlineTree.rootNode.descendantsOfType('image')
 
     for (const linkText of linkTexts) {
@@ -88,10 +105,23 @@ export function hasIncompleteImageOpening(text: string, inlineParser: Parser | n
         return false
     }
 
+    // Tree-sitter does not produce an image_description until the closing `]`
+    // arrives, so detect an unterminated `![` prefix before inspecting the tree.
+    if (text.endsWith('!')) {
+        return true
+    }
+
+    const imageStart = text.lastIndexOf('![')
+    if (imageStart !== -1 && text.indexOf(']', imageStart + 2) === -1) {
+        return true
+    }
+
     // Tree-sitter parses incomplete image structures.
     // Look for image_description nodes that aren't part of a complete image
     const imageDescs = inlineTree.rootNode.descendantsOfType('image_description')
-    const completeImages = inlineTree.rootNode.descendantsOfType('image')
+    const completeImages = inlineTree.rootNode
+        .descendantsOfType('image')
+        .filter(img => img.descendantsOfType('link_destination').length > 0)
 
     for (const desc of imageDescs) {
         let isPartOfComplete = false

src/tree-sitter/segment-generator.ts

@@ -142,11 +142,13 @@ function detectSpanType(nodeType: string): SpanType | null {
 // Extract span metadata (URL for links, src/alt for images)
 function extractSpanMetadata(node: Node): { url?: string; src?: string; alt?: string } {
     if (node.type === 'inline_link') {
-        const destNode = node.descendantsOfType('link_destination')[0]
+        // Nested images also contain a link_destination. Use the link's own
+        // destination rather than the first descendant in document order.
+        const destNode = node.children.find(child => child.type === 'link_destination')
         return { url: destNode?.text ?? '' }
     }
     if (node.type === 'image') {
-        const destNode = node.descendantsOfType('link_destination')[0]
+        const destNode = node.children.find(child => child.type === 'link_destination')
         const descNode = node.descendantsOfType('image_description')[0]
         return {
             src: destNode?.text ?? '',
@@ -445,7 +447,7 @@ export function generateSegments(
             }
 
             // Check for incomplete image opening ![
-            if (newPortion.includes('![')) {
+            if (newPortion.includes('![') || newPortion.endsWith('!')) {
                 const hasCompleteImage = hasCompleteImageAt(inlineTree.rootNode, newPortionStart, newPortionEnd)
                 if (!hasCompleteImage && hasIncompleteImageOpening(newPortion, inlineParser)) {
                     state.pendingInlineContent = newContent