Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .changeset/update-marked-v18.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
"streamdown": patch
---

Update `marked` from `^17.0.1` to `^18.0.11`.

marked v18 no longer folds a block token's trailing blank line(s) into its own `raw`; that whitespace now surfaces as a separate `space` token immediately following the block (e.g. after `html`, `heading`, and `table` tokens). This changed the token stream shape that `parseMarkdownIntoBlocks` consumes, causing a dangling `space` token after a closed custom-tag HTML block to be emitted as its own standalone block. `parseMarkdownIntoBlocks` now folds a `space` token into the preceding block instead of pushing it as a new one, restoring v17-identical block boundaries and counts. The lossless `token.raw` concatenation invariant is preserved.
23 changes: 23 additions & 0 deletions packages/streamdown/__tests__/custom-tag-markdown.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -149,4 +149,27 @@ Content for snippet 2
expect(blocks[0]).toContain("</ai-thinking>");
expect(blocks[1]).toContain("# Outside");
});

it("does not create a standalone block from a trailing blank-line space token (marked v18)", () => {
// marked v18 no longer folds a block token's trailing blank line(s) into
// its own `raw`; it now emits that whitespace as a separate `space`
// token. Block boundaries/count must stay identical to marked v17.
const md = `<ai-thinking>

**bold**

</ai-thinking>

# Outside`;

const blocks = parseMarkdownIntoBlocks(md);

expect(blocks.length).toBe(2);
// Lossless concat invariant: joining blocks must reconstruct the input.
expect(blocks.join("")).toBe(md);
// No block should be pure whitespace.
for (const block of blocks) {
expect(block.trim().length).toBeGreaterThan(0);
}
});
});
10 changes: 10 additions & 0 deletions packages/streamdown/lib/parse-blocks.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,16 @@ export const parseMarkdownIntoBlocks = (markdown: string): string[] => {
}
}

// marked v18 no longer absorbs a block token's trailing blank line(s) into
// its own `raw`; instead that whitespace surfaces as a separate `space`
// token immediately after (e.g. html/heading/table blocks). A bare space
// token is never meaningful content on its own, so fold it into the
// previous block to keep block boundaries/counts identical to v17.
if (token.type === "space" && mergedBlocksLen > 0) {
mergedBlocks[mergedBlocksLen - 1] += currentBlock;
continue;
}

// Math block merging logic
// If previous block has unclosed math (odd number of $$), merge current block into it.
// This handles cases where marked's Lexer splits math blocks (e.g. = on its own line
Expand Down
2 changes: 1 addition & 1 deletion packages/streamdown/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
"clsx": "^2.1.1",
"hast-util-to-jsx-runtime": "^2.3.6",
"html-url-attributes": "^3.0.1",
"marked": "^17.0.1",
"marked": "^18.0.11",
"rehype-harden": "^1.1.8",
"rehype-raw": "^7.0.0",
"rehype-sanitize": "^6.0.0",
Expand Down
19 changes: 13 additions & 6 deletions pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading