diff --git a/.changeset/update-marked-v18.md b/.changeset/update-marked-v18.md new file mode 100644 index 00000000..17b139a3 --- /dev/null +++ b/.changeset/update-marked-v18.md @@ -0,0 +1,7 @@ +--- +"streamdown": patch +--- + +Update `marked` from `^17.0.1` to `^18.0.11`. + +marked v18 no longer folds a block token's trailing blank line(s) into its own `raw`; that whitespace now surfaces as a separate `space` token immediately following the block (e.g. after `html`, `heading`, and `table` tokens). This changed the token stream shape that `parseMarkdownIntoBlocks` consumes, causing a dangling `space` token after a closed custom-tag HTML block to be emitted as its own standalone block. `parseMarkdownIntoBlocks` now folds a `space` token into the preceding block instead of pushing it as a new one, restoring v17-identical block boundaries and counts. The lossless `token.raw` concatenation invariant is preserved. diff --git a/packages/streamdown/__tests__/custom-tag-markdown.test.tsx b/packages/streamdown/__tests__/custom-tag-markdown.test.tsx index 8310fd02..85661f1a 100644 --- a/packages/streamdown/__tests__/custom-tag-markdown.test.tsx +++ b/packages/streamdown/__tests__/custom-tag-markdown.test.tsx @@ -149,4 +149,27 @@ Content for snippet 2 expect(blocks[0]).toContain(""); expect(blocks[1]).toContain("# Outside"); }); + + it("does not create a standalone block from a trailing blank-line space token (marked v18)", () => { + // marked v18 no longer folds a block token's trailing blank line(s) into + // its own `raw`; it now emits that whitespace as a separate `space` + // token. Block boundaries/count must stay identical to marked v17. + const md = ` + +**bold** + + + +# Outside`; + + const blocks = parseMarkdownIntoBlocks(md); + + expect(blocks.length).toBe(2); + // Lossless concat invariant: joining blocks must reconstruct the input. + expect(blocks.join("")).toBe(md); + // No block should be pure whitespace. + for (const block of blocks) { + expect(block.trim().length).toBeGreaterThan(0); + } + }); }); diff --git a/packages/streamdown/lib/parse-blocks.tsx b/packages/streamdown/lib/parse-blocks.tsx index 8ad09eee..14061bff 100644 --- a/packages/streamdown/lib/parse-blocks.tsx +++ b/packages/streamdown/lib/parse-blocks.tsx @@ -156,6 +156,16 @@ export const parseMarkdownIntoBlocks = (markdown: string): string[] => { } } + // marked v18 no longer absorbs a block token's trailing blank line(s) into + // its own `raw`; instead that whitespace surfaces as a separate `space` + // token immediately after (e.g. html/heading/table blocks). A bare space + // token is never meaningful content on its own, so fold it into the + // previous block to keep block boundaries/counts identical to v17. + if (token.type === "space" && mergedBlocksLen > 0) { + mergedBlocks[mergedBlocksLen - 1] += currentBlock; + continue; + } + // Math block merging logic // If previous block has unclosed math (odd number of $$), merge current block into it. // This handles cases where marked's Lexer splits math blocks (e.g. = on its own line diff --git a/packages/streamdown/package.json b/packages/streamdown/package.json index 9c8b1caf..577934c6 100644 --- a/packages/streamdown/package.json +++ b/packages/streamdown/package.json @@ -40,7 +40,7 @@ "clsx": "^2.1.1", "hast-util-to-jsx-runtime": "^2.3.6", "html-url-attributes": "^3.0.1", - "marked": "^17.0.1", + "marked": "^18.0.11", "rehype-harden": "^1.1.8", "rehype-raw": "^7.0.0", "rehype-sanitize": "^6.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 493efb0f..3bda9877 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -158,7 +158,7 @@ importers: version: 1.6.1(next@16.3.1(@opentelemetry/api@1.9.0)(@types/node@24.10.10)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react@19.2.3) '@vercel/geistdocs': specifier: 1.23.1 - version: 1.23.1(@svta/cml-cta@1.0.1(@svta/cml-structured-field-values@1.0.1(@svta/cml-utils@1.0.1))(@svta/cml-utils@1.0.1))(@svta/cml-structured-field-values@1.0.1(@svta/cml-utils@1.0.1))(@svta/cml-utils@1.0.1)(@tanstack/react-router@1.151.6(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(@types/mdast@4.0.4)(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(next@16.3.1(@opentelemetry/api@1.9.0)(@types/node@24.10.10)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-dom@19.2.3(react@19.2.3))(react@19.2.3)(tailwindcss@4.3.3)(vite@7.3.1(@types/node@24.10.10)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0)) + version: 1.23.1(@svta/cml-cta@1.0.1(@svta/cml-structured-field-values@1.0.1(@svta/cml-utils@1.0.1))(@svta/cml-utils@1.0.1))(@svta/cml-structured-field-values@1.0.1(@svta/cml-utils@1.0.1))(@svta/cml-utils@1.0.1)(@tanstack/react-router@1.151.6(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(@types/mdast@4.0.4)(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(micromark-util-types@2.0.2)(micromark@4.0.2)(next@16.3.1(@opentelemetry/api@1.9.0)(@types/node@24.10.10)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-dom@19.2.3(react@19.2.3))(react@19.2.3)(tailwindcss@4.3.3)(vite@7.3.1(@types/node@24.10.10)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0)) '@vercel/speed-insights': specifier: ^1.3.1 version: 1.3.1(next@16.3.1(@opentelemetry/api@1.9.0)(@types/node@24.10.10)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react@19.2.3) @@ -290,8 +290,8 @@ importers: specifier: ^3.0.1 version: 3.0.1 marked: - specifier: ^17.0.1 - version: 17.0.1 + specifier: ^18.0.11 + version: 18.0.11 react: specifier: ^18.0.0 || ^19.0.0 version: 19.2.3 @@ -5643,6 +5643,11 @@ packages: engines: {node: '>= 20'} hasBin: true + marked@18.0.11: + resolution: {integrity: sha512-HnslJfsZkRPBDJRHvVtAaWlZHEpSu7u8LgQuJCELjRKuWR+hpq4A7sLq3p8HaI9ypVoXDXxV34CsQJEe1+J5Aw==} + engines: {node: '>= 20'} + hasBin: true + mdast-util-find-and-replace@3.0.2: resolution: {integrity: sha512-Tmd1Vg/m3Xz43afeNxDIhWRtFZgM2VLyaf4vSTYwudTyeuTneoL3qtWMA5jeLyz/O1vDJmmV4QuScFCA2tBPwg==} @@ -10196,7 +10201,7 @@ snapshots: '@standard-schema/spec@1.1.0': {} - '@streamdown/cjk@1.0.3(@types/mdast@4.0.4)(react@19.2.3)(unified@11.0.5)': + '@streamdown/cjk@1.0.3(@types/mdast@4.0.4)(micromark-util-types@2.0.2)(micromark@4.0.2)(react@19.2.3)(unified@11.0.5)': dependencies: react: 19.2.3 remark-cjk-friendly: 2.3.1(@types/mdast@4.0.4)(micromark-util-types@2.0.2)(micromark@4.0.2)(unified@11.0.5) @@ -10722,13 +10727,13 @@ snapshots: dependencies: execa: 5.1.1 - '@vercel/geistdocs@1.23.1(@svta/cml-cta@1.0.1(@svta/cml-structured-field-values@1.0.1(@svta/cml-utils@1.0.1))(@svta/cml-utils@1.0.1))(@svta/cml-structured-field-values@1.0.1(@svta/cml-utils@1.0.1))(@svta/cml-utils@1.0.1)(@tanstack/react-router@1.151.6(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(@types/mdast@4.0.4)(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(next@16.3.1(@opentelemetry/api@1.9.0)(@types/node@24.10.10)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-dom@19.2.3(react@19.2.3))(react@19.2.3)(tailwindcss@4.3.3)(vite@7.3.1(@types/node@24.10.10)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0))': + '@vercel/geistdocs@1.23.1(@svta/cml-cta@1.0.1(@svta/cml-structured-field-values@1.0.1(@svta/cml-utils@1.0.1))(@svta/cml-utils@1.0.1))(@svta/cml-structured-field-values@1.0.1(@svta/cml-utils@1.0.1))(@svta/cml-utils@1.0.1)(@tanstack/react-router@1.151.6(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(@types/mdast@4.0.4)(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(micromark-util-types@2.0.2)(micromark@4.0.2)(next@16.3.1(@opentelemetry/api@1.9.0)(@types/node@24.10.10)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-dom@19.2.3(react@19.2.3))(react@19.2.3)(tailwindcss@4.3.3)(vite@7.3.1(@types/node@24.10.10)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0))': dependencies: '@ai-sdk/react': 3.0.201(react@19.2.3)(zod@4.4.3) '@clack/prompts': 0.11.0 '@icons-pack/react-simple-icons': 13.15.1(react@19.2.3) '@orama/tokenizers': 3.1.18 - '@streamdown/cjk': 1.0.3(@types/mdast@4.0.4)(react@19.2.3)(unified@11.0.5) + '@streamdown/cjk': 1.0.3(@types/mdast@4.0.4)(micromark-util-types@2.0.2)(micromark@4.0.2)(react@19.2.3)(unified@11.0.5) '@streamdown/code': 1.1.1(react@19.2.3) '@vercel/agent-readability': 0.6.0(next@16.3.1(@opentelemetry/api@1.9.0)(@types/node@24.10.10)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)) '@vercel/oidc': 3.8.2 @@ -12302,6 +12307,8 @@ snapshots: marked@17.0.1: {} + marked@18.0.11: {} + mdast-util-find-and-replace@3.0.2: dependencies: '@types/mdast': 4.0.4