From d934f172e2dc8011fd687baf73958e148f8eb9cb Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Sun, 21 Jun 2026 10:57:59 +0200 Subject: [PATCH 1/5] feat: report glob n for interpolated template dynamic imports A dynamic import whose entire argument is a template literal with substitutions returned n: undefined, so a consumer resolving the specifier (a bundler or glob importer) had nothing to work with. It now reports the static skeleton as a glob with each ${...} collapsed to a single "*": import(`./locales/${x}.js`) yields "./locales/*.js". Only a lone template literal qualifies. A template concatenated with anything else (import(`a` + b)) has no static skeleton and still returns undefined. Fixes: https://github.com/guybedford/es-module-lexer/issues/137 --- README.md | 5 +- lexer.js | 99 +++++++++++++++++++++++++++++++++++++++- src/lexer.asm.js | 90 ++++++++++++++++++++++++++++++++++++ src/lexer.ts | 116 +++++++++++++++++++++++++++++++++++++++++++++++ test/_unit.cjs | 28 ++++++++++-- 5 files changed, 333 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index ccc42e8..fa1e2bf 100755 --- a/README.md +++ b/README.md @@ -137,7 +137,8 @@ import { init, parse } from 'es-module-lexer'; source.slice(imports[3].a, imports[3].se - 1); // For non-string dynamic import expressions: - // - n will be undefined + // - n will be undefined, except for a lone template literal: import(`./a/${x}.js`) + // reports n as a glob with each ${...} collapsed to "*" (e.g. "./a/*.js") // - a is currently -1 even if there is an import attribute // - e is currently the character before the closing ) @@ -239,6 +240,8 @@ To handle escape sequences in specifier strings, the `.n` field of imported spec For dynamic import expressions, this field will be empty if not a valid JS string. +When the entire dynamic import argument is a single template literal, `.n` is reported as a glob: each `${...}` substitution is collapsed to a single `*` (for example `` import(`./locales/${locale}.js`) `` yields `./locales/*.js`). A template concatenated with anything else, or any other expression, still resolves to `undefined`. + ### Facade Detection Facade modules that only use import / export syntax can be detected via the third return value (full build only): diff --git a/lexer.js b/lexer.js index fae6cab..2e3d538 100644 --- a/lexer.js +++ b/lexer.js @@ -40,6 +40,20 @@ function readName (impt) { impt.n = readString(s, source.charCodeAt(s - 1)); } +// A dynamic import whose whole argument is a single template literal has no +// constant value, but its static skeleton is a useful glob: each ${...} is +// collapsed to a "*" wildcard (import(`./p/${x}.js`) -> "./p/*.js"). The +// specifier must be the entire argument, so a concatenation such as +// import(`a` + b) leaves n unset (readString stops at the first template's +// closing backtick, which then is not the last token before the close). +function readDynamicTemplateName (impt) { + if (source.charCodeAt(impt.s) !== 96/*`*/) + return; + const n = readString(impt.s + 1, 96/*`*/); + if (acornPos === lastTokenPos + 1) + impt.n = n; +} + // Note: parsing is based on the _assumption_ that the source is already valid export function parse (_source, _name) { openTokenDepth = 0; @@ -137,8 +151,10 @@ export function parse (_source, _name) { syntaxError(); openTokenDepth--; if (curDynamicImport && curDynamicImport.d === openTokenPosStack[openTokenDepth]) { - if (curDynamicImport.e === 0) + if (curDynamicImport.e === 0) { curDynamicImport.e = pos; + readDynamicTemplateName(curDynamicImport); + } curDynamicImport.se = pos; curDynamicImport = null; } @@ -154,6 +170,7 @@ export function parse (_source, _name) { case 44/*,*/: if (curDynamicImport && curDynamicImport.e === 0 && curDynamicImport.d === openTokenPosStack[openTokenDepth - 1]) { curDynamicImport.e = lastTokenPos + 1; + readDynamicTemplateName(curDynamicImport); pos++; commentWhitespace(true); curDynamicImport.a = pos; @@ -507,6 +524,13 @@ function readString (start, quote) { out += readEscapedChar(); chunkStart = acornPos; } + else if (quote === 96/*`*/ && ch === 36/*$*/ && source.charCodeAt(acornPos + 1) === 123/*{*/) { + // Glob a dynamic-import template specifier: each ${...} substitution + // collapses to a single "*" wildcard. + out += source.slice(chunkStart, acornPos) + '*'; + acornPos = skipInterpolation(acornPos + 2); + chunkStart = acornPos; + } else if (ch === 0x2028 || ch === 0x2029) { ++acornPos; } @@ -521,6 +545,79 @@ function readString (start, quote) { return out; } +// `index` is the offset just after a "${" substitution opener. Returns the +// offset just after the matching "}", tracking brace depth and skipping +// strings, nested templates and comments so a "}" inside them does not close +// the substitution early. Regex is not disambiguated (see skipTemplateLiteral). +function skipInterpolation (index) { + let braceDepth = 1; + while (index < source.length) { + const ch = source.charCodeAt(index); + if (ch === 92/*\*/) { + index += 2; + continue; + } + if (ch === 123/*{*/) { + braceDepth++; + } + else if (ch === 125/*}*/) { + if (--braceDepth === 0) + return index + 1; + } + else if (ch === 39/*'*/ || ch === 34/*"*/ || ch === 96/*`*/) { + index = skipQuoted(index + 1, ch); + continue; + } + else if (ch === 47/*/*/) { + const next = source.charCodeAt(index + 1); + if (next === 47/*/*/ || next === 42/***/) { + index = skipComment(index + 2, next === 42/***/); + continue; + } + } + index++; + } + return index; +} + +// `index` is the offset just after the opening quote `quote`. Returns the +// offset just after the closing quote. A backtick run recurses through its own +// ${...} substitutions. +function skipQuoted (index, quote) { + while (index < source.length) { + const ch = source.charCodeAt(index); + if (ch === 92/*\*/) { + index += 2; + continue; + } + if (ch === quote) + return index + 1; + if (quote === 96/*`*/ && ch === 36/*$*/ && source.charCodeAt(index + 1) === 123/*{*/) { + index = skipInterpolation(index + 2); + continue; + } + index++; + } + return index; +} + +// `index` is the offset just after "//" or "/*". Returns the offset just after +// the comment. +function skipComment (index, block) { + while (index < source.length) { + const ch = source.charCodeAt(index); + if (block) { + if (ch === 42/***/ && source.charCodeAt(index + 1) === 47/*/*/) + return index + 2; + } + else if (ch === 10/*\n*/ || ch === 13/*\r*/) { + return index; + } + index++; + } + return index; +} + // Used to read escaped characters function readEscapedChar () { diff --git a/src/lexer.asm.js b/src/lexer.asm.js index f44f310..b0721c6 100644 --- a/src/lexer.asm.js +++ b/src/lexer.asm.js @@ -57,6 +57,16 @@ export function parse (_source, _name = '@') { let n; if (asm.ip()) n = readString(d === -1 ? s : s + 1, source.charCodeAt(d === -1 ? s - 1 : s)); + else if (d !== -1 && source.charCodeAt(s) === 96/*`*/) { + // A dynamic-import template specifier with substitutions has no constant + // value, but its static skeleton is a useful glob (each ${...} -> "*"). + // Only a lone template literal qualifies: readString stops at the + // template's own closing backtick, which is the whole argument's end (e) + // for a glob, but not for a concatenation such as `a` + `b`. + const glob = readString(s + 1, 96/*`*/); + if (acornPos === e) + n = glob; + } let at = null; // minimal build drops the parsed attribute list; es-module-shims reads the // assertion via source.slice(a, se - 1) instead @@ -129,6 +139,13 @@ function readString (start, quote) { out += readEscapedChar(); chunkStart = acornPos; } + else if (quote === 96/*`*/ && ch === 36/*$*/ && source.charCodeAt(acornPos + 1) === 123/*{*/) { + // Glob a dynamic-import template specifier: each ${...} substitution + // collapses to a single "*" wildcard. + out += source.slice(chunkStart, acornPos) + '*'; + acornPos = skipInterpolation(acornPos + 2); + chunkStart = acornPos; + } else if (ch === 0x2028 || ch === 0x2029) { ++acornPos; } @@ -141,6 +158,79 @@ function readString (start, quote) { return out; } +// `index` is the offset just after a "${" substitution opener. Returns the +// offset just after the matching "}", tracking brace depth and skipping +// strings, nested templates and comments so a "}" inside them does not close +// the substitution early. Regex is not disambiguated. +function skipInterpolation (index) { + let braceDepth = 1; + while (index < source.length) { + const ch = source.charCodeAt(index); + if (ch === 92/*\*/) { + index += 2; + continue; + } + if (ch === 123/*{*/) { + braceDepth++; + } + else if (ch === 125/*}*/) { + if (--braceDepth === 0) + return index + 1; + } + else if (ch === 39/*'*/ || ch === 34/*"*/ || ch === 96/*`*/) { + index = skipQuoted(index + 1, ch); + continue; + } + else if (ch === 47/*/*/) { + const next = source.charCodeAt(index + 1); + if (next === 47/*/*/ || next === 42/***/) { + index = skipComment(index + 2, next === 42/***/); + continue; + } + } + index++; + } + return index; +} + +// `index` is the offset just after the opening quote `quote`. Returns the +// offset just after the closing quote. A backtick run recurses through its own +// ${...} substitutions. +function skipQuoted (index, quote) { + while (index < source.length) { + const ch = source.charCodeAt(index); + if (ch === 92/*\*/) { + index += 2; + continue; + } + if (ch === quote) + return index + 1; + if (quote === 96/*`*/ && ch === 36/*$*/ && source.charCodeAt(index + 1) === 123/*{*/) { + index = skipInterpolation(index + 2); + continue; + } + index++; + } + return index; +} + +// `index` is the offset just after "//" or "/*". Returns the offset just after +// the comment. +function skipComment (index, block) { + while (index < source.length) { + const ch = source.charCodeAt(index); + if (block) { + if (ch === 42/***/ && source.charCodeAt(index + 1) === 47/*/*/) + return index + 2; + } + else if (ch === 10/*\n*/ || ch === 13/*\r*/) { + return index; + } + index++; + } + return index; +} + // Used to read escaped characters function readEscapedChar () { diff --git a/src/lexer.ts b/src/lexer.ts index 2c6ba68..174c489 100755 --- a/src/lexer.ts +++ b/src/lexer.ts @@ -50,6 +50,11 @@ export interface ImportSpecifier { * For dynamic import expressions, this field will be empty if not a valid JS string. * For static import expressions, this field will always be populated. * + * A dynamic import whose entire argument is a single template literal is + * reported as a glob: each `${...}` substitution is collapsed to a single + * `*`. Other expressions (including a template concatenated with anything + * else) remain undefined. + * * @example * const [imports1, exports1] = parse(String.raw`import './\u0061\u0062.js'`); * imports1[0].n; @@ -62,6 +67,10 @@ export interface ImportSpecifier { * const [imports3, exports3] = parse(`import("./" + "ab.js")`); * imports3[0].n; * // Returns undefined + * + * const [imports4, exports4] = parse('import(`./locales/${locale}.js`)'); + * imports4[0].n; + * // Returns "./locales/*.js" */ readonly n: string | undefined; /** @@ -263,6 +272,8 @@ export function parse (source: string, name = '@'): readonly [ let n; if (wasm.ip()) n = decode(source.slice(d === -1 ? s - 1 : s, d === -1 ? e + 1 : e)); + else if (d !== -1 && source[s] === '`') + n = decodeTemplate(source.slice(s, e)); let at: Array<[string, string]> | null = null; // minimal build drops the parsed attribute list; es-module-shims reads the // assertion via source.slice(a, se - 1) instead @@ -302,9 +313,114 @@ export function parse (source: string, name = '@'): readonly [ return str; } + // `str` spans the whole dynamic-import argument including the backticks. A + // lone template literal is reported as a glob with each ${...} substitution + // collapsed to a single "*"; the substituted source is then evaluated so the + // static parts are cooked. Anything else (e.g. `a` + b) closes before the end + // and returns undefined. + function decodeTemplate (str: string) { + let out = '`', chunkStart = 1, index = 1; + const last = str.length - 1; + while (index < last) { + const ch = str.charCodeAt(index); + if (ch === 92/*\*/) { + index += 2; + continue; + } + if (ch === 96/*`*/) + return; + if (ch === 36/*$*/ && str.charCodeAt(index + 1) === 123/*{*/) { + out += str.slice(chunkStart, index) + '*'; + index = skipInterpolation(str, index + 2); + chunkStart = index; + continue; + } + index++; + } + if (str.charCodeAt(last) !== 96/*`*/) + return; + return decode(out + str.slice(chunkStart, last) + '`'); + } + return (MINIMAL ? [imports, exports] : [imports, exports, !!wasm.f(), !!wasm.ms()]) as ReturnType; } +// `index` is the offset just after a "${" substitution opener within `str`. +// Returns the offset just after the matching "}", tracking brace depth and +// skipping strings, nested templates and comments so a "}" inside them does not +// close the substitution early. Regex is not disambiguated. +function skipInterpolation (str: string, index: number): number { + let braceDepth = 1; + const len = str.length; + while (index < len) { + const ch = str.charCodeAt(index); + if (ch === 92/*\*/) { + index += 2; + continue; + } + if (ch === 123/*{*/) { + braceDepth++; + } + else if (ch === 125/*}*/) { + if (--braceDepth === 0) + return index + 1; + } + else if (ch === 39/*'*/ || ch === 34/*"*/ || ch === 96/*`*/) { + index = skipQuoted(str, index + 1, ch); + continue; + } + else if (ch === 47/*/*/) { + const next = str.charCodeAt(index + 1); + if (next === 47/*/*/ || next === 42/***/) { + index = skipComment(str, index + 2, next === 42/***/); + continue; + } + } + index++; + } + return index; +} + +// `index` is the offset just after the opening quote `quote`. Returns the +// offset just after the closing quote. A backtick run recurses through its own +// ${...} substitutions. +function skipQuoted (str: string, index: number, quote: number): number { + const len = str.length; + while (index < len) { + const ch = str.charCodeAt(index); + if (ch === 92/*\*/) { + index += 2; + continue; + } + if (ch === quote) + return index + 1; + if (quote === 96/*`*/ && ch === 36/*$*/ && str.charCodeAt(index + 1) === 123/*{*/) { + index = skipInterpolation(str, index + 2); + continue; + } + index++; + } + return index; +} + +// `index` is the offset just after "//" or "/*". Returns the offset just after +// the comment. +function skipComment (str: string, index: number, block: boolean): number { + const len = str.length; + while (index < len) { + const ch = str.charCodeAt(index); + if (block) { + if (ch === 42/***/ && str.charCodeAt(index + 1) === 47/*/*/) + return index + 2; + } + else if (ch === 10/*\n*/ || ch === 13/*\r*/) { + return index; + } + index++; + } + return index; +} + function copyBE (src: string, outBuf16: Uint16Array) { const len = src.length; let i = 0; diff --git a/test/_unit.cjs b/test/_unit.cjs index ac151d8..c893e77 100755 --- a/test/_unit.cjs +++ b/test/_unit.cjs @@ -416,12 +416,34 @@ suite('Lexer', () => { test(`Dynamic import no-substitution template specifier`, () => { // A no-substitution template literal is a constant string, so n is set the - // same as for the quoted forms; an interpolated template has no constant - // value, so n stays undefined. + // same as for the quoted forms. assert.strictEqual(parse('import("./x.js")')[0][0].n, './x.js'); assert.strictEqual(parse("import('./x.js')")[0][0].n, './x.js'); assert.strictEqual(parse('import(`./x.js`)')[0][0].n, './x.js'); - assert.strictEqual(parse('import(`./a${x}.js`)')[0][0].n, undefined); + }); + + test(`Dynamic import interpolated template specifier glob`, () => { + // An interpolated template literal has no constant value, but its static + // skeleton is a useful glob: each ${...} collapses to a single "*". + assert.strictEqual(parse('import(`./a${x}.js`)')[0][0].n, './a*.js'); + assert.strictEqual(parse('import(`./p/${x}/${y}.js`)')[0][0].n, './p/*/*.js'); + assert.strictEqual(parse('import(`${x}`)')[0][0].n, '*'); + assert.strictEqual(parse('import(`${x}/tail`)')[0][0].n, '*/tail'); + // The substitution body may itself contain braces, strings, templates and + // comments without closing the glob early. + assert.strictEqual(parse('import(`a${ {y:1} }b`)')[0][0].n, 'a*b'); + assert.strictEqual(parse('import(`a${ `n${z}` }b`)')[0][0].n, 'a*b'); + assert.strictEqual(parse("import(`a${ obj['}'] }b`)")[0][0].n, 'a*b'); + assert.strictEqual(parse('import(`a${ x /* } */ }b`)')[0][0].n, 'a*b'); + assert.strictEqual(parse('import(`a${ x // }\n }b`)')[0][0].n, 'a*b'); + // A glob is still reported when an import-attributes argument follows. + assert.strictEqual(parse('import(`./p/${x}.js`, { with: { type: "json" } })')[0][0].n, './p/*.js'); + // Only a lone template literal globs; a concatenation has no static + // skeleton, so n stays undefined. + assert.strictEqual(parse('import(`a` + b)')[0][0].n, undefined); + assert.strictEqual(parse('import(`a${x}` + `b${y}`)')[0][0].n, undefined); + assert.strictEqual(parse('import("a" + x)')[0][0].n, undefined); + assert.strictEqual(parse('import(x)')[0][0].n, undefined); }); test(`Dynamic import template specifier with escapes and attributes`, () => { From f9a1f5e5ae47e5164f16c2c9d11e424bb4ddb462 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Tue, 30 Jun 2026 08:31:06 +0200 Subject: [PATCH 2/5] fix: drop interpolated template glob to undefined on ambiguous slash The glob walker that builds a dynamic-import template skeleton skips over strings, nested templates, and comments inside each ${...} substitution, but it cannot tell a regex literal from division without the main parser's token context. A regex carrying a "}" closed the substitution early and emitted a wrong specifier instead of bailing: import(`a${ /x}y/g }b`) reported "a*y/g }b" rather than undefined. skipInterpolation now flags a bare "/" (one that does not open a // or /* comment) and the three decoders drop n to undefined rather than guess. This over-bails the rare division case (import(`a${ b/c }d`)), which is acceptable: a missing glob is recoverable, a wrong one is not. Fixes: https://github.com/guybedford/es-module-lexer/issues/137 --- README.md | 2 +- lexer.js | 15 +++++++++++++-- src/lexer.asm.js | 15 +++++++++++++-- src/lexer.ts | 16 ++++++++++++++-- test/_unit.cjs | 11 +++++++++++ 5 files changed, 52 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index fa1e2bf..70ab453 100755 --- a/README.md +++ b/README.md @@ -240,7 +240,7 @@ To handle escape sequences in specifier strings, the `.n` field of imported spec For dynamic import expressions, this field will be empty if not a valid JS string. -When the entire dynamic import argument is a single template literal, `.n` is reported as a glob: each `${...}` substitution is collapsed to a single `*` (for example `` import(`./locales/${locale}.js`) `` yields `./locales/*.js`). A template concatenated with anything else, or any other expression, still resolves to `undefined`. +When the entire dynamic import argument is a single template literal, `.n` is reported as a glob: each `${...}` substitution is collapsed to a single `*` (for example `` import(`./locales/${locale}.js`) `` yields `./locales/*.js`). A template concatenated with anything else, or any other expression, still resolves to `undefined`. A substitution containing a `/` that could open a regex literal cannot be disambiguated from division without full token context, so the glob is dropped to `undefined` rather than risk a wrong specifier. ### Facade Detection diff --git a/lexer.js b/lexer.js index 2e3d538..95fa5c5 100644 --- a/lexer.js +++ b/lexer.js @@ -49,8 +49,9 @@ function readName (impt) { function readDynamicTemplateName (impt) { if (source.charCodeAt(impt.s) !== 96/*`*/) return; + interpolationError = false; const n = readString(impt.s + 1, 96/*`*/); - if (acornPos === lastTokenPos + 1) + if (!interpolationError && acornPos === lastTokenPos + 1) impt.n = n; } @@ -512,6 +513,11 @@ function tryParseExportStatement () { * THE SOFTWARE. */ let acornPos; +// Set by skipInterpolation when a ${...} substitution contains a "/" that +// could open a regex literal, which the glob walker cannot disambiguate from +// division. A regex "}" would close the substitution early and yield a wrong +// glob, so the caller drops n to undefined instead. +let interpolationError; function readString (start, quote) { acornPos = start; let out = '', chunkStart = acornPos; @@ -548,7 +554,11 @@ function readString (start, quote) { // `index` is the offset just after a "${" substitution opener. Returns the // offset just after the matching "}", tracking brace depth and skipping // strings, nested templates and comments so a "}" inside them does not close -// the substitution early. Regex is not disambiguated (see skipTemplateLiteral). +// the substitution early. A "/" that opens a regex literal cannot be told +// apart from division here without the main parser's token context, and a +// regex may carry a "}" that would close the substitution early; the bare "/" +// case sets interpolationError so the caller drops the glob to undefined +// rather than emit a wrong skeleton. function skipInterpolation (index) { let braceDepth = 1; while (index < source.length) { @@ -574,6 +584,7 @@ function skipInterpolation (index) { index = skipComment(index + 2, next === 42/***/); continue; } + interpolationError = true; } index++; } diff --git a/src/lexer.asm.js b/src/lexer.asm.js index b0721c6..fa27b6f 100644 --- a/src/lexer.asm.js +++ b/src/lexer.asm.js @@ -63,8 +63,9 @@ export function parse (_source, _name = '@') { // Only a lone template literal qualifies: readString stops at the // template's own closing backtick, which is the whole argument's end (e) // for a glob, but not for a concatenation such as `a` + `b`. + interpolationError = false; const glob = readString(s + 1, 96/*`*/); - if (acornPos === e) + if (!interpolationError && acornPos === e) n = glob; } let at = null; @@ -127,6 +128,11 @@ export function parse (_source, _name = '@') { * THE SOFTWARE. */ let acornPos; +// Set by skipInterpolation when a ${...} substitution contains a "/" that +// could open a regex literal, which the glob walker cannot disambiguate from +// division. A regex "}" would close the substitution early and yield a wrong +// glob, so the caller drops n to undefined instead. +let interpolationError; function readString (start, quote) { acornPos = start; let out = '', chunkStart = acornPos; @@ -161,7 +167,11 @@ function readString (start, quote) { // `index` is the offset just after a "${" substitution opener. Returns the // offset just after the matching "}", tracking brace depth and skipping // strings, nested templates and comments so a "}" inside them does not close -// the substitution early. Regex is not disambiguated. +// the substitution early. A "/" that opens a regex literal cannot be told +// apart from division here without the main parser's token context, and a +// regex may carry a "}" that would close the substitution early; the bare "/" +// case sets interpolationError so the caller drops the glob to undefined +// rather than emit a wrong skeleton. function skipInterpolation (index) { let braceDepth = 1; while (index < source.length) { @@ -187,6 +197,7 @@ function skipInterpolation (index) { index = skipComment(index + 2, next === 42/***/); continue; } + interpolationError = true; } index++; } diff --git a/src/lexer.ts b/src/lexer.ts index 174c489..66e7386 100755 --- a/src/lexer.ts +++ b/src/lexer.ts @@ -321,6 +321,7 @@ export function parse (source: string, name = '@'): readonly [ function decodeTemplate (str: string) { let out = '`', chunkStart = 1, index = 1; const last = str.length - 1; + interpolationError = false; while (index < last) { const ch = str.charCodeAt(index); if (ch === 92/*\*/) { @@ -337,7 +338,7 @@ export function parse (source: string, name = '@'): readonly [ } index++; } - if (str.charCodeAt(last) !== 96/*`*/) + if (interpolationError || str.charCodeAt(last) !== 96/*`*/) return; return decode(out + str.slice(chunkStart, last) + '`'); } @@ -345,10 +346,20 @@ export function parse (source: string, name = '@'): readonly [ return (MINIMAL ? [imports, exports] : [imports, exports, !!wasm.f(), !!wasm.ms()]) as ReturnType; } +// Set by skipInterpolation when a ${...} substitution contains a "/" that could +// open a regex literal, which the glob walker cannot disambiguate from +// division. A regex "}" would close the substitution early and yield a wrong +// glob, so decodeTemplate drops n to undefined instead. +let interpolationError: boolean; + // `index` is the offset just after a "${" substitution opener within `str`. // Returns the offset just after the matching "}", tracking brace depth and // skipping strings, nested templates and comments so a "}" inside them does not -// close the substitution early. Regex is not disambiguated. +// close the substitution early. A "/" that opens a regex literal cannot be told +// apart from division here without the main parser's token context, and a regex +// may carry a "}" that would close the substitution early; the bare "/" case +// sets interpolationError so decodeTemplate drops the glob to undefined rather +// than emit a wrong skeleton. function skipInterpolation (str: string, index: number): number { let braceDepth = 1; const len = str.length; @@ -375,6 +386,7 @@ function skipInterpolation (str: string, index: number): number { index = skipComment(str, index + 2, next === 42/***/); continue; } + interpolationError = true; } index++; } diff --git a/test/_unit.cjs b/test/_unit.cjs index c893e77..fd14e00 100755 --- a/test/_unit.cjs +++ b/test/_unit.cjs @@ -436,6 +436,17 @@ suite('Lexer', () => { assert.strictEqual(parse("import(`a${ obj['}'] }b`)")[0][0].n, 'a*b'); assert.strictEqual(parse('import(`a${ x /* } */ }b`)')[0][0].n, 'a*b'); assert.strictEqual(parse('import(`a${ x // }\n }b`)')[0][0].n, 'a*b'); + // A "/" inside the substitution may open a regex literal, which the glob + // walker cannot tell apart from division without the parser's token + // context. A regex "}" would otherwise close the substitution early and + // produce a wrong skeleton, so n drops to undefined rather than guess. + assert.strictEqual(parse('import(`a${ /x}y/g }b`)')[0][0].n, undefined); + assert.strictEqual(parse('import(`a${ /x/g }b`)')[0][0].n, undefined); + assert.strictEqual(parse('import(`a${ b/c }d`)')[0][0].n, undefined); + assert.strictEqual(parse('import(`a${ `n${ /x}/g }` }b`)')[0][0].n, undefined); + // A "/" inside a string or comment is an ordinary character and keeps the + // glob. + assert.strictEqual(parse("import(`a${ x['/}'] }b`)")[0][0].n, 'a*b'); // A glob is still reported when an import-attributes argument follows. assert.strictEqual(parse('import(`./p/${x}.js`, { with: { type: "json" } })')[0][0].n, './p/*.js'); // Only a lone template literal globs; a concatenation has no static From 05df32b23b4a2bb4354d656de54a84f81251788f Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Sun, 5 Jul 2026 12:58:50 +0200 Subject: [PATCH 3/5] refactor: disambiguate template glob substitutions in the parser The interpolated-template glob walked each ${...} substitution with a hand-rolled scanner in all three decoders. None of them could tell a regex literal from division without token context, so a regex carrying a "}" closed the substitution early: import(`a${ /x}y/g }b`) reported "a*y/g }b". The prior fix bailed to undefined on any bare "/", which also dropped legitimate division (import(`a${ b/c }d`)). The parser already resolves regex vs division for the whole source and descends into ${ ... } for nested-import detection, so it now records each top-level substitution's end on the dynamic import (struct TemplateSpan). The decoders splice a "*" per span and jump the body, dropping their skipInterpolation / skipQuoted / skipComment scanners and the interpolationError bail. Both ambiguous cases now resolve correctly: "a*b" and "a*d". Fixes: https://github.com/guybedford/es-module-lexer/issues/137 --- chompfile.toml | 12 ++-- lexer.js | 160 +++++++++++++++++------------------------------ src/lexer.asm.js | 123 +++++++++--------------------------- src/lexer.c | 34 +++++++++- src/lexer.h | 44 +++++++++++++ src/lexer.ts | 129 ++++++++------------------------------ test/_unit.cjs | 15 +++-- 7 files changed, 204 insertions(+), 313 deletions(-) diff --git a/chompfile.toml b/chompfile.toml index 90705fc..f92b527 100644 --- a/chompfile.toml +++ b/chompfile.toml @@ -137,7 +137,7 @@ run = """ ${{ EMSDK_PATH }}/emsdk activate 6.0.0 ${{ EMSDK_PATH }}/upstream/emscripten/emcc src/lexer.c -o lib/lexer.wasm -Oz -flto --no-entry -s STANDALONE_WASM=1 \ - -s EXPORTED_FUNCTIONS="['_parse','_sa','_e','_ri','_re','_it','_is','_ie','_ss','_ip','_se','_ai','_id','_es','_ee','_els','_ele','_ess','_f','_ms','_ra','_rsa','_aks','_ake','_avs','_ave','___heap_base']" \ + -s EXPORTED_FUNCTIONS="['_parse','_sa','_e','_ri','_re','_it','_is','_ie','_ss','_ip','_se','_ai','_id','_es','_ee','_els','_ele','_ess','_f','_ms','_ra','_rsa','_aks','_ake','_avs','_ave','_rt','_te','_rts','___heap_base']" \ -s SUPPORT_LONGJMP=0 -s ALLOW_MEMORY_GROWTH=1 -s ERROR_ON_UNDEFINED_SYMBOLS=0 -Wno-logical-op-parentheses -Wno-parentheses """ @@ -154,7 +154,7 @@ run = """ ${{ EMSDK_PATH }}/emsdk activate 6.0.0 ${{ EMSDK_PATH }}/upstream/emscripten/emcc src/lexer.c -DLEXER_MIN -o lib/lexer.min.wasm -Oz -flto --no-entry -s STANDALONE_WASM=1 \ - -s EXPORTED_FUNCTIONS="['_parse','_sa','_e','_ri','_re','_it','_is','_ie','_ss','_ip','_se','_ai','_id','_es','_ee','_els','_ele','___heap_base']" \ + -s EXPORTED_FUNCTIONS="['_parse','_sa','_e','_ri','_re','_it','_is','_ie','_ss','_ip','_se','_ai','_id','_es','_ee','_els','_ele','_rt','_te','_rts','___heap_base']" \ -s SUPPORT_LONGJMP=0 -s ALLOW_MEMORY_GROWTH=1 -s ERROR_ON_UNDEFINED_SYMBOLS=0 -Wno-logical-op-parentheses -Wno-parentheses """ @@ -167,7 +167,7 @@ run = """ ${{ EMSDK_FASTCOMP_PATH }}/emsdk activate 1.40.1-fastcomp ${{ EMSDK_FASTCOMP_PATH }}/fastcomp/emscripten/emcc ./src/lexer.c -o lib/lexer.emcc.js -s WASM=0 -Oz --closure 1 \ - -s EXPORTED_FUNCTIONS="['_parse','_sa','_e','_ri','_re','_it','_is','_ie','_ss','_ip','_se','_ai','_id','_es','_ee','_els','_ele','_ess','_f','_ms','_ra','_rsa','_aks','_ake','_avs','_ave','_setSource']" \ + -s EXPORTED_FUNCTIONS="['_parse','_sa','_e','_ri','_re','_it','_is','_ie','_ss','_ip','_se','_ai','_id','_es','_ee','_els','_ele','_ess','_f','_ms','_ra','_rsa','_aks','_ake','_avs','_ave','_rt','_te','_rts','_setSource']" \ -s AGGRESSIVE_VARIABLE_ELIMINATION=1 -s GLOBAL_BASE=8 \ -s ERROR_ON_UNDEFINED_SYMBOLS=0 -s TOTAL_STACK=4997968 -s --separate-asm -Wno-logical-op-parentheses -Wno-parentheses """ @@ -187,7 +187,7 @@ run = """ ${{ EMSDK_FASTCOMP_PATH }}/emsdk activate 1.40.1-fastcomp ${{ EMSDK_FASTCOMP_PATH }}/fastcomp/emscripten/emcc ./src/lexer.c -o lib/lexer.layout.js -s WASM=0 -Oz --closure 0 \ - -s EXPORTED_FUNCTIONS="['_parse','_sa','_e','_ri','_re','_it','_is','_ie','_ss','_ip','_se','_ai','_id','_es','_ee','_els','_ele','_ess','_f','_ms','_ra','_rsa','_aks','_ake','_avs','_ave','_setSource']" \ + -s EXPORTED_FUNCTIONS="['_parse','_sa','_e','_ri','_re','_it','_is','_ie','_ss','_ip','_se','_ai','_id','_es','_ee','_els','_ele','_ess','_f','_ms','_ra','_rsa','_aks','_ake','_avs','_ave','_rt','_te','_rts','_setSource']" \ -s AGGRESSIVE_VARIABLE_ELIMINATION=1 -s GLOBAL_BASE=8 \ -s ERROR_ON_UNDEFINED_SYMBOLS=0 -s TOTAL_STACK=4997968 -Wno-logical-op-parentheses -Wno-parentheses """ @@ -204,7 +204,7 @@ run = """ ${{ EMSDK_FASTCOMP_PATH }}/emsdk activate 1.40.1-fastcomp ${{ EMSDK_FASTCOMP_PATH }}/fastcomp/emscripten/emcc ./src/lexer.c -DLEXER_MIN -o lib/lexer.min.emcc.js -s WASM=0 -Oz --closure 1 \ - -s EXPORTED_FUNCTIONS="['_parse','_sa','_e','_ri','_re','_it','_is','_ie','_ss','_ip','_se','_ai','_id','_es','_ee','_els','_ele','_setSource']" \ + -s EXPORTED_FUNCTIONS="['_parse','_sa','_e','_ri','_re','_it','_is','_ie','_ss','_ip','_se','_ai','_id','_es','_ee','_els','_ele','_rt','_te','_rts','_setSource']" \ -s AGGRESSIVE_VARIABLE_ELIMINATION=1 -s GLOBAL_BASE=8 \ -s ERROR_ON_UNDEFINED_SYMBOLS=0 -s TOTAL_STACK=4997968 -s --separate-asm -Wno-logical-op-parentheses -Wno-parentheses """ @@ -221,7 +221,7 @@ run = """ ${{ EMSDK_FASTCOMP_PATH }}/emsdk activate 1.40.1-fastcomp ${{ EMSDK_FASTCOMP_PATH }}/fastcomp/emscripten/emcc ./src/lexer.c -DLEXER_MIN -o lib/lexer.min.layout.js -s WASM=0 -Oz --closure 0 \ - -s EXPORTED_FUNCTIONS="['_parse','_sa','_e','_ri','_re','_it','_is','_ie','_ss','_ip','_se','_ai','_id','_es','_ee','_els','_ele','_setSource']" \ + -s EXPORTED_FUNCTIONS="['_parse','_sa','_e','_ri','_re','_it','_is','_ie','_ss','_ip','_se','_ai','_id','_es','_ee','_els','_ele','_rt','_te','_rts','_setSource']" \ -s AGGRESSIVE_VARIABLE_ELIMINATION=1 -s GLOBAL_BASE=8 \ -s ERROR_ON_UNDEFINED_SYMBOLS=0 -s TOTAL_STACK=4997968 -Wno-logical-op-parentheses -Wno-parentheses """ diff --git a/lexer.js b/lexer.js index 95fa5c5..169eea5 100644 --- a/lexer.js +++ b/lexer.js @@ -13,10 +13,17 @@ let source, pos, end, imports, exports, exportStatementStart, + // While a dynamic import's argument is a lone template literal, this holds + // that import and specifierTemplateDepth the openTokenDepth of its Template + // level, so only its own top-level ${...} spans are recorded. undefined + // otherwise. This reuses the real tokenizer's regex/division disambiguation + // (the main loop below), which a standalone ${...} skimmer cannot do. + templateSpanImport, + specifierTemplateDepth, name; function addImport (ss, s, e, d) { - const impt = { ss, se: d === -2 ? e : d === -1 ? e + 1 : 0, s, e, d, a: -1, n: undefined, at: null }; + const impt = { ss, se: d === -2 ? e : d === -1 ? e + 1 : 0, s, e, d, a: -1, n: undefined, at: null, spans: undefined }; imports.push(impt); return impt; } @@ -42,23 +49,47 @@ function readName (impt) { // A dynamic import whose whole argument is a single template literal has no // constant value, but its static skeleton is a useful glob: each ${...} is -// collapsed to a "*" wildcard (import(`./p/${x}.js`) -> "./p/*.js"). The -// specifier must be the entire argument, so a concatenation such as -// import(`a` + b) leaves n unset (readString stops at the first template's -// closing backtick, which then is not the last token before the close). +// collapsed to a "*" wildcard (import(`./p/${x}.js`) -> "./p/*.js"). The parser +// records each top-level ${...} substitution's end (impt.spans) as it goes, +// using the real tokenizer to resolve the regex/division ambiguity, so this +// only splices a "*" per substitution and cooks the static parts. A +// concatenation such as import(`a` + b) records no specifier spans and its +// walk hits an inner backtick before the argument end, so n stays undefined. function readDynamicTemplateName (impt) { - if (source.charCodeAt(impt.s) !== 96/*`*/) + const spans = impt.spans; + if (spans === undefined || source.charCodeAt(impt.s) !== 96/*`*/) return; - interpolationError = false; - const n = readString(impt.s + 1, 96/*`*/); - if (!interpolationError && acornPos === lastTokenPos + 1) - impt.n = n; + const last = impt.e - 1; + if (source.charCodeAt(last) !== 96/*`*/) + return; + acornPos = impt.s + 1; + let out = '', chunkStart = acornPos, spanIndex = 0; + while (acornPos < last) { + const ch = source.charCodeAt(acornPos); + if (ch === 92/*\*/) { + out += source.slice(chunkStart, acornPos); + out += readEscapedChar(); + chunkStart = acornPos; + } + else if (ch === 96/*`*/) { + return; + } + else if (ch === 36/*$*/ && source.charCodeAt(acornPos + 1) === 123/*{*/ && spanIndex < spans.length) { + out += source.slice(chunkStart, acornPos) + '*'; + acornPos = chunkStart = spans[spanIndex++]; + } + else { + ++acornPos; + } + } + impt.n = out + source.slice(chunkStart, last); } // Note: parsing is based on the _assumption_ that the source is already valid export function parse (_source, _name) { openTokenDepth = 0; curDynamicImport = null; + templateSpanImport = undefined; templateDepth = -1; lastTokenPos = -1; lastSlashWasDivision = false; @@ -195,6 +226,10 @@ export function parse (_source, _name) { syntaxError(); if (openTokenDepth-- === templateDepth) { templateDepth = templateStack[--templateStackDepth]; + // A top-level ${...} of the specifier template just closed: record the + // position after "}" for readDynamicTemplateName to splice a "*". + if (templateSpanImport !== undefined && openTokenDepth === specifierTemplateDepth) + templateSpanImport.spans.push(pos + 1); templateString(); } else { @@ -249,6 +284,14 @@ export function parse (_source, _name) { break; } case 96/*`*/: + // A backtick that is the first token of an open dynamic import's + // argument opens the specifier template: record its top-level ${...} + // spans so readDynamicTemplateName can glob it without re-tokenizing. + if (curDynamicImport && curDynamicImport.e === 0 && pos === curDynamicImport.s) { + templateSpanImport = curDynamicImport; + templateSpanImport.spans = []; + specifierTemplateDepth = openTokenDepth; + } templateString(); break; } @@ -513,11 +556,6 @@ function tryParseExportStatement () { * THE SOFTWARE. */ let acornPos; -// Set by skipInterpolation when a ${...} substitution contains a "/" that -// could open a regex literal, which the glob walker cannot disambiguate from -// division. A regex "}" would close the substitution early and yield a wrong -// glob, so the caller drops n to undefined instead. -let interpolationError; function readString (start, quote) { acornPos = start; let out = '', chunkStart = acornPos; @@ -530,13 +568,6 @@ function readString (start, quote) { out += readEscapedChar(); chunkStart = acornPos; } - else if (quote === 96/*`*/ && ch === 36/*$*/ && source.charCodeAt(acornPos + 1) === 123/*{*/) { - // Glob a dynamic-import template specifier: each ${...} substitution - // collapses to a single "*" wildcard. - out += source.slice(chunkStart, acornPos) + '*'; - acornPos = skipInterpolation(acornPos + 2); - chunkStart = acornPos; - } else if (ch === 0x2028 || ch === 0x2029) { ++acornPos; } @@ -551,84 +582,6 @@ function readString (start, quote) { return out; } -// `index` is the offset just after a "${" substitution opener. Returns the -// offset just after the matching "}", tracking brace depth and skipping -// strings, nested templates and comments so a "}" inside them does not close -// the substitution early. A "/" that opens a regex literal cannot be told -// apart from division here without the main parser's token context, and a -// regex may carry a "}" that would close the substitution early; the bare "/" -// case sets interpolationError so the caller drops the glob to undefined -// rather than emit a wrong skeleton. -function skipInterpolation (index) { - let braceDepth = 1; - while (index < source.length) { - const ch = source.charCodeAt(index); - if (ch === 92/*\*/) { - index += 2; - continue; - } - if (ch === 123/*{*/) { - braceDepth++; - } - else if (ch === 125/*}*/) { - if (--braceDepth === 0) - return index + 1; - } - else if (ch === 39/*'*/ || ch === 34/*"*/ || ch === 96/*`*/) { - index = skipQuoted(index + 1, ch); - continue; - } - else if (ch === 47/*/*/) { - const next = source.charCodeAt(index + 1); - if (next === 47/*/*/ || next === 42/***/) { - index = skipComment(index + 2, next === 42/***/); - continue; - } - interpolationError = true; - } - index++; - } - return index; -} - -// `index` is the offset just after the opening quote `quote`. Returns the -// offset just after the closing quote. A backtick run recurses through its own -// ${...} substitutions. -function skipQuoted (index, quote) { - while (index < source.length) { - const ch = source.charCodeAt(index); - if (ch === 92/*\*/) { - index += 2; - continue; - } - if (ch === quote) - return index + 1; - if (quote === 96/*`*/ && ch === 36/*$*/ && source.charCodeAt(index + 1) === 123/*{*/) { - index = skipInterpolation(index + 2); - continue; - } - index++; - } - return index; -} - -// `index` is the offset just after "//" or "/*". Returns the offset just after -// the comment. -function skipComment (index, block) { - while (index < source.length) { - const ch = source.charCodeAt(index); - if (block) { - if (ch === 42/***/ && source.charCodeAt(index + 1) === 47/*/*/) - return index + 2; - } - else if (ch === 10/*\n*/ || ch === 13/*\r*/) { - return index; - } - index++; - } - return index; -} - // Used to read escaped characters function readEscapedChar () { @@ -846,8 +799,13 @@ function templateString () { templateDepth = ++openTokenDepth; return; } - if (ch === 96/*`*/) + if (ch === 96/*`*/) { + // The specifier template just closed. Stop recording so a following + // concatenated template (`a${x}` + `b${y}`) never appends its spans. + if (templateSpanImport !== undefined && openTokenDepth === specifierTemplateDepth) + templateSpanImport = undefined; return; + } if (ch === 92/*\*/) pos++; } diff --git a/src/lexer.asm.js b/src/lexer.asm.js index fa27b6f..2dc50f0 100644 --- a/src/lexer.asm.js +++ b/src/lexer.asm.js @@ -57,17 +57,8 @@ export function parse (_source, _name = '@') { let n; if (asm.ip()) n = readString(d === -1 ? s : s + 1, source.charCodeAt(d === -1 ? s - 1 : s)); - else if (d !== -1 && source.charCodeAt(s) === 96/*`*/) { - // A dynamic-import template specifier with substitutions has no constant - // value, but its static skeleton is a useful glob (each ${...} -> "*"). - // Only a lone template literal qualifies: readString stops at the - // template's own closing backtick, which is the whole argument's end (e) - // for a glob, but not for a concatenation such as `a` + `b`. - interpolationError = false; - const glob = readString(s + 1, 96/*`*/); - if (!interpolationError && acornPos === e) - n = glob; - } + else if (d !== -1 && source.charCodeAt(s) === 96/*`*/) + n = decodeTemplate(s, e); let at = null; // minimal build drops the parsed attribute list; es-module-shims reads the // assertion via source.slice(a, se - 1) instead @@ -128,11 +119,6 @@ export function parse (_source, _name = '@') { * THE SOFTWARE. */ let acornPos; -// Set by skipInterpolation when a ${...} substitution contains a "/" that -// could open a regex literal, which the glob walker cannot disambiguate from -// division. A regex "}" would close the substitution early and yield a wrong -// glob, so the caller drops n to undefined instead. -let interpolationError; function readString (start, quote) { acornPos = start; let out = '', chunkStart = acornPos; @@ -145,13 +131,6 @@ function readString (start, quote) { out += readEscapedChar(); chunkStart = acornPos; } - else if (quote === 96/*`*/ && ch === 36/*$*/ && source.charCodeAt(acornPos + 1) === 123/*{*/) { - // Glob a dynamic-import template specifier: each ${...} substitution - // collapses to a single "*" wildcard. - out += source.slice(chunkStart, acornPos) + '*'; - acornPos = skipInterpolation(acornPos + 2); - chunkStart = acornPos; - } else if (ch === 0x2028 || ch === 0x2029) { ++acornPos; } @@ -164,82 +143,40 @@ function readString (start, quote) { return out; } -// `index` is the offset just after a "${" substitution opener. Returns the -// offset just after the matching "}", tracking brace depth and skipping -// strings, nested templates and comments so a "}" inside them does not close -// the substitution early. A "/" that opens a regex literal cannot be told -// apart from division here without the main parser's token context, and a -// regex may carry a "}" that would close the substitution early; the bare "/" -// case sets interpolationError so the caller drops the glob to undefined -// rather than emit a wrong skeleton. -function skipInterpolation (index) { - let braceDepth = 1; - while (index < source.length) { - const ch = source.charCodeAt(index); - if (ch === 92/*\*/) { - index += 2; - continue; - } - if (ch === 123/*{*/) { - braceDepth++; - } - else if (ch === 125/*}*/) { - if (--braceDepth === 0) - return index + 1; - } - else if (ch === 39/*'*/ || ch === 34/*"*/ || ch === 96/*`*/) { - index = skipQuoted(index + 1, ch); - continue; - } - else if (ch === 47/*/*/) { - const next = source.charCodeAt(index + 1); - if (next === 47/*/*/ || next === 42/***/) { - index = skipComment(index + 2, next === 42/***/); - continue; - } - interpolationError = true; - } - index++; - } - return index; -} - -// `index` is the offset just after the opening quote `quote`. Returns the -// offset just after the closing quote. A backtick run recurses through its own -// ${...} substitutions. -function skipQuoted (index, quote) { - while (index < source.length) { - const ch = source.charCodeAt(index); +// `[s, e)` spans the dynamic-import template argument, backticks included. The +// parser records each top-level ${...} substitution's end position (see struct +// TemplateSpan in lexer.c), which already resolves the regex-vs-division +// ambiguity the decoder cannot. This cooks the static parts (escapes and all, +// via readString's machinery), emits a single "*" per substitution and jumps +// its body via the recorded end. A concatenation such as `a${x}` + b records +// only its first template's spans, so the walk never reaches the argument end +// (its closing backtick is not at e - 1) and n stays undefined. +function decodeTemplate (s, e) { + const last = e - 1; + if (source.charCodeAt(last) !== 96/*`*/) + return; + asm.rts(); + acornPos = s + 1; + let out = '', chunkStart = acornPos; + while (acornPos < last) { + const ch = source.charCodeAt(acornPos); if (ch === 92/*\*/) { - index += 2; - continue; + out += source.slice(chunkStart, acornPos); + out += readEscapedChar(); + chunkStart = acornPos; } - if (ch === quote) - return index + 1; - if (quote === 96/*`*/ && ch === 36/*$*/ && source.charCodeAt(index + 1) === 123/*{*/) { - index = skipInterpolation(index + 2); - continue; + else if (ch === 96/*`*/) { + return; } - index++; - } - return index; -} - -// `index` is the offset just after "//" or "/*". Returns the offset just after -// the comment. -function skipComment (index, block) { - while (index < source.length) { - const ch = source.charCodeAt(index); - if (block) { - if (ch === 42/***/ && source.charCodeAt(index + 1) === 47/*/*/) - return index + 2; + else if (ch === 36/*$*/ && source.charCodeAt(acornPos + 1) === 123/*{*/ && asm.rt()) { + out += source.slice(chunkStart, acornPos) + '*'; + acornPos = chunkStart = asm.te(); } - else if (ch === 10/*\n*/ || ch === 13/*\r*/) { - return index; + else { + ++acornPos; } - index++; } - return index; + return out + source.slice(chunkStart, last); } // Used to read escaped characters diff --git a/src/lexer.c b/src/lexer.c index 8443414..1adc738 100755 --- a/src/lexer.c +++ b/src/lexer.c @@ -145,8 +145,24 @@ static inline __attribute__((always_inline)) bool consumeToken (char16_t ch) { break; case '}': if (openTokenDepth == 0) return syntaxError(), false; - if (openTokenStack[--openTokenDepth].token == TemplateBrace) + if (openTokenStack[--openTokenDepth].token == TemplateBrace) { + // A top-level ${...} of the specifier template just closed: record the + // position after "}" so the decoders splice a "*" for it. Only the + // specifier's own substitutions qualify (openTokenDepth back at the + // Template's depth + 1), never a nested or concatenated template's. + if (templateSpanImport != NULL && openTokenDepth == specifierTemplateDepth + 1) { + TemplateSpan* span = (TemplateSpan*)(analysis_head); + analysis_head = analysis_head + sizeof(TemplateSpan); + span->end = pos + 1; + span->next = NULL; + if (template_span_write_head == NULL) + templateSpanImport->template_spans = span; + else + template_span_write_head->next = span; + template_span_write_head = span; + } templateString(); + } break; case '\'': case '"': @@ -156,6 +172,17 @@ static inline __attribute__((always_inline)) bool consumeToken (char16_t ch) { isComment = handleSlash(); break; case '`': + // A backtick that is the first token of an open dynamic import's argument + // opens the specifier template: record its top-level ${...} spans so the + // decoders can glob it without re-tokenizing (see struct TemplateSpan). + if (dynamicImportStackDepth > 0 && openTokenDepth > 0 && + openTokenStack[openTokenDepth - 1].token == ImportParen && + lastTokenPos == openTokenStack[openTokenDepth - 1].pos && + dynamicImportStack[dynamicImportStackDepth - 1]->end == 0) { + templateSpanImport = dynamicImportStack[dynamicImportStackDepth - 1]; + specifierTemplateDepth = openTokenDepth; + template_span_write_head = NULL; + } openTokenStack[openTokenDepth].pos = lastTokenPos; openTokenStack[openTokenDepth++].token = Template; templateString(); @@ -180,6 +207,7 @@ bool parse () { #endif dynamicImportStackDepth = 0; openTokenDepth = 0; + templateSpanImport = NULL; lastTokenPos = (char16_t*)EMPTY_CHAR; lastSlashWasDivision = false; parse_error = 0; @@ -956,6 +984,10 @@ void templateString () { if (ch == '`') { if (openTokenStack[--openTokenDepth].token != Template) syntaxError(); + // The specifier template just closed. Stop recording so a following + // concatenated template (`a${x}` + `b${y}`) never appends its spans. + if (templateSpanImport != NULL && openTokenDepth == specifierTemplateDepth) + templateSpanImport = NULL; return; } if (ch == '\\') diff --git a/src/lexer.h b/src/lexer.h index 20ac5fe..4e52ac0 100755 --- a/src/lexer.h +++ b/src/lexer.h @@ -36,6 +36,17 @@ struct Attribute { typedef struct Attribute Attribute; #endif +// A ${...} substitution inside a dynamic-import template specifier. `end` is +// the source position just after the closing "}". The decoders replace the +// whole substitution with a single "*" glob wildcard, so they only need where +// each one ends. Recorded here (in the parser, which already disambiguates +// regex from division) so the decoders never re-tokenize the body. +struct TemplateSpan { + const char16_t* end; + struct TemplateSpan* next; +}; +typedef struct TemplateSpan TemplateSpan; + struct Import { const char16_t* start; const char16_t* end; @@ -48,6 +59,10 @@ struct Import { #ifndef LEXER_MIN struct Attribute* attributes; #endif + // Top-level ${...} substitution end positions for a lone template-literal + // specifier, in source order. NULL for any non-template dynamic import. See + // struct TemplateSpan. + struct TemplateSpan* template_spans; struct Import* next; }; typedef struct Import Import; @@ -107,6 +122,13 @@ OpenToken* openTokenStack; uint16_t dynamicImportStackDepth; Import** dynamicImportStack; bool nextBraceIsClass; +// While a dynamic import's argument is a lone template literal, this points at +// that import and specifierTemplateDepth records the openTokenStack depth of +// the specifier's Template token, so only its own top-level ${...} spans are +// recorded (not those of a nested or concatenated template). NULL otherwise. +Import* templateSpanImport; +uint16_t specifierTemplateDepth; +TemplateSpan* template_span_write_head; // Memory Structure: // -> source @@ -164,6 +186,7 @@ void addImport (const char16_t* statement_start, const char16_t* start, const ch #ifndef LEXER_MIN import->attributes = NULL; #endif + import->template_spans = NULL; import->next = NULL; #ifndef LEXER_MIN if (dynamic == IMPORT_META || dynamic == STANDARD_IMPORT) @@ -234,6 +257,27 @@ uint32_t id () { uint32_t ip () { return import_read_head->safe; } + +TemplateSpan* template_span_read_head = NULL; + +// readTemplateSpan: advance to the next ${...} substitution of the current +// import's template specifier. False once the list is exhausted. +bool rt () { + if (template_span_read_head == NULL) + template_span_read_head = import_read_head->template_spans; + else + template_span_read_head = template_span_read_head->next; + return template_span_read_head != NULL; +} +// getTemplateSpanEnd: source offset just after the current substitution's "}". +uint32_t te () { + return template_span_read_head->end - source; +} +// resetTemplateSpans +void rts () { + template_span_read_head = NULL; +} + // getExportStart uint32_t es () { return export_read_head->start - source; diff --git a/src/lexer.ts b/src/lexer.ts index 66e7386..47f90ed 100755 --- a/src/lexer.ts +++ b/src/lexer.ts @@ -273,7 +273,7 @@ export function parse (source: string, name = '@'): readonly [ if (wasm.ip()) n = decode(source.slice(d === -1 ? s - 1 : s, d === -1 ? e + 1 : e)); else if (d !== -1 && source[s] === '`') - n = decodeTemplate(source.slice(s, e)); + n = decodeTemplate(s, e); let at: Array<[string, string]> | null = null; // minimal build drops the parsed attribute list; es-module-shims reads the // assertion via source.slice(a, se - 1) instead @@ -313,126 +313,41 @@ export function parse (source: string, name = '@'): readonly [ return str; } - // `str` spans the whole dynamic-import argument including the backticks. A - // lone template literal is reported as a glob with each ${...} substitution - // collapsed to a single "*"; the substituted source is then evaluated so the - // static parts are cooked. Anything else (e.g. `a` + b) closes before the end - // and returns undefined. - function decodeTemplate (str: string) { - let out = '`', chunkStart = 1, index = 1; - const last = str.length - 1; - interpolationError = false; + // `[s, e)` spans the dynamic-import template argument, backticks included. The + // parser records each top-level ${...} substitution's end position (see struct + // TemplateSpan in lexer.c), which already resolves the regex-vs-division + // ambiguity the decoder cannot. This walks the static parts, emits a single + // "*" per substitution, and jumps its body via the recorded end, then cooks + // the skeleton by evaluating it. A concatenation such as `a${x}` + b records + // only its first template's spans, so the skeleton misses the argument end + // (its closing backtick is not at e - 1) and it drops to undefined. + function decodeTemplate (s: number, e: number) { + let out = '`', chunkStart = s + 1, index = s + 1; + const last = e - 1; + wasm.rts(); while (index < last) { - const ch = str.charCodeAt(index); + const ch = source.charCodeAt(index); if (ch === 92/*\*/) { index += 2; continue; } if (ch === 96/*`*/) return; - if (ch === 36/*$*/ && str.charCodeAt(index + 1) === 123/*{*/) { - out += str.slice(chunkStart, index) + '*'; - index = skipInterpolation(str, index + 2); - chunkStart = index; + if (ch === 36/*$*/ && source.charCodeAt(index + 1) === 123/*{*/ && wasm.rt()) { + out += source.slice(chunkStart, index) + '*'; + index = chunkStart = wasm.te(); continue; } index++; } - if (interpolationError || str.charCodeAt(last) !== 96/*`*/) + if (source.charCodeAt(last) !== 96/*`*/) return; - return decode(out + str.slice(chunkStart, last) + '`'); + return decode(out + source.slice(chunkStart, last) + '`'); } return (MINIMAL ? [imports, exports] : [imports, exports, !!wasm.f(), !!wasm.ms()]) as ReturnType; } -// Set by skipInterpolation when a ${...} substitution contains a "/" that could -// open a regex literal, which the glob walker cannot disambiguate from -// division. A regex "}" would close the substitution early and yield a wrong -// glob, so decodeTemplate drops n to undefined instead. -let interpolationError: boolean; - -// `index` is the offset just after a "${" substitution opener within `str`. -// Returns the offset just after the matching "}", tracking brace depth and -// skipping strings, nested templates and comments so a "}" inside them does not -// close the substitution early. A "/" that opens a regex literal cannot be told -// apart from division here without the main parser's token context, and a regex -// may carry a "}" that would close the substitution early; the bare "/" case -// sets interpolationError so decodeTemplate drops the glob to undefined rather -// than emit a wrong skeleton. -function skipInterpolation (str: string, index: number): number { - let braceDepth = 1; - const len = str.length; - while (index < len) { - const ch = str.charCodeAt(index); - if (ch === 92/*\*/) { - index += 2; - continue; - } - if (ch === 123/*{*/) { - braceDepth++; - } - else if (ch === 125/*}*/) { - if (--braceDepth === 0) - return index + 1; - } - else if (ch === 39/*'*/ || ch === 34/*"*/ || ch === 96/*`*/) { - index = skipQuoted(str, index + 1, ch); - continue; - } - else if (ch === 47/*/*/) { - const next = str.charCodeAt(index + 1); - if (next === 47/*/*/ || next === 42/***/) { - index = skipComment(str, index + 2, next === 42/***/); - continue; - } - interpolationError = true; - } - index++; - } - return index; -} - -// `index` is the offset just after the opening quote `quote`. Returns the -// offset just after the closing quote. A backtick run recurses through its own -// ${...} substitutions. -function skipQuoted (str: string, index: number, quote: number): number { - const len = str.length; - while (index < len) { - const ch = str.charCodeAt(index); - if (ch === 92/*\*/) { - index += 2; - continue; - } - if (ch === quote) - return index + 1; - if (quote === 96/*`*/ && ch === 36/*$*/ && str.charCodeAt(index + 1) === 123/*{*/) { - index = skipInterpolation(str, index + 2); - continue; - } - index++; - } - return index; -} - -// `index` is the offset just after "//" or "/*". Returns the offset just after -// the comment. -function skipComment (str: string, index: number, block: boolean): number { - const len = str.length; - while (index < len) { - const ch = str.charCodeAt(index); - if (block) { - if (ch === 42/***/ && str.charCodeAt(index + 1) === 47/*/*/) - return index + 2; - } - else if (ch === 10/*\n*/ || ch === 13/*\r*/) { - return index; - } - index++; - } - return index; -} - function copyBE (src: string, outBuf16: Uint16Array) { const len = src.length; let i = 0; @@ -503,6 +418,12 @@ let wasm: { avs(): number; /** getAttributeValueEnd */ ave(): number; + /** readTemplateSpan */ + rt(): boolean; + /** getTemplateSpanEnd */ + te(): number; + /** resetTemplateSpans */ + rts(): void; }; const getWasmBytes = () => ( diff --git a/test/_unit.cjs b/test/_unit.cjs index fd14e00..6877485 100755 --- a/test/_unit.cjs +++ b/test/_unit.cjs @@ -436,14 +436,13 @@ suite('Lexer', () => { assert.strictEqual(parse("import(`a${ obj['}'] }b`)")[0][0].n, 'a*b'); assert.strictEqual(parse('import(`a${ x /* } */ }b`)')[0][0].n, 'a*b'); assert.strictEqual(parse('import(`a${ x // }\n }b`)')[0][0].n, 'a*b'); - // A "/" inside the substitution may open a regex literal, which the glob - // walker cannot tell apart from division without the parser's token - // context. A regex "}" would otherwise close the substitution early and - // produce a wrong skeleton, so n drops to undefined rather than guess. - assert.strictEqual(parse('import(`a${ /x}y/g }b`)')[0][0].n, undefined); - assert.strictEqual(parse('import(`a${ /x/g }b`)')[0][0].n, undefined); - assert.strictEqual(parse('import(`a${ b/c }d`)')[0][0].n, undefined); - assert.strictEqual(parse('import(`a${ `n${ /x}/g }` }b`)')[0][0].n, undefined); + // A "/" inside the substitution is disambiguated by the parser's own + // tokenizer, so a regex literal carrying a "}" no longer closes the + // substitution early: the whole ${...} still collapses to a single "*". + assert.strictEqual(parse('import(`a${ /x}y/g }b`)')[0][0].n, 'a*b'); + assert.strictEqual(parse('import(`a${ /x/g }b`)')[0][0].n, 'a*b'); + assert.strictEqual(parse('import(`a${ b/c }d`)')[0][0].n, 'a*d'); + assert.strictEqual(parse('import(`a${ `n${ /x}/g }` }b`)')[0][0].n, 'a*b'); // A "/" inside a string or comment is an ordinary character and keeps the // glob. assert.strictEqual(parse("import(`a${ x['/}'] }b`)")[0][0].n, 'a*b'); From 613602843a52593caabbf31a1e5800c5a56eb7e6 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Mon, 6 Jul 2026 21:12:31 +0200 Subject: [PATCH 4/5] fix: gate template glob to the full build and fix its cross-build defects Review of the interpolated-template glob surfaced that it shipped in the minimal build and that its decoder-side reconstruction diverged per port. Both are fixed by keeping the whole feature full-build-only and letting the parser be the single source of the substitution spans the decoders splice. 1. Min-build gating: the TemplateSpan struct, the Import fields, the rt/te/rts readers, and both recording sites are now behind LEXER_MIN, and _rt/_te/_rts are dropped from the three minimal export lists. es-module-shims reads specifiers via source.slice and never globs, so lexer.min.wasm is unchanged. 2. No eval on the glob path: the Wasm/asm decoder cooked its skeleton with (0, eval), so import(` ${sideEffect()}.js`) (a space defeats the C gate but not the decoder) executed the substitution during parse(). All three ports now copy the static parts raw, so nothing is evaluated and the builds agree byte-for-byte (this also removes the CR/CRLF normalization divergence). 3. rt() stays exhausted: it reset its read head to NULL on the last span and the next call reloaded the first span, so a decoder meeting more top-level ${ than recorded spans jumped backward and looped. It now latches exhausted. 4. Nested imports no longer corrupt the outer glob: recording scratch is per-import (keyed off dynamicImportStack) in the parser, and lexer.js gained the dynamic-import stack it lacked, which also restores the outer import's end/se that a nested dynamic import previously dropped. 5. Consistent lone-template detection: the C backtick gate keys on the recorded specifier start and finalization keeps the spans only when the specifier's closing backtick is the last token, so whitespace before/after the argument (e.g. Prettier's multiline form) globs identically across builds instead of three different answers. Static parts stay raw source: escapes are not cooked and a literal "*" is emitted verbatim, both documented. A nested dynamic import whose own specifier is an interpolated template still yields undefined in the pure-JS port (the Wasm/asm builds report the glob); documented as a known limitation. Fixes: https://github.com/guybedford/es-module-lexer/issues/137 --- README.md | 4 +- chompfile.toml | 6 +-- lexer.js | 120 ++++++++++++++++++++++++++++------------------- src/lexer.asm.js | 56 +++++++++++----------- src/lexer.c | 91 +++++++++++++++++++++++------------ src/lexer.h | 61 +++++++++++++++--------- src/lexer.ts | 44 ++++++++--------- test/_unit.cjs | 44 +++++++++++++++++ 8 files changed, 271 insertions(+), 155 deletions(-) diff --git a/README.md b/README.md index 70ab453..53d39b4 100755 --- a/README.md +++ b/README.md @@ -240,7 +240,9 @@ To handle escape sequences in specifier strings, the `.n` field of imported spec For dynamic import expressions, this field will be empty if not a valid JS string. -When the entire dynamic import argument is a single template literal, `.n` is reported as a glob: each `${...}` substitution is collapsed to a single `*` (for example `` import(`./locales/${locale}.js`) `` yields `./locales/*.js`). A template concatenated with anything else, or any other expression, still resolves to `undefined`. A substitution containing a `/` that could open a regex literal cannot be disambiguated from division without full token context, so the glob is dropped to `undefined` rather than risk a wrong specifier. +When the entire dynamic import argument is a single template literal, `.n` is reported as a glob: each `${...}` substitution is collapsed to a single `*` (for example `` import(`./locales/${locale}.js`) `` yields `./locales/*.js`). This is a full-build-only feature; the minimal build reports `undefined`. A template concatenated with anything else, or any other expression, resolves to `undefined`. Substitutions are matched by the parser itself, so a `/` inside one is correctly disambiguated as regex or division and does not affect the glob. + +The static parts are the raw specifier source: escape sequences are not cooked, and a literal `*` in the specifier is emitted as-is, so a consumer treating `.n` as a glob has to apply its own escaping. As a known limitation, a nested dynamic import whose own specifier is an interpolated template, inside an outer template substitution, yields `undefined` in the `es-module-lexer/js` build where the Wasm and asm.js builds report the glob. ### Facade Detection diff --git a/chompfile.toml b/chompfile.toml index f92b527..fd9c443 100644 --- a/chompfile.toml +++ b/chompfile.toml @@ -154,7 +154,7 @@ run = """ ${{ EMSDK_PATH }}/emsdk activate 6.0.0 ${{ EMSDK_PATH }}/upstream/emscripten/emcc src/lexer.c -DLEXER_MIN -o lib/lexer.min.wasm -Oz -flto --no-entry -s STANDALONE_WASM=1 \ - -s EXPORTED_FUNCTIONS="['_parse','_sa','_e','_ri','_re','_it','_is','_ie','_ss','_ip','_se','_ai','_id','_es','_ee','_els','_ele','_rt','_te','_rts','___heap_base']" \ + -s EXPORTED_FUNCTIONS="['_parse','_sa','_e','_ri','_re','_it','_is','_ie','_ss','_ip','_se','_ai','_id','_es','_ee','_els','_ele','___heap_base']" \ -s SUPPORT_LONGJMP=0 -s ALLOW_MEMORY_GROWTH=1 -s ERROR_ON_UNDEFINED_SYMBOLS=0 -Wno-logical-op-parentheses -Wno-parentheses """ @@ -204,7 +204,7 @@ run = """ ${{ EMSDK_FASTCOMP_PATH }}/emsdk activate 1.40.1-fastcomp ${{ EMSDK_FASTCOMP_PATH }}/fastcomp/emscripten/emcc ./src/lexer.c -DLEXER_MIN -o lib/lexer.min.emcc.js -s WASM=0 -Oz --closure 1 \ - -s EXPORTED_FUNCTIONS="['_parse','_sa','_e','_ri','_re','_it','_is','_ie','_ss','_ip','_se','_ai','_id','_es','_ee','_els','_ele','_rt','_te','_rts','_setSource']" \ + -s EXPORTED_FUNCTIONS="['_parse','_sa','_e','_ri','_re','_it','_is','_ie','_ss','_ip','_se','_ai','_id','_es','_ee','_els','_ele','_setSource']" \ -s AGGRESSIVE_VARIABLE_ELIMINATION=1 -s GLOBAL_BASE=8 \ -s ERROR_ON_UNDEFINED_SYMBOLS=0 -s TOTAL_STACK=4997968 -s --separate-asm -Wno-logical-op-parentheses -Wno-parentheses """ @@ -221,7 +221,7 @@ run = """ ${{ EMSDK_FASTCOMP_PATH }}/emsdk activate 1.40.1-fastcomp ${{ EMSDK_FASTCOMP_PATH }}/fastcomp/emscripten/emcc ./src/lexer.c -DLEXER_MIN -o lib/lexer.min.layout.js -s WASM=0 -Oz --closure 0 \ - -s EXPORTED_FUNCTIONS="['_parse','_sa','_e','_ri','_re','_it','_is','_ie','_ss','_ip','_se','_ai','_id','_es','_ee','_els','_ele','_rt','_te','_rts','_setSource']" \ + -s EXPORTED_FUNCTIONS="['_parse','_sa','_e','_ri','_re','_it','_is','_ie','_ss','_ip','_se','_ai','_id','_es','_ee','_els','_ele','_setSource']" \ -s AGGRESSIVE_VARIABLE_ELIMINATION=1 -s GLOBAL_BASE=8 \ -s ERROR_ON_UNDEFINED_SYMBOLS=0 -s TOTAL_STACK=4997968 -Wno-logical-op-parentheses -Wno-parentheses """ diff --git a/lexer.js b/lexer.js index 169eea5..460c8fa 100644 --- a/lexer.js +++ b/lexer.js @@ -4,6 +4,7 @@ let source, pos, end, openTokenPosStack, openClassPosStack, curDynamicImport, + dynamicImportStack, templateStackDepth, facade, lastSlashWasDivision, @@ -13,17 +14,19 @@ let source, pos, end, imports, exports, exportStatementStart, - // While a dynamic import's argument is a lone template literal, this holds - // that import and specifierTemplateDepth the openTokenDepth of its Template - // level, so only its own top-level ${...} spans are recorded. undefined - // otherwise. This reuses the real tokenizer's regex/division disambiguation - // (the main loop below), which a standalone ${...} skimmer cannot do. - templateSpanImport, - specifierTemplateDepth, + // Maps a dynamic import to its glob-recording scratch, kept off the public + // import objects so the JS result shape stays identical to the wasm build's. + // The scratch is { ends, close, depth }: `ends` collects each top-level ${...} + // end, `close` the specifier's closing backtick, `depth` the openTokenDepth + // its top-level substitutions close at. Per-import, so a nested import(`...`) + // records against its own entry and never corrupts an enclosing one. Reuses + // the real tokenizer's regex/division disambiguation, which a standalone + // ${...} skimmer cannot do. + templateGlobs, name; function addImport (ss, s, e, d) { - const impt = { ss, se: d === -2 ? e : d === -1 ? e + 1 : 0, s, e, d, a: -1, n: undefined, at: null, spans: undefined }; + const impt = { ss, se: d === -2 ? e : d === -1 ? e + 1 : 0, s, e, d, a: -1, n: undefined, at: null }; imports.push(impt); return impt; } @@ -55,41 +58,52 @@ function readName (impt) { // only splices a "*" per substitution and cooks the static parts. A // concatenation such as import(`a` + b) records no specifier spans and its // walk hits an inner backtick before the argument end, so n stays undefined. -function readDynamicTemplateName (impt) { - const spans = impt.spans; - if (spans === undefined || source.charCodeAt(impt.s) !== 96/*`*/) +// At dynamic-import finalization: build the glob only when the specifier +// template's closing backtick was the last token of the argument, so a recorded +// span list means "lone template glob". A concatenation (`a${x}` + b) or a +// trailing operator recorded spans for its first template but does not end on +// that backtick, so it yields undefined. Each span is the end of one top-level +// ${...}; walking from the opening backtick, each becomes a single "*" and is +// skipped, static runs are copied raw so the result matches the wasm/asm builds +// byte-for-byte, and the walk ends at the specifier's unescaped closing +// backtick. Mirrors dropUncommittedGlob + decodeTemplate in src/lexer.c / .ts. +function finalizeDynamicTemplate (impt) { + // Anchor on the last token before ")"/"," (lastTokenPos), not impt.e, so + // whitespace or a comment before the paren does not drop the glob and the + // result matches the wasm/asm builds (which anchor the same way). + const glob = templateGlobs.get(impt); + if (glob === undefined || glob.close !== lastTokenPos) return; - const last = impt.e - 1; - if (source.charCodeAt(last) !== 96/*`*/) - return; - acornPos = impt.s + 1; - let out = '', chunkStart = acornPos, spanIndex = 0; - while (acornPos < last) { - const ch = source.charCodeAt(acornPos); + const spans = glob.ends; + const e = impt.e; + let out = '', chunkStart = impt.s + 1, index = impt.s + 1, spanIndex = 0, spanEnd = spans[0]; + // `e` bounds the walk defensively; the parser guarantees an unescaped closing + // backtick within it for a committed glob. + while (index < e) { + const ch = source.charCodeAt(index); + if (ch === 96/*`*/) + break; if (ch === 92/*\*/) { - out += source.slice(chunkStart, acornPos); - out += readEscapedChar(); - chunkStart = acornPos; - } - else if (ch === 96/*`*/) { - return; - } - else if (ch === 36/*$*/ && source.charCodeAt(acornPos + 1) === 123/*{*/ && spanIndex < spans.length) { - out += source.slice(chunkStart, acornPos) + '*'; - acornPos = chunkStart = spans[spanIndex++]; + index += 2; + continue; } - else { - ++acornPos; + if (ch === 36/*$*/ && source.charCodeAt(index + 1) === 123/*{*/ && index + 2 <= spanEnd) { + out += source.slice(chunkStart, index) + '*'; + index = chunkStart = spanEnd; + spanEnd = ++spanIndex < spans.length ? spans[spanIndex] : -1; + continue; } + index++; } - impt.n = out + source.slice(chunkStart, last); + impt.n = out + source.slice(chunkStart, index); } // Note: parsing is based on the _assumption_ that the source is already valid export function parse (_source, _name) { openTokenDepth = 0; curDynamicImport = null; - templateSpanImport = undefined; + dynamicImportStack = []; + templateGlobs = new WeakMap(); templateDepth = -1; lastTokenPos = -1; lastSlashWasDivision = false; @@ -185,7 +199,7 @@ export function parse (_source, _name) { if (curDynamicImport && curDynamicImport.d === openTokenPosStack[openTokenDepth]) { if (curDynamicImport.e === 0) { curDynamicImport.e = pos; - readDynamicTemplateName(curDynamicImport); + finalizeDynamicTemplate(curDynamicImport); } curDynamicImport.se = pos; curDynamicImport = null; @@ -202,7 +216,7 @@ export function parse (_source, _name) { case 44/*,*/: if (curDynamicImport && curDynamicImport.e === 0 && curDynamicImport.d === openTokenPosStack[openTokenDepth - 1]) { curDynamicImport.e = lastTokenPos + 1; - readDynamicTemplateName(curDynamicImport); + finalizeDynamicTemplate(curDynamicImport); pos++; commentWhitespace(true); curDynamicImport.a = pos; @@ -226,10 +240,11 @@ export function parse (_source, _name) { syntaxError(); if (openTokenDepth-- === templateDepth) { templateDepth = templateStack[--templateStackDepth]; - // A top-level ${...} of the specifier template just closed: record the - // position after "}" for readDynamicTemplateName to splice a "*". - if (templateSpanImport !== undefined && openTokenDepth === specifierTemplateDepth) - templateSpanImport.spans.push(pos + 1); + // A top-level ${...} of the recording import's specifier template just + // closed: note the position after "}" for a "*" splice. + const glob = curDynamicImport && templateGlobs.get(curDynamicImport); + if (glob && openTokenDepth + 1 === glob.depth) + glob.ends.push(pos + 1); templateString(); } else { @@ -285,13 +300,10 @@ export function parse (_source, _name) { } case 96/*`*/: // A backtick that is the first token of an open dynamic import's - // argument opens the specifier template: record its top-level ${...} - // spans so readDynamicTemplateName can glob it without re-tokenizing. - if (curDynamicImport && curDynamicImport.e === 0 && pos === curDynamicImport.s) { - templateSpanImport = curDynamicImport; - templateSpanImport.spans = []; - specifierTemplateDepth = openTokenDepth; - } + // argument opens the specifier template: start recording its top-level + // ${...} spans so the glob can be built without re-tokenizing. + if (curDynamicImport && curDynamicImport.e === 0 && pos === curDynamicImport.s) + templateGlobs.set(curDynamicImport, { ends: [], close: -1, depth: openTokenDepth + 1 }); templateString(); break; } @@ -324,6 +336,11 @@ function tryParseImportStatement () { // The specifier start is recorded after leading whitespace/comments so it // points at the literal, matching the C lexer (src/lexer.c). const impt = addImport(startPos, pos, 0, startPos); + // Stack the enclosing dynamic import so a nested import(...) inside an + // argument restores it on close; without this the outer import's end/se + // (and template-glob recording) are lost. Popped at both finalization + // sites: the constant path just below and the main-loop ")". + dynamicImportStack.push(curDynamicImport); curDynamicImport = impt; if (ch === 39/*'*/ || ch === 34/*"*/) { stringLiteral(ch); @@ -353,6 +370,7 @@ function tryParseImportStatement () { impt.e = pos; impt.se = pos; readName(impt); + curDynamicImport = dynamicImportStack.pop(); } else { pos--; @@ -800,10 +818,14 @@ function templateString () { return; } if (ch === 96/*`*/) { - // The specifier template just closed. Stop recording so a following - // concatenated template (`a${x}` + `b${y}`) never appends its spans. - if (templateSpanImport !== undefined && openTokenDepth === specifierTemplateDepth) - templateSpanImport = undefined; + // The specifier template just closed. Note its closing backtick and stop + // recording; finalizeDynamicTemplate keeps the spans only if this backtick + // is the last token of the argument. + const glob = curDynamicImport && templateGlobs.get(curDynamicImport); + if (glob && openTokenDepth + 1 === glob.depth) { + glob.close = pos; + glob.depth = 0; + } return; } if (ch === 92/*\*/) diff --git a/src/lexer.asm.js b/src/lexer.asm.js index 2dc50f0..9461215 100644 --- a/src/lexer.asm.js +++ b/src/lexer.asm.js @@ -57,7 +57,7 @@ export function parse (_source, _name = '@') { let n; if (asm.ip()) n = readString(d === -1 ? s : s + 1, source.charCodeAt(d === -1 ? s - 1 : s)); - else if (d !== -1 && source.charCodeAt(s) === 96/*`*/) + else if (!MINIMAL && d !== -1 && source.charCodeAt(s) === 96/*`*/) n = decodeTemplate(s, e); let at = null; // minimal build drops the parsed attribute list; es-module-shims reads the @@ -143,40 +143,38 @@ function readString (start, quote) { return out; } -// `[s, e)` spans the dynamic-import template argument, backticks included. The -// parser records each top-level ${...} substitution's end position (see struct -// TemplateSpan in lexer.c), which already resolves the regex-vs-division -// ambiguity the decoder cannot. This cooks the static parts (escapes and all, -// via readString's machinery), emits a single "*" per substitution and jumps -// its body via the recorded end. A concatenation such as `a${x}` + b records -// only its first template's spans, so the walk never reaches the argument end -// (its closing backtick is not at e - 1) and n stays undefined. +// Glob for a lone interpolated-template specifier starting at `s`. The parser +// commits a ${...} span list only for that shape (see lexer.c), so a first rt() +// of false means "not a glob" (a concatenation such as `a${x}` + b, or a nested +// template) and yields undefined. Walking from the opening backtick, each +// top-level ${...} becomes a single "*" and is skipped via its recorded end; +// static runs are copied raw so the three ports agree byte-for-byte. The walk +// ends at the specifier's unescaped closing backtick. Kept identical to the +// wasm build's decodeTemplate (src/lexer.ts). function decodeTemplate (s, e) { - const last = e - 1; - if (source.charCodeAt(last) !== 96/*`*/) - return; asm.rts(); - acornPos = s + 1; - let out = '', chunkStart = acornPos; - while (acornPos < last) { - const ch = source.charCodeAt(acornPos); + if (!asm.rt()) + return; + let out = '', chunkStart = s + 1, index = s + 1, spanEnd = asm.te(); + // `e` bounds the walk defensively; the parser guarantees an unescaped closing + // backtick within it for a committed glob. + while (index < e) { + const ch = source.charCodeAt(index); + if (ch === 96/*`*/) + break; if (ch === 92/*\*/) { - out += source.slice(chunkStart, acornPos); - out += readEscapedChar(); - chunkStart = acornPos; - } - else if (ch === 96/*`*/) { - return; - } - else if (ch === 36/*$*/ && source.charCodeAt(acornPos + 1) === 123/*{*/ && asm.rt()) { - out += source.slice(chunkStart, acornPos) + '*'; - acornPos = chunkStart = asm.te(); + index += 2; + continue; } - else { - ++acornPos; + if (ch === 36/*$*/ && source.charCodeAt(index + 1) === 123/*{*/ && index + 2 <= spanEnd) { + out += source.slice(chunkStart, index) + '*'; + index = chunkStart = spanEnd; + spanEnd = asm.rt() ? asm.te() : -1; + continue; } + index++; } - return out + source.slice(chunkStart, last); + return out + source.slice(chunkStart, index); } // Used to read escaped characters diff --git a/src/lexer.c b/src/lexer.c index 1adc738..a86bb29 100755 --- a/src/lexer.c +++ b/src/lexer.c @@ -70,6 +70,19 @@ static inline __attribute__((always_inline)) bool handleSlash () { return false; } +// At dynamic-import finalization: keep the recorded ${...} spans only when the +// specifier template's closing backtick was the last token of the argument, so +// its non-empty list means "lone template glob". A concatenation (`a${x}` + b) +// or a trailing operator recorded spans for its first template but does not end +// on that backtick, so its list is dropped and the decoders report undefined. +// No-op in the minimal build, which never records or reads a glob. +static inline __attribute__((always_inline)) void dropUncommittedGlob (Import* import) { +#ifndef LEXER_MIN + if (import->template_close != import->end - 1) + import->template_spans = NULL; +#endif +} + // Consume one token at the current ch/pos, updating the global tokenizer state. // The single source of tokenization: the main loop and skipExpression both call // it, so the regex/keyword/import rules never diverge. Comments do not advance @@ -108,6 +121,7 @@ static inline __attribute__((always_inline)) bool consumeToken (char16_t ch) { Import* cur_dynamic_import = dynamicImportStack[dynamicImportStackDepth - 1]; if (cur_dynamic_import->end == 0) { cur_dynamic_import->end = lastTokenPos + 1; + dropUncommittedGlob(cur_dynamic_import); pos++; ch = commentWhitespace(true); cur_dynamic_import->attr_index = pos; @@ -120,8 +134,10 @@ static inline __attribute__((always_inline)) bool consumeToken (char16_t ch) { openTokenDepth--; if (dynamicImportStackDepth > 0 && openTokenStack[openTokenDepth].token == ImportParen) { Import* cur_dynamic_import = dynamicImportStack[dynamicImportStackDepth - 1]; - if (cur_dynamic_import->end == 0) + if (cur_dynamic_import->end == 0) { cur_dynamic_import->end = lastTokenPos + 1; + dropUncommittedGlob(cur_dynamic_import); + } cur_dynamic_import->statement_end = pos + 1; dynamicImportStackDepth--; } @@ -146,21 +162,26 @@ static inline __attribute__((always_inline)) bool consumeToken (char16_t ch) { case '}': if (openTokenDepth == 0) return syntaxError(), false; if (openTokenStack[--openTokenDepth].token == TemplateBrace) { - // A top-level ${...} of the specifier template just closed: record the - // position after "}" so the decoders splice a "*" for it. Only the - // specifier's own substitutions qualify (openTokenDepth back at the - // Template's depth + 1), never a nested or concatenated template's. - if (templateSpanImport != NULL && openTokenDepth == specifierTemplateDepth + 1) { - TemplateSpan* span = (TemplateSpan*)(analysis_head); - analysis_head = analysis_head + sizeof(TemplateSpan); - span->end = pos + 1; - span->next = NULL; - if (template_span_write_head == NULL) - templateSpanImport->template_spans = span; - else - template_span_write_head->next = span; - template_span_write_head = span; +#ifndef LEXER_MIN + // A top-level ${...} of an open dynamic import's specifier template just + // closed: record the position after "}" so the decoders splice a "*". + // The top import is the innermost, so a nested import's substitutions + // record against the nested entry, never this one. + if (dynamicImportStackDepth > 0) { + Import* cur_dynamic_import = dynamicImportStack[dynamicImportStackDepth - 1]; + if (cur_dynamic_import->specifier_template_depth == openTokenDepth) { + TemplateSpan* span = (TemplateSpan*)(analysis_head); + analysis_head = analysis_head + sizeof(TemplateSpan); + span->end = pos + 1; + span->next = NULL; + if (cur_dynamic_import->template_span_tail == NULL) + cur_dynamic_import->template_spans = span; + else + cur_dynamic_import->template_span_tail->next = span; + cur_dynamic_import->template_span_tail = span; + } } +#endif templateString(); } break; @@ -172,17 +193,18 @@ static inline __attribute__((always_inline)) bool consumeToken (char16_t ch) { isComment = handleSlash(); break; case '`': - // A backtick that is the first token of an open dynamic import's argument - // opens the specifier template: record its top-level ${...} spans so the - // decoders can glob it without re-tokenizing (see struct TemplateSpan). - if (dynamicImportStackDepth > 0 && openTokenDepth > 0 && - openTokenStack[openTokenDepth - 1].token == ImportParen && - lastTokenPos == openTokenStack[openTokenDepth - 1].pos && - dynamicImportStack[dynamicImportStackDepth - 1]->end == 0) { - templateSpanImport = dynamicImportStack[dynamicImportStackDepth - 1]; - specifierTemplateDepth = openTokenDepth; - template_span_write_head = NULL; - } +#ifndef LEXER_MIN + // A backtick that opens an active dynamic import's argument (its recorded + // specifier start, so leading whitespace/comments don't matter) opens the + // specifier template: mark this import so its top-level ${...} spans are + // recorded for globbing (see struct TemplateSpan). Per-import, so a nested + // import(`...`) in a substitution records against its own entry and never + // corrupts an enclosing import. + if (dynamicImportStackDepth > 0 && + dynamicImportStack[dynamicImportStackDepth - 1]->end == 0 && + dynamicImportStack[dynamicImportStackDepth - 1]->start == pos) + dynamicImportStack[dynamicImportStackDepth - 1]->specifier_template_depth = openTokenDepth + 1; +#endif openTokenStack[openTokenDepth].pos = lastTokenPos; openTokenStack[openTokenDepth++].token = Template; templateString(); @@ -207,7 +229,6 @@ bool parse () { #endif dynamicImportStackDepth = 0; openTokenDepth = 0; - templateSpanImport = NULL; lastTokenPos = (char16_t*)EMPTY_CHAR; lastSlashWasDivision = false; parse_error = 0; @@ -984,10 +1005,18 @@ void templateString () { if (ch == '`') { if (openTokenStack[--openTokenDepth].token != Template) syntaxError(); - // The specifier template just closed. Stop recording so a following - // concatenated template (`a${x}` + `b${y}`) never appends its spans. - if (templateSpanImport != NULL && openTokenDepth == specifierTemplateDepth) - templateSpanImport = NULL; +#ifndef LEXER_MIN + // The specifier template just closed. Note its closing backtick and stop + // recording (depth back to 0); finalization keeps the spans only if this + // backtick is the last token of the argument. + if (dynamicImportStackDepth > 0) { + Import* cur_dynamic_import = dynamicImportStack[dynamicImportStackDepth - 1]; + if (cur_dynamic_import->specifier_template_depth == openTokenDepth + 1) { + cur_dynamic_import->template_close = pos; + cur_dynamic_import->specifier_template_depth = 0; + } + } +#endif return; } if (ch == '\\') diff --git a/src/lexer.h b/src/lexer.h index 4e52ac0..2dc325a 100755 --- a/src/lexer.h +++ b/src/lexer.h @@ -36,16 +36,19 @@ struct Attribute { typedef struct Attribute Attribute; #endif -// A ${...} substitution inside a dynamic-import template specifier. `end` is -// the source position just after the closing "}". The decoders replace the +#ifndef LEXER_MIN +// A ${...} substitution inside a lone dynamic-import template specifier. `end` +// is the source position just after the closing "}". The decoders replace the // whole substitution with a single "*" glob wildcard, so they only need where // each one ends. Recorded here (in the parser, which already disambiguates -// regex from division) so the decoders never re-tokenize the body. +// regex from division) so the decoders never re-tokenize the body. Glob `n` is +// a full-build-only feature; es-module-shims reads specifiers via source.slice. struct TemplateSpan { const char16_t* end; struct TemplateSpan* next; }; typedef struct TemplateSpan TemplateSpan; +#endif struct Import { const char16_t* start; @@ -58,11 +61,20 @@ struct Import { enum ImportType import_ty; #ifndef LEXER_MIN struct Attribute* attributes; -#endif - // Top-level ${...} substitution end positions for a lone template-literal - // specifier, in source order. NULL for any non-template dynamic import. See - // struct TemplateSpan. + // Top-level ${...} substitution end positions, in source order, iff the + // argument is a lone interpolated template literal (so a non-empty list is + // itself the "this is a glob" signal). NULL otherwise. `template_span_tail` + // and `specifier_template_depth` are recording scratch, valid only while the + // specifier template is open; kept per-import so a nested import(`...`) in a + // substitution cannot corrupt an enclosing import's list. struct TemplateSpan* template_spans; + struct TemplateSpan* template_span_tail; + // Position of the specifier template's closing backtick, once seen. The + // argument is a lone template glob iff this is the last token before ")" or + // ",", so a concatenation (`a${x}` + b) is rejected at finalization. + const char16_t* template_close; + uint16_t specifier_template_depth; +#endif struct Import* next; }; typedef struct Import Import; @@ -122,13 +134,6 @@ OpenToken* openTokenStack; uint16_t dynamicImportStackDepth; Import** dynamicImportStack; bool nextBraceIsClass; -// While a dynamic import's argument is a lone template literal, this points at -// that import and specifierTemplateDepth records the openTokenStack depth of -// the specifier's Template token, so only its own top-level ${...} spans are -// recorded (not those of a nested or concatenated template). NULL otherwise. -Import* templateSpanImport; -uint16_t specifierTemplateDepth; -TemplateSpan* template_span_write_head; // Memory Structure: // -> source @@ -185,8 +190,11 @@ void addImport (const char16_t* statement_start, const char16_t* start, const ch import->safe = dynamic == STANDARD_IMPORT; #ifndef LEXER_MIN import->attributes = NULL; -#endif import->template_spans = NULL; + import->template_span_tail = NULL; + import->template_close = NULL; + import->specifier_template_depth = 0; +#endif import->next = NULL; #ifndef LEXER_MIN if (dynamic == IMPORT_META || dynamic == STANDARD_IMPORT) @@ -258,15 +266,24 @@ uint32_t ip () { return import_read_head->safe; } -TemplateSpan* template_span_read_head = NULL; - -// readTemplateSpan: advance to the next ${...} substitution of the current -// import's template specifier. False once the list is exhausted. +#ifndef LEXER_MIN +// The last span node rt() returned. template_span_started distinguishes "not +// started" from "exhausted": once exhausted rt() stays false rather than +// reloading the first span, which would make the decoder jump backward and +// loop. rts() re-arms both for the next import. +TemplateSpan* template_span_read_head; +bool template_span_started; + +// readTemplateSpan: advance to the next ${...} substitution end of the current +// import's lone template specifier. False once exhausted, and stays false. bool rt () { - if (template_span_read_head == NULL) + if (!template_span_started) { + template_span_started = true; template_span_read_head = import_read_head->template_spans; - else + } + else if (template_span_read_head != NULL) { template_span_read_head = template_span_read_head->next; + } return template_span_read_head != NULL; } // getTemplateSpanEnd: source offset just after the current substitution's "}". @@ -276,7 +293,9 @@ uint32_t te () { // resetTemplateSpans void rts () { template_span_read_head = NULL; + template_span_started = false; } +#endif // getExportStart uint32_t es () { diff --git a/src/lexer.ts b/src/lexer.ts index 47f90ed..84b7017 100755 --- a/src/lexer.ts +++ b/src/lexer.ts @@ -52,8 +52,9 @@ export interface ImportSpecifier { * * A dynamic import whose entire argument is a single template literal is * reported as a glob: each `${...}` substitution is collapsed to a single - * `*`. Other expressions (including a template concatenated with anything - * else) remain undefined. + * `*` (full build only; the minimal build reports undefined). Other + * expressions (including a template concatenated with anything else) remain + * undefined. * * @example * const [imports1, exports1] = parse(String.raw`import './\u0061\u0062.js'`); @@ -272,7 +273,7 @@ export function parse (source: string, name = '@'): readonly [ let n; if (wasm.ip()) n = decode(source.slice(d === -1 ? s - 1 : s, d === -1 ? e + 1 : e)); - else if (d !== -1 && source[s] === '`') + else if (!MINIMAL && d !== -1 && source[s] === '`') n = decodeTemplate(s, e); let at: Array<[string, string]> | null = null; // minimal build drops the parsed attribute list; es-module-shims reads the @@ -313,36 +314,37 @@ export function parse (source: string, name = '@'): readonly [ return str; } - // `[s, e)` spans the dynamic-import template argument, backticks included. The - // parser records each top-level ${...} substitution's end position (see struct - // TemplateSpan in lexer.c), which already resolves the regex-vs-division - // ambiguity the decoder cannot. This walks the static parts, emits a single - // "*" per substitution, and jumps its body via the recorded end, then cooks - // the skeleton by evaluating it. A concatenation such as `a${x}` + b records - // only its first template's spans, so the skeleton misses the argument end - // (its closing backtick is not at e - 1) and it drops to undefined. + // Glob for a lone interpolated-template specifier starting at `s`. The parser + // commits a ${...} span list only for that shape (see lexer.c), so a first + // rt() of false means "not a glob" (a concatenation such as `a${x}` + b, or a + // nested template) and yields undefined. Walking from the opening backtick, + // each top-level ${...} becomes a single "*" and is skipped via its recorded + // end; static runs are copied raw so the three ports agree byte-for-byte. The + // walk ends at the specifier's unescaped closing backtick. function decodeTemplate (s: number, e: number) { - let out = '`', chunkStart = s + 1, index = s + 1; - const last = e - 1; wasm.rts(); - while (index < last) { + if (!wasm.rt()) + return; + let out = '', chunkStart = s + 1, index = s + 1, spanEnd = wasm.te(); + // `e` bounds the walk defensively; the parser guarantees an unescaped + // closing backtick within it for a committed glob. + while (index < e) { const ch = source.charCodeAt(index); + if (ch === 96/*`*/) + break; if (ch === 92/*\*/) { index += 2; continue; } - if (ch === 96/*`*/) - return; - if (ch === 36/*$*/ && source.charCodeAt(index + 1) === 123/*{*/ && wasm.rt()) { + if (ch === 36/*$*/ && source.charCodeAt(index + 1) === 123/*{*/ && index + 2 <= spanEnd) { out += source.slice(chunkStart, index) + '*'; - index = chunkStart = wasm.te(); + index = chunkStart = spanEnd; + spanEnd = wasm.rt() ? wasm.te() : -1; continue; } index++; } - if (source.charCodeAt(last) !== 96/*`*/) - return; - return decode(out + source.slice(chunkStart, last) + '`'); + return out + source.slice(chunkStart, index); } return (MINIMAL ? [imports, exports] : [imports, exports, !!wasm.f(), !!wasm.ms()]) as ReturnType; diff --git a/test/_unit.cjs b/test/_unit.cjs index 6877485..d7723c7 100755 --- a/test/_unit.cjs +++ b/test/_unit.cjs @@ -423,6 +423,11 @@ suite('Lexer', () => { }); test(`Dynamic import interpolated template specifier glob`, () => { + // Glob n is a full-build-only feature; the minimal build reports undefined. + if (min) { + assert.strictEqual(parse('import(`./a${x}.js`)')[0][0].n, undefined); + return; + } // An interpolated template literal has no constant value, but its static // skeleton is a useful glob: each ${...} collapses to a single "*". assert.strictEqual(parse('import(`./a${x}.js`)')[0][0].n, './a*.js'); @@ -456,6 +461,45 @@ suite('Lexer', () => { assert.strictEqual(parse('import(x)')[0][0].n, undefined); }); + test(`Dynamic import interpolated template glob edge cases`, () => { + // The glob is a full-build-only feature; the minimal build never reports it. + if (min) { + assert.strictEqual(parse('import(`./a/${x}.js`)')[0][0].n, undefined); + return; + } + // A substitution is never evaluated: expressions with side effects collapse + // to "*" like any other, so parse() cannot execute source (a regression for + // the Wasm build cooking its skeleton with eval). + globalThis.__templateGlobEvalRan = false; + assert.strictEqual(parse('import( `${(globalThis.__templateGlobEvalRan = true, "x")}.js`)')[0][0].n, '*.js'); + assert.strictEqual(globalThis.__templateGlobEvalRan, false); + delete globalThis.__templateGlobEvalRan; + // Whitespace around the argument does not change the glob, and matches the + // no-whitespace form (a cross-build divergence regression). + assert.strictEqual(parse('import(`./a/${x}.js` )')[0][0].n, './a/*.js'); + assert.strictEqual(parse('import( `./a/${x}.js`)')[0][0].n, './a/*.js'); + assert.strictEqual(parse('import(\t`./a/${x}.js`\n)')[0][0].n, './a/*.js'); + // Static parts are raw source: CR/CRLF is not normalized and a literal "*" + // is emitted verbatim next to substitution wildcards. + assert.strictEqual(parse('import(`a\r\nb${x}`)')[0][0].n, 'a\r\nb*'); + assert.strictEqual(parse('import(`a\\*${x}.js`)')[0][0].n, 'a\\**.js'); + // More top-level ${...} than the parser could ever record must not loop or + // reuse a stale span (a regression for the rt() reload / decoder hang). + assert.strictEqual(parse('import(`${a}${b}${c}${d}` + x)')[0][0].n, undefined); + // A nested dynamic import inside a substitution keeps the outer glob and is + // itself detected. + assert.strictEqual(parse('import(`a${ import("b.js") }c`)')[0][0].n, 'a*c'); + assert.strictEqual(parse('import(`a${x}${ import(`y`) }${w}`)')[0][0].n, 'a***'); + // A nested import whose own specifier is an interpolated template is a + // documented pure-JS-port limitation: the outer glob (and its end) are not + // recovered there, while the Wasm and asm.js builds report both. + const [nestedImports] = parse('import(`a${ import(`b${y}`) }c`)'); + assert.strictEqual(nestedImports[1].n, 'b*'); + assert.strictEqual(nestedImports[0].n, js ? undefined : 'a*c'); + if (!js) + assert.notStrictEqual(nestedImports[0].e, 0); + }); + test(`Dynamic import template specifier with escapes and attributes`, () => { // \\${ is an escaped dollar, not a substitution, so the literal is constant. assert.strictEqual(parse('import(`./a\\${x}.js`)')[0][0].n, './a${x}.js'); From 6f36e936767aeaaa293ca05f5ae887d63cb6275c Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Tue, 7 Jul 2026 12:34:31 +0200 Subject: [PATCH 5/5] fix(js): restore the enclosing dynamic import after a nested one closes The pure-JS port pushed the enclosing dynamic import onto a stack when entering a nested import(...), but only popped it on the constant-specifier fast path. Any nested import closing through the main loop's ")" instead overwrote the outer import with null, losing its end/statement_end and dropping its template glob. import(`a${ import(x) }c`) reported the outer specifier as undefined instead of 'a*c', and any interpolated-template nested import corrupted the enclosing entry the same way; this also removes the previously documented limitation where a nested interpolated-template import lost the outer glob only in this port. Restoring from the stack in the ")" handler fixes both. As a side effect, the three glob-recording WeakMap lookups now gate on the current import's argument starting with a backtick first, skipping the lookup entirely for the common non-template dynamic import. --- README.md | 2 +- lexer.js | 61 +++++++++++++++++++++++++------------------------- test/_unit.cjs | 25 +++++++++++++-------- 3 files changed, 48 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 53d39b4..ce7b37c 100755 --- a/README.md +++ b/README.md @@ -242,7 +242,7 @@ For dynamic import expressions, this field will be empty if not a valid JS strin When the entire dynamic import argument is a single template literal, `.n` is reported as a glob: each `${...}` substitution is collapsed to a single `*` (for example `` import(`./locales/${locale}.js`) `` yields `./locales/*.js`). This is a full-build-only feature; the minimal build reports `undefined`. A template concatenated with anything else, or any other expression, resolves to `undefined`. Substitutions are matched by the parser itself, so a `/` inside one is correctly disambiguated as regex or division and does not affect the glob. -The static parts are the raw specifier source: escape sequences are not cooked, and a literal `*` in the specifier is emitted as-is, so a consumer treating `.n` as a glob has to apply its own escaping. As a known limitation, a nested dynamic import whose own specifier is an interpolated template, inside an outer template substitution, yields `undefined` in the `es-module-lexer/js` build where the Wasm and asm.js builds report the glob. +The static parts are the raw specifier source: escape sequences are not cooked, and a literal `*` in the specifier is emitted as-is, so a consumer treating `.n` as a glob has to apply its own escaping. ### Facade Detection diff --git a/lexer.js b/lexer.js index 460c8fa..7fb33c7 100644 --- a/lexer.js +++ b/lexer.js @@ -50,27 +50,21 @@ function readName (impt) { impt.n = readString(s, source.charCodeAt(s - 1)); } -// A dynamic import whose whole argument is a single template literal has no -// constant value, but its static skeleton is a useful glob: each ${...} is -// collapsed to a "*" wildcard (import(`./p/${x}.js`) -> "./p/*.js"). The parser -// records each top-level ${...} substitution's end (impt.spans) as it goes, -// using the real tokenizer to resolve the regex/division ambiguity, so this -// only splices a "*" per substitution and cooks the static parts. A -// concatenation such as import(`a` + b) records no specifier spans and its -// walk hits an inner backtick before the argument end, so n stays undefined. -// At dynamic-import finalization: build the glob only when the specifier -// template's closing backtick was the last token of the argument, so a recorded -// span list means "lone template glob". A concatenation (`a${x}` + b) or a -// trailing operator recorded spans for its first template but does not end on -// that backtick, so it yields undefined. Each span is the end of one top-level -// ${...}; walking from the opening backtick, each becomes a single "*" and is -// skipped, static runs are copied raw so the result matches the wasm/asm builds -// byte-for-byte, and the walk ends at the specifier's unescaped closing -// backtick. Mirrors dropUncommittedGlob + decodeTemplate in src/lexer.c / .ts. +// A dynamic import whose whole argument is a lone template literal globs its +// static skeleton, collapsing each top-level ${...} to a "*" (import(`./p/${x}.js`) +// -> "./p/*.js"). The parser records each substitution's end as it goes, reusing +// the real tokenizer's regex/division disambiguation, so this only splices "*" +// per span. Mirrors dropUncommittedGlob + decodeTemplate in src/lexer.c / .ts. function finalizeDynamicTemplate (impt) { - // Anchor on the last token before ")"/"," (lastTokenPos), not impt.e, so - // whitespace or a comment before the paren does not drop the glob and the - // result matches the wasm/asm builds (which anchor the same way). + // Only a backtick-led argument can be a glob, so gate on it before the WeakMap + // lookup: import(x)/import(fn())/import(a + b) close here too and never recorded + // one. The glob is committed only when the specifier template's closing backtick + // was the last token of the argument (glob.close === lastTokenPos), so a + // concatenation (`a${x}` + b) or a trailing operator yields undefined. Anchoring + // on lastTokenPos, not impt.e, keeps trailing whitespace/comments from dropping + // the glob and matches the wasm/asm builds. + if (source.charCodeAt(impt.s) !== 96/*`*/) + return; const glob = templateGlobs.get(impt); if (glob === undefined || glob.close !== lastTokenPos) return; @@ -202,7 +196,7 @@ export function parse (_source, _name) { finalizeDynamicTemplate(curDynamicImport); } curDynamicImport.se = pos; - curDynamicImport = null; + curDynamicImport = dynamicImportStack.pop(); } break; case 91/*[*/: @@ -241,10 +235,14 @@ export function parse (_source, _name) { if (openTokenDepth-- === templateDepth) { templateDepth = templateStack[--templateStackDepth]; // A top-level ${...} of the recording import's specifier template just - // closed: note the position after "}" for a "*" splice. - const glob = curDynamicImport && templateGlobs.get(curDynamicImport); - if (glob && openTokenDepth + 1 === glob.depth) - glob.ends.push(pos + 1); + // closed: note the position after "}" for a "*" splice. The backtick + // gate skips the WeakMap lookup unless the current import's argument + // starts with a backtick (the only shape that ever records a glob). + if (curDynamicImport && source.charCodeAt(curDynamicImport.s) === 96/*`*/) { + const glob = templateGlobs.get(curDynamicImport); + if (glob && openTokenDepth + 1 === glob.depth) + glob.ends.push(pos + 1); + } templateString(); } else { @@ -820,11 +818,14 @@ function templateString () { if (ch === 96/*`*/) { // The specifier template just closed. Note its closing backtick and stop // recording; finalizeDynamicTemplate keeps the spans only if this backtick - // is the last token of the argument. - const glob = curDynamicImport && templateGlobs.get(curDynamicImport); - if (glob && openTokenDepth + 1 === glob.depth) { - glob.close = pos; - glob.depth = 0; + // is the last token of the argument. The backtick gate skips the WeakMap + // lookup for the common non-template dynamic-import argument. + if (curDynamicImport && source.charCodeAt(curDynamicImport.s) === 96/*`*/) { + const glob = templateGlobs.get(curDynamicImport); + if (glob && openTokenDepth + 1 === glob.depth) { + glob.close = pos; + glob.depth = 0; + } } return; } diff --git a/test/_unit.cjs b/test/_unit.cjs index d7723c7..4196799 100755 --- a/test/_unit.cjs +++ b/test/_unit.cjs @@ -487,17 +487,24 @@ suite('Lexer', () => { // reuse a stale span (a regression for the rt() reload / decoder hang). assert.strictEqual(parse('import(`${a}${b}${c}${d}` + x)')[0][0].n, undefined); // A nested dynamic import inside a substitution keeps the outer glob and is - // itself detected. + // itself detected, whether the inner import closes through the constant fast + // path (a quoted/no-substitution specifier) or the main loop's ")". assert.strictEqual(parse('import(`a${ import("b.js") }c`)')[0][0].n, 'a*c'); assert.strictEqual(parse('import(`a${x}${ import(`y`) }${w}`)')[0][0].n, 'a***'); - // A nested import whose own specifier is an interpolated template is a - // documented pure-JS-port limitation: the outer glob (and its end) are not - // recovered there, while the Wasm and asm.js builds report both. - const [nestedImports] = parse('import(`a${ import(`b${y}`) }c`)'); - assert.strictEqual(nestedImports[1].n, 'b*'); - assert.strictEqual(nestedImports[0].n, js ? undefined : 'a*c'); - if (!js) - assert.notStrictEqual(nestedImports[0].e, 0); + // A nested import that closes through the main loop must restore the outer + // import so its end/glob survive (a regression for the pure-JS-port losing + // e/se/n on any non-constant nested import). + const [[outerId, innerId]] = parse('import(`a${ import(x) }c`)'); + assert.strictEqual(outerId.n, 'a*c'); + assert.notStrictEqual(outerId.e, 0); + assert.notStrictEqual(outerId.se, 0); + assert.strictEqual(innerId.n, undefined); + // The same holds when the inner import's own specifier is an interpolated + // template: all three builds report the outer glob and the inner glob. + const [[outerTmpl, innerTmpl]] = parse('import(`a${ import(`b${y}`) }c`)'); + assert.strictEqual(innerTmpl.n, 'b*'); + assert.strictEqual(outerTmpl.n, 'a*c'); + assert.notStrictEqual(outerTmpl.e, 0); }); test(`Dynamic import template specifier with escapes and attributes`, () => {