feat(lexer): report glob n for interpolated template dynamic imports

README.md

@@ -137,7 +137,8 @@ import { init, parse } from 'es-module-lexer';
   source.slice(imports[3].a, imports[3].se - 1);
 
   // For non-string dynamic import expressions:
-  // - n will be undefined
+  // - n will be undefined, except for a lone template literal: import(`./a/${x}.js`)
+  //   reports n as a glob with each ${...} collapsed to "*" (e.g. "./a/*.js")
   // - a is currently -1 even if there is an import attribute
   // - e is currently the character before the closing )
 
@@ -239,6 +240,10 @@ To handle escape sequences in specifier strings, the `.n` field of imported spec
 
 For dynamic import expressions, this field will be empty if not a valid JS string.
 
+When the entire dynamic import argument is a single template literal, `.n` is reported as a glob: each `${...}` substitution is collapsed to a single `*` (for example `` import(`./locales/${locale}.js`) `` yields `./locales/*.js`). This is a full-build-only feature; the minimal build reports `undefined`. A template concatenated with anything else, or any other expression, resolves to `undefined`. Substitutions are matched by the parser itself, so a `/` inside one is correctly disambiguated as regex or division and does not affect the glob.
+
+The static parts are the raw specifier source: escape sequences are not cooked, and a literal `*` in the specifier is emitted as-is, so a consumer treating `.n` as a glob has to apply its own escaping.
+
 ### Facade Detection
 
 Facade modules that only use import / export syntax can be detected via the third return value (full build only):

chompfile.toml

@@ -137,7 +137,7 @@ run = """
 	${{ EMSDK_PATH }}/emsdk activate 6.0.0
 
 	${{ EMSDK_PATH }}/upstream/emscripten/emcc src/lexer.c -o lib/lexer.wasm -Oz -flto --no-entry -s STANDALONE_WASM=1 \
-	-s EXPORTED_FUNCTIONS="['_parse','_sa','_e','_ri','_re','_it','_is','_ie','_ss','_ip','_se','_ai','_id','_es','_ee','_els','_ele','_ess','_f','_ms','_ra','_rsa','_aks','_ake','_avs','_ave','___heap_base']" \
+	-s EXPORTED_FUNCTIONS="['_parse','_sa','_e','_ri','_re','_it','_is','_ie','_ss','_ip','_se','_ai','_id','_es','_ee','_els','_ele','_ess','_f','_ms','_ra','_rsa','_aks','_ake','_avs','_ave','_rt','_te','_rts','___heap_base']" \
 	-s SUPPORT_LONGJMP=0 -s ALLOW_MEMORY_GROWTH=1 -s ERROR_ON_UNDEFINED_SYMBOLS=0 -Wno-logical-op-parentheses -Wno-parentheses
 """
 
@@ -167,7 +167,7 @@ run = """
 	${{ EMSDK_FASTCOMP_PATH }}/emsdk activate 1.40.1-fastcomp
 
 	${{ EMSDK_FASTCOMP_PATH }}/fastcomp/emscripten/emcc ./src/lexer.c -o lib/lexer.emcc.js -s WASM=0 -Oz --closure 1 \
-	-s EXPORTED_FUNCTIONS="['_parse','_sa','_e','_ri','_re','_it','_is','_ie','_ss','_ip','_se','_ai','_id','_es','_ee','_els','_ele','_ess','_f','_ms','_ra','_rsa','_aks','_ake','_avs','_ave','_setSource']" \
+	-s EXPORTED_FUNCTIONS="['_parse','_sa','_e','_ri','_re','_it','_is','_ie','_ss','_ip','_se','_ai','_id','_es','_ee','_els','_ele','_ess','_f','_ms','_ra','_rsa','_aks','_ake','_avs','_ave','_rt','_te','_rts','_setSource']" \
 	-s AGGRESSIVE_VARIABLE_ELIMINATION=1 -s GLOBAL_BASE=8 \
 	-s ERROR_ON_UNDEFINED_SYMBOLS=0 -s TOTAL_STACK=4997968 -s --separate-asm -Wno-logical-op-parentheses -Wno-parentheses
 """
@@ -187,7 +187,7 @@ run = """
 	${{ EMSDK_FASTCOMP_PATH }}/emsdk activate 1.40.1-fastcomp
 
 	${{ EMSDK_FASTCOMP_PATH }}/fastcomp/emscripten/emcc ./src/lexer.c -o lib/lexer.layout.js -s WASM=0 -Oz --closure 0 \
-	-s EXPORTED_FUNCTIONS="['_parse','_sa','_e','_ri','_re','_it','_is','_ie','_ss','_ip','_se','_ai','_id','_es','_ee','_els','_ele','_ess','_f','_ms','_ra','_rsa','_aks','_ake','_avs','_ave','_setSource']" \
+	-s EXPORTED_FUNCTIONS="['_parse','_sa','_e','_ri','_re','_it','_is','_ie','_ss','_ip','_se','_ai','_id','_es','_ee','_els','_ele','_ess','_f','_ms','_ra','_rsa','_aks','_ake','_avs','_ave','_rt','_te','_rts','_setSource']" \
 	-s AGGRESSIVE_VARIABLE_ELIMINATION=1 -s GLOBAL_BASE=8 \
 	-s ERROR_ON_UNDEFINED_SYMBOLS=0 -s TOTAL_STACK=4997968 -Wno-logical-op-parentheses -Wno-parentheses
 """

lexer.js

@@ -4,6 +4,7 @@ let source, pos, end,
   openTokenPosStack,
   openClassPosStack,
   curDynamicImport,
+  dynamicImportStack,
   templateStackDepth,
   facade,
   lastSlashWasDivision,
@@ -13,6 +14,15 @@ let source, pos, end,
   imports,
   exports,
   exportStatementStart,
+  // Maps a dynamic import to its glob-recording scratch, kept off the public
+  // import objects so the JS result shape stays identical to the wasm build's.
+  // The scratch is { ends, close, depth }: `ends` collects each top-level ${...}
+  // end, `close` the specifier's closing backtick, `depth` the openTokenDepth
+  // its top-level substitutions close at. Per-import, so a nested import(`...`)
+  // records against its own entry and never corrupts an enclosing one. Reuses
+  // the real tokenizer's regex/division disambiguation, which a standalone
+  // ${...} skimmer cannot do.
+  templateGlobs,
   name;
 
 function addImport (ss, s, e, d) {
@@ -40,10 +50,54 @@ function readName (impt) {
   impt.n = readString(s, source.charCodeAt(s - 1));
 }
 
+// A dynamic import whose whole argument is a lone template literal globs its
+// static skeleton, collapsing each top-level ${...} to a "*" (import(`./p/${x}.js`)
+// -> "./p/*.js"). The parser records each substitution's end as it goes, reusing
+// the real tokenizer's regex/division disambiguation, so this only splices "*"
+// per span. Mirrors dropUncommittedGlob + decodeTemplate in src/lexer.c / .ts.
+function finalizeDynamicTemplate (impt) {
+  // Only a backtick-led argument can be a glob, so gate on it before the WeakMap
+  // lookup: import(x)/import(fn())/import(a + b) close here too and never recorded
+  // one. The glob is committed only when the specifier template's closing backtick
+  // was the last token of the argument (glob.close === lastTokenPos), so a
+  // concatenation (`a${x}` + b) or a trailing operator yields undefined. Anchoring
+  // on lastTokenPos, not impt.e, keeps trailing whitespace/comments from dropping
+  // the glob and matches the wasm/asm builds.
+  if (source.charCodeAt(impt.s) !== 96/*`*/)
+    return;
+  const glob = templateGlobs.get(impt);
+  if (glob === undefined || glob.close !== lastTokenPos)
+    return;
+  const spans = glob.ends;
+  const e = impt.e;
+  let out = '', chunkStart = impt.s + 1, index = impt.s + 1, spanIndex = 0, spanEnd = spans[0];
+  // `e` bounds the walk defensively; the parser guarantees an unescaped closing
+  // backtick within it for a committed glob.
+  while (index < e) {
+    const ch = source.charCodeAt(index);
+    if (ch === 96/*`*/)
+      break;
+    if (ch === 92/*\*/) {
+      index += 2;
+      continue;
+    }
+    if (ch === 36/*$*/ && source.charCodeAt(index + 1) === 123/*{*/ && index + 2 <= spanEnd) {
+      out += source.slice(chunkStart, index) + '*';
+      index = chunkStart = spanEnd;
+      spanEnd = ++spanIndex < spans.length ? spans[spanIndex] : -1;
+      continue;
+    }
+    index++;
+  }
+  impt.n = out + source.slice(chunkStart, index);
+}
+
 // Note: parsing is based on the _assumption_ that the source is already valid
 export function parse (_source, _name) {
   openTokenDepth = 0;
   curDynamicImport = null;
+  dynamicImportStack = [];
+  templateGlobs = new WeakMap();
   templateDepth = -1;
   lastTokenPos = -1;
   lastSlashWasDivision = false;
@@ -137,10 +191,12 @@ export function parse (_source, _name) {
           syntaxError();
         openTokenDepth--;
         if (curDynamicImport && curDynamicImport.d === openTokenPosStack[openTokenDepth]) {
-          if (curDynamicImport.e === 0)
+          if (curDynamicImport.e === 0) {
             curDynamicImport.e = pos;
+            finalizeDynamicTemplate(curDynamicImport);
+          }
           curDynamicImport.se = pos;
-          curDynamicImport = null;
+          curDynamicImport = dynamicImportStack.pop();
         }
         break;
       case 91/*[*/:
@@ -154,6 +210,7 @@ export function parse (_source, _name) {
       case 44/*,*/:
         if (curDynamicImport && curDynamicImport.e === 0 && curDynamicImport.d === openTokenPosStack[openTokenDepth - 1]) {
           curDynamicImport.e = lastTokenPos + 1;
+          finalizeDynamicTemplate(curDynamicImport);
           pos++;
           commentWhitespace(true);
           curDynamicImport.a = pos;
@@ -177,6 +234,15 @@ export function parse (_source, _name) {
           syntaxError();
         if (openTokenDepth-- === templateDepth) {
           templateDepth = templateStack[--templateStackDepth];
+          // A top-level ${...} of the recording import's specifier template just
+          // closed: note the position after "}" for a "*" splice. The backtick
+          // gate skips the WeakMap lookup unless the current import's argument
+          // starts with a backtick (the only shape that ever records a glob).
+          if (curDynamicImport && source.charCodeAt(curDynamicImport.s) === 96/*`*/) {
+            const glob = templateGlobs.get(curDynamicImport);
+            if (glob && openTokenDepth + 1 === glob.depth)
+              glob.ends.push(pos + 1);
+          }
           templateString();
         }
         else {
@@ -231,6 +297,11 @@ export function parse (_source, _name) {
         break;
       }
       case 96/*`*/:
+        // A backtick that is the first token of an open dynamic import's
+        // argument opens the specifier template: start recording its top-level
+        // ${...} spans so the glob can be built without re-tokenizing.
+        if (curDynamicImport && curDynamicImport.e === 0 && pos === curDynamicImport.s)
+          templateGlobs.set(curDynamicImport, { ends: [], close: -1, depth: openTokenDepth + 1 });
         templateString();
         break;
     }
@@ -263,6 +334,11 @@ function tryParseImportStatement () {
       // The specifier start is recorded after leading whitespace/comments so it
       // points at the literal, matching the C lexer (src/lexer.c).
       const impt = addImport(startPos, pos, 0, startPos);
+      // Stack the enclosing dynamic import so a nested import(...) inside an
+      // argument restores it on close; without this the outer import's end/se
+      // (and template-glob recording) are lost. Popped at both finalization
+      // sites: the constant path just below and the main-loop ")".
+      dynamicImportStack.push(curDynamicImport);
       curDynamicImport = impt;
       if (ch === 39/*'*/ || ch === 34/*"*/) {
         stringLiteral(ch);
@@ -292,6 +368,7 @@ function tryParseImportStatement () {
         impt.e = pos;
         impt.se = pos;
         readName(impt);
+        curDynamicImport = dynamicImportStack.pop();
       }
       else {
         pos--;
@@ -738,8 +815,20 @@ function templateString () {
       templateDepth = ++openTokenDepth;
       return;
     }
-    if (ch === 96/*`*/)
+    if (ch === 96/*`*/) {
+      // The specifier template just closed. Note its closing backtick and stop
+      // recording; finalizeDynamicTemplate keeps the spans only if this backtick
+      // is the last token of the argument. The backtick gate skips the WeakMap
+      // lookup for the common non-template dynamic-import argument.
+      if (curDynamicImport && source.charCodeAt(curDynamicImport.s) === 96/*`*/) {
+        const glob = templateGlobs.get(curDynamicImport);
+        if (glob && openTokenDepth + 1 === glob.depth) {
+          glob.close = pos;
+          glob.depth = 0;
+        }
+      }
       return;
+    }
     if (ch === 92/*\*/)
       pos++;
   }

src/lexer.asm.js

@@ -57,6 +57,8 @@ export function parse (_source, _name = '@') {
     let n;
     if (asm.ip())
       n = readString(d === -1 ? s : s + 1, source.charCodeAt(d === -1 ? s - 1 : s));
+    else if (!MINIMAL && d !== -1 && source.charCodeAt(s) === 96/*`*/)
+      n = decodeTemplate(s, e);
     let at = null;
     // minimal build drops the parsed attribute list; es-module-shims reads the
     // assertion via source.slice(a, se - 1) instead
@@ -141,6 +143,40 @@ function readString (start, quote) {
   return out;
 }
 
+// Glob for a lone interpolated-template specifier starting at `s`. The parser
+// commits a ${...} span list only for that shape (see lexer.c), so a first rt()
+// of false means "not a glob" (a concatenation such as `a${x}` + b, or a nested
+// template) and yields undefined. Walking from the opening backtick, each
+// top-level ${...} becomes a single "*" and is skipped via its recorded end;
+// static runs are copied raw so the three ports agree byte-for-byte. The walk
+// ends at the specifier's unescaped closing backtick. Kept identical to the
+// wasm build's decodeTemplate (src/lexer.ts).
+function decodeTemplate (s, e) {
+  asm.rts();
+  if (!asm.rt())
+    return;
+  let out = '', chunkStart = s + 1, index = s + 1, spanEnd = asm.te();
+  // `e` bounds the walk defensively; the parser guarantees an unescaped closing
+  // backtick within it for a committed glob.
+  while (index < e) {
+    const ch = source.charCodeAt(index);
+    if (ch === 96/*`*/)
+      break;
+    if (ch === 92/*\*/) {
+      index += 2;
+      continue;
+    }
+    if (ch === 36/*$*/ && source.charCodeAt(index + 1) === 123/*{*/ && index + 2 <= spanEnd) {
+      out += source.slice(chunkStart, index) + '*';
+      index = chunkStart = spanEnd;
+      spanEnd = asm.rt() ? asm.te() : -1;
+      continue;
+    }
+    index++;
+  }
+  return out + source.slice(chunkStart, index);
+}
+
 // Used to read escaped characters
 
 function readEscapedChar () {

src/lexer.c

@@ -70,6 +70,19 @@ static inline __attribute__((always_inline)) bool handleSlash () {
   return false;
 }
 
+// At dynamic-import finalization: keep the recorded ${...} spans only when the
+// specifier template's closing backtick was the last token of the argument, so
+// its non-empty list means "lone template glob". A concatenation (`a${x}` + b)
+// or a trailing operator recorded spans for its first template but does not end
+// on that backtick, so its list is dropped and the decoders report undefined.
+// No-op in the minimal build, which never records or reads a glob.
+static inline __attribute__((always_inline)) void dropUncommittedGlob (Import* import) {
+#ifndef LEXER_MIN
+  if (import->template_close != import->end - 1)
+    import->template_spans = NULL;
+#endif
+}
+
 // Consume one token at the current ch/pos, updating the global tokenizer state.
 // The single source of tokenization: the main loop and skipExpression both call
 // it, so the regex/keyword/import rules never diverge. Comments do not advance
@@ -108,6 +121,7 @@ static inline __attribute__((always_inline)) bool consumeToken (char16_t ch) {
         Import* cur_dynamic_import = dynamicImportStack[dynamicImportStackDepth - 1];
         if (cur_dynamic_import->end == 0) {
           cur_dynamic_import->end = lastTokenPos + 1;
+          dropUncommittedGlob(cur_dynamic_import);
           pos++;
           ch = commentWhitespace(true);
           cur_dynamic_import->attr_index = pos;
@@ -120,8 +134,10 @@ static inline __attribute__((always_inline)) bool consumeToken (char16_t ch) {
       openTokenDepth--;
       if (dynamicImportStackDepth > 0 && openTokenStack[openTokenDepth].token == ImportParen) {
         Import* cur_dynamic_import = dynamicImportStack[dynamicImportStackDepth - 1];
-        if (cur_dynamic_import->end == 0)
+        if (cur_dynamic_import->end == 0) {
           cur_dynamic_import->end = lastTokenPos + 1;
+          dropUncommittedGlob(cur_dynamic_import);
+        }
         cur_dynamic_import->statement_end = pos + 1;
         dynamicImportStackDepth--;
       }
@@ -145,8 +161,29 @@ static inline __attribute__((always_inline)) bool consumeToken (char16_t ch) {
       break;
     case '}':
       if (openTokenDepth == 0) return syntaxError(), false;
-      if (openTokenStack[--openTokenDepth].token == TemplateBrace)
+      if (openTokenStack[--openTokenDepth].token == TemplateBrace) {
+#ifndef LEXER_MIN
+        // A top-level ${...} of an open dynamic import's specifier template just
+        // closed: record the position after "}" so the decoders splice a "*".
+        // The top import is the innermost, so a nested import's substitutions
+        // record against the nested entry, never this one.
+        if (dynamicImportStackDepth > 0) {
+          Import* cur_dynamic_import = dynamicImportStack[dynamicImportStackDepth - 1];
+          if (cur_dynamic_import->specifier_template_depth == openTokenDepth) {
+            TemplateSpan* span = (TemplateSpan*)(analysis_head);
+            analysis_head = analysis_head + sizeof(TemplateSpan);
+            span->end = pos + 1;
+            span->next = NULL;
+            if (cur_dynamic_import->template_span_tail == NULL)
+              cur_dynamic_import->template_spans = span;
+            else
+              cur_dynamic_import->template_span_tail->next = span;
+            cur_dynamic_import->template_span_tail = span;
+          }
+        }
+#endif
         templateString();
+      }
       break;
     case '\'':
     case '"':
@@ -156,6 +193,18 @@ static inline __attribute__((always_inline)) bool consumeToken (char16_t ch) {
       isComment = handleSlash();
       break;
     case '`':
+#ifndef LEXER_MIN
+      // A backtick that opens an active dynamic import's argument (its recorded
+      // specifier start, so leading whitespace/comments don't matter) opens the
+      // specifier template: mark this import so its top-level ${...} spans are
+      // recorded for globbing (see struct TemplateSpan). Per-import, so a nested
+      // import(`...`) in a substitution records against its own entry and never
+      // corrupts an enclosing import.
+      if (dynamicImportStackDepth > 0 &&
+          dynamicImportStack[dynamicImportStackDepth - 1]->end == 0 &&
+          dynamicImportStack[dynamicImportStackDepth - 1]->start == pos)
+        dynamicImportStack[dynamicImportStackDepth - 1]->specifier_template_depth = openTokenDepth + 1;
+#endif
       openTokenStack[openTokenDepth].pos = lastTokenPos;
       openTokenStack[openTokenDepth++].token = Template;
       templateString();
@@ -956,6 +1005,18 @@ void templateString () {
     if (ch == '`') {
       if (openTokenStack[--openTokenDepth].token != Template)
         syntaxError();
+#ifndef LEXER_MIN
+      // The specifier template just closed. Note its closing backtick and stop
+      // recording (depth back to 0); finalization keeps the spans only if this
+      // backtick is the last token of the argument.
+      if (dynamicImportStackDepth > 0) {
+        Import* cur_dynamic_import = dynamicImportStack[dynamicImportStackDepth - 1];
+        if (cur_dynamic_import->specifier_template_depth == openTokenDepth + 1) {
+          cur_dynamic_import->template_close = pos;
+          cur_dynamic_import->specifier_template_depth = 0;
+        }
+      }
+#endif
       return;
     }
     if (ch == '\\')