feat(loomgen): #loom.line_pattern annotation for line-oriented block-level token production

loomgen/README.md

@@ -25,6 +25,29 @@ string escapes), so write engine-valid MoonBit `re`-dialect regex — e.g. a lit
 hyphen in a class is `\-`, not a bare `-`. A pattern that can match the empty string is
 rejected at generation time.
 
+
+## Line-mode lexer generation (`--line-lexer`, #561)
+
+`#loom.line_pattern("regex")` on token variants — emits `line_lexer.g.mbt` with
+per-mode lexer functions for each `#loom.line_mode` `#loom.lexmode`. Each function
+matches `#loom.line_pattern` regexes against the current line (pos → newline) in
+declaration order. Patterns with a declared `#loom.lexmode("ModeName")` are only
+emitted in that mode's function; patterns without one appear in all line_mode
+functions. Nullary tokens are produced directly; payload-carrying tokens require
+`#loom.custom_lex` for extraction. Generated function names follow the same
+`lex_<lowercase_mode>` convention as `lexer_skeleton.g.mbt` stubs, so they can
+replace the skeleton's `abort("not implemented")` stubs.
+
+Reuses the same supported regex subset, nullability checks, and structural
+validation as `#loom.pattern`. Alternation `|` is rejected (leftmost-match, not
+longest-match). Patterns are wrapped in `^(?:...)` so they should not include
+their own `^` anchor.
+
+Usage:
+```bash
+moon run loomgen --target native -- --line-lexer <path>/line_lexer.g.mbt \
+  --term <term.mbt> <token.mbt> <token_out> <syntax_out>
+```
 ### Supported `#loom.pattern` subset
 
 Literals, *literal-meta* escapes (`\.`, `\-`, `\+`, `\xHH`, …), character classes `[..]`

loomgen/emit_lexer.mbt

@@ -64,11 +64,18 @@ fn collect_lexer_inputs(token_enum : EnumDecl) -> Result[LexerInputs, String] {
       Some(Keyword(kw)) => keywords.push((kw, v.name))
       _ => ()
     }
-    if v.custom_lex is Some(fn_name) {
+    // custom_lex functions for #loom.line_pattern variants belong to the
+    // line-mode lexer, not the character-level lexer — skip them here.
+    if v.custom_lex is Some(fn_name) && v.line_pattern is None {
       custom_lex_fns.push(fn_name)
     }
   }
   for v in variants {
+    // #loom.line_pattern variants are produced by the line-mode lexer, not the
+    // character-level lexer — skip them entirely.
+    if v.line_pattern is Some(_) {
+      continue
+    }
     let is_candidate = v.pattern is Some(_) ||
       v.custom_lex is Some(_) ||
       v.role is Some(Punct(_)) ||
@@ -124,13 +131,13 @@ fn collect_lexer_inputs(token_enum : EnumDecl) -> Result[LexerInputs, String] {
       "#loom.token enum has #loom.keyword variants and more than one pattern-backed #loom.ident variant; keyword post-classification is ambiguous (it keys off a single identifier match). Keep exactly one #loom.ident, or handle the others via #loom.custom_lex",
     )
   }
-  // A #loom.punct whose text equals a keyword shadows it: the punct wins
-  // longest-match at equal length, so the keyword variant would never be
-  // produced. (General #loom.pattern/keyword overlap is the author's
-  // responsibility — it cannot be decided statically.)
+  // Filter out #loom.line_pattern variants — they're produced by the line-mode
+  // lexer, not the character-level lexer.
+  let char_variants = variants.filter(fn(v) { v.line_pattern is None })
+  // Skip line_pattern variants in keyword/punct collision check
   for kw in keywords {
     let (kw_text, kw_name) = kw
-    for v in variants {
+    for v in char_variants {
       if v.role is Some(Punct(lit)) && lit == kw_text {
         return Err(
           "#loom.punct variant " +
@@ -146,7 +153,12 @@ fn collect_lexer_inputs(token_enum : EnumDecl) -> Result[LexerInputs, String] {
       }
     }
   }
-  Ok({ variants, keywords, custom_lex_fns, has_keywords: keywords.length() > 0 })
+  Ok({
+    variants: char_variants,
+    keywords,
+    custom_lex_fns,
+    has_keywords: keywords.length() > 0,
+  })
 }
 
 ///|

loomgen/emit_lexer_wbtest.mbt

@@ -694,3 +694,277 @@ test "emit_lexer golden matches fixtures/default_lex_fixture.g.mbt" {
   }
   assert_eq(generated, golden)
 }
+
+// ── #561 line_pattern / line_mode tests ──────────────────────────────
+
+///|
+/// Helper: parse a fixture file that has both #loom.token and #loom.term enums.
+fn full_enums_of(src : String) -> (EnumDecl, EnumDecl) {
+  match parse_annotations(src) {
+    Ok(a) =>
+      (
+        match a.token_enum {
+          Some(te) => te
+          None => abort("no token enum parsed")
+        },
+        match a.term_enum {
+          Some(te) => te
+          None => abort("no term enum parsed")
+        },
+      )
+    Err(m) => abort("parse failed: " + m)
+  }
+}
+
+///|
+test "emit_line_lexer generates line-mode lexer functions for line_pattern tokens" {
+  let src = read_fixture("fixtures/line_pattern_fixture.mbt")
+  let (te, term_enum) = full_enums_of(src)
+  let generated = match emit_line_lexer(te, Some(term_enum), "@core") {
+    Some(s) => s
+    None => abort("emit_line_lexer returned None")
+  }
+  // Should contain the lex_line_start function
+  // Should contain custom_lex call for HeadingMarker
+  inspect(
+    generated.contains("lex_heading_marker(source, pos, len)"),
+    content="true",
+  )
+  // Should contain pattern checks for each line_pattern token
+  inspect(generated.contains("ListMarkerDash"), content="true")
+  inspect(generated.contains("ListMarkerStar"), content="true")
+  inspect(generated.contains("ListMarkerPlus"), content="true")
+  inspect(generated.contains("ThematicBreak"), content="true")
+  inspect(generated.contains("BlockQuoteMarker"), content="true")
+  // Should contain the regex patterns
+  inspect(generated.contains("re\"^"), content="true")
+  inspect(generated.contains("#{1,6} "), content="true")
+  // Check ListMarkerDash uses escaped hyphen in regex
+  inspect(generated.contains("\\- "), content="true")
+  // Should NOT contain Inline mode function (only LineStart has #loom.line_mode)
+  inspect(generated.contains("fn lex_inline"), content="false")
+  // Should return Done when pos >= length
+  inspect(generated.contains("return (Done, LineStart)"), content="true")
+  // Should have Invalid fallthrough
+  inspect(
+    generated.contains("Not a block-level token (line_pattern)"),
+    content="true",
+  )
+}
+
+///|
+test "emit_line_lexer returns None when no line_pattern variants exist" {
+  let src = read_fixture("fixtures/pattern_lexer_fixture.mbt")
+  match parse_annotations(src) {
+    Ok(a) =>
+      match a.token_enum {
+        Some(te) => {
+          let result = emit_line_lexer(te, a.term_enum, "@core")
+          guard result is None else {
+            fail("expected None for pattern-only fixture")
+          }
+        }
+        None => abort("no token enum")
+      }
+    Err(m) => abort("parse failed: " + m)
+  }
+}
+
+///|
+test "emit_line_lexer returns None when no term enum has line_mode" {
+  let src =
+    #|#loom.token
+    #|pub(all) enum T {
+    #|  #loom.line_pattern("^test")
+    #|  TestToken
+    #|  #loom.error
+    #|  Error(String)
+    #|  #loom.eof
+    #|  E
+    #|} derive(Eq)
+    #|#loom.term
+    #|pub(all) enum Term {
+    #|  #loom.leaf
+    #|  #loom.lexmode("Inline")
+    #|  SomeNode
+    #|  #loom.root
+    #|  Root
+    #|  #loom.errornode
+    #|  ErrorNode
+    #|} derive(Eq)
+  let (te, term_enum) = full_enums_of(src)
+  guard emit_line_lexer(te, Some(term_enum), "@core") is None else {
+    fail("expected None when no mode has line_mode")
+  }
+}
+
+///|
+test "parse_annotations rejects #loom.line_pattern with #loom.keyword" {
+  let src =
+    #|#loom.token
+    #|pub(all) enum T {
+    #|  #loom.keyword("let")
+    #|  #loom.line_pattern("^let")
+    #|  Let
+    #|  #loom.eof
+    #|  E
+    #|} derive(Eq)
+  match parse_annotations(src) {
+    Err(msg) => inspect(msg.contains("#loom.keyword"), content="true")
+    Ok(_) => abort("expected rejection")
+  }
+}
+
+///|
+test "parse_annotations rejects nullable #loom.line_pattern" {
+  let src =
+    #|#loom.token
+    #|pub(all) enum T {
+    #|  #loom.line_pattern("a*")
+    #|  A
+    #|  #loom.eof
+    #|  E
+    #|} derive(Eq)
+  match parse_annotations(src) {
+    Err(msg) => inspect(msg.contains("nullable"), content="true")
+    Ok(_) => abort("expected rejection")
+  }
+}
+
+///|
+test "parse_annotations rejects #loom.line_pattern on #loom.eof" {
+  let src =
+    #|#loom.token
+    #|pub(all) enum T {
+    #|  #loom.line_pattern("^test")
+    #|  Test
+    #|  #loom.eof
+    #|  #loom.line_pattern("^$")
+    #|  E
+    #|} derive(Eq)
+  match parse_annotations(src) {
+    Err(msg) => inspect(msg.contains("#loom.eof"), content="true")
+    Ok(_) => abort("expected rejection")
+  }
+}
+
+///|
+test "parse_annotations rejects #loom.line_pattern with payload but no #loom.custom_lex" {
+  let src =
+    #|#loom.token
+    #|pub(all) enum T {
+    #|  #loom.line_pattern("#{1,6} ")
+    #|  HeadingMarker(Int)
+    #|  #loom.error
+    #|  Error(String)
+    #|  #loom.eof
+    #|  E
+    #|} derive(Eq)
+  match parse_annotations(src) {
+    Err(msg) => inspect(msg.contains("#loom.custom_lex"), content="true")
+    Ok(_) =>
+      abort(
+        "expected rejection for payload-carrying line_pattern without custom_lex",
+      )
+  }
+}
+
+///|
+test "emit_line_lexer golden matches fixtures/line_pattern_fixture.g.mbt" {
+  let src = read_fixture("fixtures/line_pattern_fixture.mbt")
+  let golden = read_fixture("fixtures/line_pattern_fixture.g.mbt")
+  let (te, term_enum) = full_enums_of(src)
+  let generated = match emit_line_lexer(te, Some(term_enum), "@core") {
+    Some(s) => s
+    None => abort("emit_line_lexer returned None")
+  }
+  assert_eq(generated, golden)
+}
+
+///|
+test "emit_line_lexer accepts #loom.line_pattern with #loom.custom_lex" {
+  let src =
+    #|#loom.token
+    #|pub(all) enum T {
+    #|  #loom.line_pattern("#{1,6} ")
+    #|  #loom.custom_lex("lex_heading")
+    #|  HeadingMarker(Int)
+    #|  #loom.line_pattern(">")
+    #|  BlockQuoteMarker
+    #|  #loom.error
+    #|  Error(String)
+    #|  #loom.eof
+    #|  E
+    #|} derive(Eq)
+    #|#loom.term
+    #|pub(all) enum Term {
+    #|  #loom.leaf
+    #|  #loom.lexmode("LineStart")
+    #|  #loom.line_mode
+    #|  Heading
+    #|  #loom.root
+    #|  Root
+    #|  #loom.errornode
+    #|  ErrorNode
+    #|} derive(Eq)
+  let (te, term_enum) = full_enums_of(src)
+  guard emit_line_lexer(te, Some(term_enum), "@core") is Some(_) else {
+    fail("expected Some for valid line_pattern with custom_lex")
+  }
+}
+
+///|
+test "emit_line_lexer filters candidates by mode via lexmode_name" {
+  let src =
+    #|#loom.token
+    #|pub(all) enum T {
+    #|  #loom.line_pattern("#{1,6} ")
+    #|  HeadingMarker
+    #|  #loom.line_pattern(">")
+    #|  #loom.lexmode("BlockQuoteLineStart")
+    #|  BlockQuoteMarker
+    #|  #loom.line_pattern("--- ")
+    #|  ThematicBreak
+    #|  #loom.error
+    #|  Error(String)
+    #|  #loom.eof
+    #|  E
+    #|} derive(Eq)
+    #|#loom.term
+    #|pub(all) enum Term {
+    #|  #loom.leaf
+    #|  #loom.lexmode("LineStart")
+    #|  #loom.line_mode
+    #|  Heading
+    #|  #loom.leaf
+    #|  #loom.lexmode("BlockQuoteLineStart")
+    #|  #loom.line_mode
+    #|  BlockQuote
+    #|  #loom.root
+    #|  Root
+    #|  #loom.errornode
+    #|  ErrorNode
+    #|} derive(Eq)
+  let (te, term_enum) = full_enums_of(src)
+  let generated = match emit_line_lexer(te, Some(term_enum), "@core") {
+    Some(s) => s
+    None => abort("emit_line_lexer returned None")
+  }
+  // lex_line_start should include HeadingMarker and ThematicBreak (no lexmode constraint)
+  inspect(generated.contains("HeadingMarker"), content="true")
+  inspect(generated.contains("ThematicBreak"), content="true")
+  // lex_line_start should NOT include BlockQuoteMarker (assigned to BlockQuoteLineStart)
+  // Check that lex_line_start function body contains neither the name nor its pattern
+  inspect(generated.contains("fn lex_line_start"), content="true")
+  inspect(generated.contains("fn lex_block_quote_line_start"), content="true")
+  // BlockQuoteMarker should only appear in the block_quote_line_start function.
+  let line_start_idx = generated.find("fn lex_line_start")
+  let bq_start_idx = generated.find("fn lex_block_quote_line_start")
+  guard line_start_idx is Some(ls) && bq_start_idx is Some(bq) else {
+    fail("expected both lex functions in output")
+  }
+  let line_start_section = generated[ls:bq]
+  inspect(line_start_section.contains("BlockQuoteMarker"), content="false")
+  let bq_section = generated[bq:]
+  inspect(bq_section.contains("BlockQuoteMarker"), content="true")
+}