From 144a8e4edcfa98b786ed48e6e888d2f66248b707 Mon Sep 17 00:00:00 2001
From: Koji Ishimoto <koji.ishimoto@gmail.com>
Date: Mon, 13 Jul 2026 07:23:27 +0900
Subject: [PATCH 1/2] test: add generated line lexer markdown fixture

---
 .../line_lexer_regression/lexmode_def.mbt     |  5 ++
 .../line_lexer_regression/line_lexer.g.mbt    | 47 ++++++++++++
 .../line_lexer_support.mbt                    | 72 +++++++++++++++++++
 .../line_lexer_wbtest.mbt                     | 66 +++++++++++++++++
 fixtures/line_lexer_regression/moon.mod       | 15 ++++
 fixtures/line_lexer_regression/moon.pkg       |  3 +
 .../line_lexer_regression/pkg.generated.mbti  | 33 +++++++++
 fixtures/line_lexer_regression/regenerate.sh  | 19 +++++
 .../src/line_lexer_src.mbt                    | 33 +++++++++
 fixtures/line_lexer_regression/token_def.mbt  | 10 +++
 moon.work                                     |  1 +
 11 files changed, 304 insertions(+)
 create mode 100644 fixtures/line_lexer_regression/lexmode_def.mbt
 create mode 100644 fixtures/line_lexer_regression/line_lexer.g.mbt
 create mode 100644 fixtures/line_lexer_regression/line_lexer_support.mbt
 create mode 100644 fixtures/line_lexer_regression/line_lexer_wbtest.mbt
 create mode 100644 fixtures/line_lexer_regression/moon.mod
 create mode 100644 fixtures/line_lexer_regression/moon.pkg
 create mode 100644 fixtures/line_lexer_regression/pkg.generated.mbti
 create mode 100755 fixtures/line_lexer_regression/regenerate.sh
 create mode 100644 fixtures/line_lexer_regression/src/line_lexer_src.mbt
 create mode 100644 fixtures/line_lexer_regression/token_def.mbt

diff --git a/fixtures/line_lexer_regression/lexmode_def.mbt b/fixtures/line_lexer_regression/lexmode_def.mbt
new file mode 100644
index 00000000..b68f03c4
--- /dev/null
+++ b/fixtures/line_lexer_regression/lexmode_def.mbt
@@ -0,0 +1,5 @@
+///|
+pub(all) enum LexMode {
+  Inline
+  LineStart
+} derive(Eq, Debug)
diff --git a/fixtures/line_lexer_regression/line_lexer.g.mbt b/fixtures/line_lexer_regression/line_lexer.g.mbt
new file mode 100644
index 00000000..652e6f32
--- /dev/null
+++ b/fixtures/line_lexer_regression/line_lexer.g.mbt
@@ -0,0 +1,47 @@
+// Generated by loomgen — do not edit.
+
+///|
+/// Line-mode lexer functions for #loom.line_mode modes.
+/// Each function matches #loom.line_pattern tokens against whole lines.
+fn lex_line_start(
+  source : String,
+  pos : Int,
+) -> (@core.LexStep[Token], LexMode) {
+  if pos >= source.length() {
+    return (Done, LineStart)
+  }
+  let mut line_end = pos
+  while line_end < source.length() {
+    match source[line_end:line_end + 1].to_owned() {
+      "\r" | "\n" => break
+      _ => line_end = line_end + 1
+    }
+  }
+  let rest = source[pos:line_end].view()
+
+  // HeadingMarker
+  if rest =~ (re"^(?:#{1,6} )", after=after) {
+    let len = rest.length() - after.length()
+    return lex_heading_marker(source, pos, len)
+  }
+  // ListMarker
+  if rest =~ (re"^(?:- )", after=after) {
+    let len = rest.length() - after.length()
+    return (
+      Produced(@core.TokenInfo::new(ListMarker, len), next_offset=pos + len),
+      LineStart,
+    )
+  }
+  // BlockQuoteMarker
+  if rest =~ (re"^(?:>)", after=after) {
+    let len = rest.length() - after.length()
+    return (
+      Produced(
+        @core.TokenInfo::new(BlockQuoteMarker, len),
+        next_offset=pos + len,
+      ),
+      LineStart,
+    )
+  }
+  return lex_inline(source, pos)
+}
diff --git a/fixtures/line_lexer_regression/line_lexer_support.mbt b/fixtures/line_lexer_regression/line_lexer_support.mbt
new file mode 100644
index 00000000..dc4d9b31
--- /dev/null
+++ b/fixtures/line_lexer_regression/line_lexer_support.mbt
@@ -0,0 +1,72 @@
+///|
+/// Consume the rest of the current line, including its line ending. This is
+/// the fallback used after a generated line marker has consumed its prefix.
+fn lex_inline(source : String, pos : Int) -> (@core.LexStep[Token], LexMode) {
+  if pos >= source.length() {
+    return (Done, LineStart)
+  }
+  let mut line_end = pos
+  while line_end < source.length() {
+    match source[line_end:line_end + 1].to_owned() {
+      "\r" | "\n" => break
+      _ => line_end = line_end + 1
+    }
+  }
+  let newline_width = if line_end >= source.length() {
+    0
+  } else if line_end + 1 < source.length() &&
+    source[line_end:line_end + 2].to_owned() == "\r\n" {
+    2
+  } else {
+    1
+  }
+  let consumed_end = line_end + newline_width
+  if line_end == pos {
+    return (
+      Produced(
+        @core.TokenInfo::new(BlankLine, newline_width),
+        next_offset=consumed_end,
+      ),
+      LineStart,
+    )
+  }
+  (
+    Produced(
+      @core.TokenInfo::new(
+        Text(source[pos:consumed_end].to_owned()),
+        consumed_end - pos,
+      ),
+      next_offset=consumed_end,
+    ),
+    LineStart,
+  )
+}
+
+///|
+/// Extract the number of leading `#` characters from the generated match.
+fn lex_heading_marker(
+  _source : String,
+  pos : Int,
+  len : Int,
+) -> (@core.LexStep[Token], LexMode) {
+  (
+    Produced(
+      @core.TokenInfo::new(HeadingMarker(len - 1), len),
+      next_offset=pos + len,
+    ),
+    LineStart,
+  )
+}
+
+///|
+/// Fixture-local dispatch mirroring the generated lexer skeleton contract.
+pub fn lex(
+  source : String,
+  pos : Int,
+  mode : LexMode,
+) -> (@core.LexStep[Token], LexMode) {
+  match mode {
+    LineStart => lex_line_start(source, pos)
+    Inline => lex_inline(source, pos)
+  }
+}
diff --git a/fixtures/line_lexer_regression/line_lexer_wbtest.mbt b/fixtures/line_lexer_regression/line_lexer_wbtest.mbt
new file mode 100644
index 00000000..dfd65397
--- /dev/null
+++ b/fixtures/line_lexer_regression/line_lexer_wbtest.mbt
@@ -0,0 +1,66 @@
+///|
+test "generated line-mode lexer tokenizes Markdown lines" {
+  let source = "# Heading\n\n- item\n> quote\n"
+  let lexer : @core.ModeLexer[Token, LexMode] = {
+    lex_step: lex,
+    initial_mode: LineStart,
+  }
+  let (tokens, modes) = @core.tokenize_with_modes(lexer, source, EOF)
+  guard tokens ==
+    [
+      @core.TokenInfo::new(HeadingMarker(1), 2),
+      @core.TokenInfo::new(Text("Heading\n"), 8),
+      @core.TokenInfo::new(BlankLine, 1),
+      @core.TokenInfo::new(ListMarker, 2),
+      @core.TokenInfo::new(Text("item\n"), 5),
+      @core.TokenInfo::new(BlockQuoteMarker, 1),
+      @core.TokenInfo::new(Text(" quote\n"), 7),
+      @core.TokenInfo::new(EOF, 0),
+    ] else {
+    fail("generated lexer token sequence mismatch")
+  }
+  guard @core.build_starts(tokens) == [0, 2, 10, 11, 13, 18, 19, 26] else {
+    fail("generated lexer token positions mismatch")
+  }
+  guard modes ==
+    [
+      LineStart,
+      LineStart,
+      LineStart,
+      LineStart,
+      LineStart,
+      LineStart,
+      LineStart,
+      LineStart,
+    ] else {
+    fail("generated lexer mode sequence mismatch")
+  }
+  match lex_line_start(source, source.length()) {
+    (Done, LineStart) => ()
+    _ => fail("generated line lexer must return Done at EOF")
+  }
+}
+
+///|
+test "generated line-mode lexer preserves CRLF line lengths" {
+  let source = "# H\r\n\r\n- x\r\n"
+  let lexer : @core.ModeLexer[Token, LexMode] = {
+    lex_step: lex,
+    initial_mode: LineStart,
+  }
+  let (tokens, _modes) = @core.tokenize_with_modes(lexer, source, EOF)
+  guard tokens ==
+    [
+      @core.TokenInfo::new(HeadingMarker(1), 2),
+      @core.TokenInfo::new(Text("H\r\n"), 3),
+      @core.TokenInfo::new(BlankLine, 2),
+      @core.TokenInfo::new(ListMarker, 2),
+      @core.TokenInfo::new(Text("x\r\n"), 3),
+      @core.TokenInfo::new(EOF, 0),
+    ] else {
+    fail("generated lexer CRLF sequence mismatch")
+  }
+  guard @core.build_starts(tokens) == [0, 2, 5, 7, 9, 12] else {
+    fail("generated lexer CRLF positions mismatch")
+  }
+}
diff --git a/fixtures/line_lexer_regression/moon.mod b/fixtures/line_lexer_regression/moon.mod
new file mode 100644
index 00000000..cf9b606b
--- /dev/null
+++ b/fixtures/line_lexer_regression/moon.mod
@@ -0,0 +1,15 @@
+name = "dowdiness/loom_tests/line_lexer_regression"
+
+version = "0.1.0"
+
+import {
+  "dowdiness/loom@0.1.0",
+}
+
+readme = "README.md"
+
+repository = "https://github.com/dowdiness/loom"
+
+license = "Apache-2.0"
+
+description = "End-to-end fixture for generated line-mode lexer output"
diff --git a/fixtures/line_lexer_regression/moon.pkg b/fixtures/line_lexer_regression/moon.pkg
new file mode 100644
index 00000000..754273e3
--- /dev/null
+++ b/fixtures/line_lexer_regression/moon.pkg
@@ -0,0 +1,3 @@
+import {
+  "dowdiness/loom/core",
+}
diff --git a/fixtures/line_lexer_regression/pkg.generated.mbti b/fixtures/line_lexer_regression/pkg.generated.mbti
new file mode 100644
index 00000000..e8835f37
--- /dev/null
+++ b/fixtures/line_lexer_regression/pkg.generated.mbti
@@ -0,0 +1,33 @@
+// Generated using `moon info`, DON'T EDIT IT
+package "dowdiness/loom_tests/line_lexer_regression"
+
+import {
+  "dowdiness/loom/core",
+  "moonbitlang/core/debug",
+}
+
+// Values
+pub fn lex(String, Int, LexMode) -> (@core.LexStep[Token], LexMode)
+
+// Errors
+
+// Types and methods
+pub(all) enum LexMode {
+  Inline
+  LineStart
+} derive(Eq, @debug.Debug)
+
+pub(all) enum Token {
+  HeadingMarker(Int)
+  ListMarker
+  BlockQuoteMarker
+  Text(String)
+  BlankLine
+  Error(String)
+  EOF
+} derive(Eq, @debug.Debug)
+
+// Type aliases
+
+// Traits
+
diff --git a/fixtures/line_lexer_regression/regenerate.sh b/fixtures/line_lexer_regression/regenerate.sh
new file mode 100755
index 00000000..1faaa586
--- /dev/null
+++ b/fixtures/line_lexer_regression/regenerate.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+# Regenerate line_lexer.g.mbt from the annotated source.
+# Run from the loom repo root.
+set -euo pipefail
+
+FIXTURE=fixtures/line_lexer_regression
+TMP=$(mktemp -d)
+trap 'rm -rf "$TMP"' EXIT
+mkdir -p "$TMP/token" "$TMP/syntax"
+
+moon run loomgen --target native -- \
+  "$FIXTURE/src/line_lexer_src.mbt" \
+  "$TMP/token" "$TMP/syntax" \
+  --token-qual "" \
+  --syntax-qual "" \
+  --core-qual "@core" \
+  --line-lexer "$FIXTURE/line_lexer.g.mbt"
+
+moon fmt "$FIXTURE/line_lexer.g.mbt"
diff --git a/fixtures/line_lexer_regression/src/line_lexer_src.mbt b/fixtures/line_lexer_regression/src/line_lexer_src.mbt
new file mode 100644
index 00000000..f0b6a9ac
--- /dev/null
+++ b/fixtures/line_lexer_regression/src/line_lexer_src.mbt
@@ -0,0 +1,33 @@
+///|
+#loom.token
+pub(all) enum Token {
+  #loom.line_pattern("#{1,6} ")
+  #loom.custom_lex("lex_heading_marker")
+  HeadingMarker(Int)
+  #loom.line_pattern("- ")
+  ListMarker
+  #loom.line_pattern(">")
+  BlockQuoteMarker
+  #loom.ident
+  Text(String)
+  #loom.error
+  Error(String)
+  #loom.eof
+  EOF
+} derive(Eq, Debug)
+
+#loom.term
+pub(all) enum Term {
+  #loom.leaf
+  #loom.lexmode("LineStart")
+  #loom.line_mode
+  #loom.fallback_lex("lex_inline")
+  Block
+  #loom.leaf
+  #loom.lexmode("Inline")
+  Inline
+  #loom.root
+  Root
+  #loom.errornode
+  ErrorNode
+} derive(Eq, Debug)
diff --git a/fixtures/line_lexer_regression/token_def.mbt b/fixtures/line_lexer_regression/token_def.mbt
new file mode 100644
index 00000000..f5cbf0f5
--- /dev/null
+++ b/fixtures/line_lexer_regression/token_def.mbt
@@ -0,0 +1,10 @@
+///|
+pub(all) enum Token {
+  HeadingMarker(Int)
+  ListMarker
+  BlockQuoteMarker
+  Text(String)
+  BlankLine
+  Error(String)
+  EOF
+} derive(Eq, Debug)
diff --git a/moon.work b/moon.work
index e2d733ae..f84858a3 100644
--- a/moon.work
+++ b/moon.work
@@ -18,4 +18,5 @@ members = [
   "./egraph",
   "./egglog",
   "./fixtures/grammar_ir_regression",
+  "./fixtures/line_lexer_regression",
 ]

From 29b37f7edcc7b2cb4dc5744e6fe2f0c4e3b72808 Mon Sep 17 00:00:00 2001
From: Koji Ishimoto <koji.ishimoto@gmail.com>
Date: Mon, 13 Jul 2026 07:33:18 +0900
Subject: [PATCH 2/2] refactor: share line lexer fixture setup

---
 .../line_lexer_regression/line_lexer_wbtest.mbt | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/fixtures/line_lexer_regression/line_lexer_wbtest.mbt b/fixtures/line_lexer_regression/line_lexer_wbtest.mbt
index dfd65397..e470fdaf 100644
--- a/fixtures/line_lexer_regression/line_lexer_wbtest.mbt
+++ b/fixtures/line_lexer_regression/line_lexer_wbtest.mbt
@@ -1,10 +1,14 @@
+///|
+
+///|
+fn fixture_lexer() -> @core.ModeLexer[Token, LexMode] {
+  { lex_step: lex, initial_mode: LineStart }
+}
+
 ///|
 test "generated line-mode lexer tokenizes Markdown lines" {
   let source = "# Heading\n\n- item\n> quote\n"
-  let lexer : @core.ModeLexer[Token, LexMode] = {
-    lex_step: lex,
-    initial_mode: LineStart,
-  }
+  let lexer = fixture_lexer()
   let (tokens, modes) = @core.tokenize_with_modes(lexer, source, EOF)
   guard tokens ==
     [
@@ -44,10 +48,7 @@ test "generated line-mode lexer tokenizes Markdown lines" {
 ///|
 test "generated line-mode lexer preserves CRLF line lengths" {
   let source = "# H\r\n\r\n- x\r\n"
-  let lexer : @core.ModeLexer[Token, LexMode] = {
-    lex_step: lex,
-    initial_mode: LineStart,
-  }
+  let lexer = fixture_lexer()
   let (tokens, _modes) = @core.tokenize_with_modes(lexer, source, EOF)
   guard tokens ==
     [